mobile-debug-mcp 0.32.4 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +20 -0
- package/.claude-plugin/marketplace.json +14 -0
- package/.cursor-plugin/marketplace.json +16 -0
- package/.github/copilot-instructions.md +7 -0
- package/AGENTS.md +21 -10
- package/README.md +62 -4
- package/agent-plugin/.claude-plugin/plugin.json +20 -0
- package/agent-plugin/.codex-plugin/plugin.json +34 -0
- package/agent-plugin/.cursor-plugin/plugin.json +23 -0
- package/agent-plugin/.mcp.json +14 -0
- package/agent-plugin/README.md +5 -3
- package/agent-plugin/assets/mobile-debug-tools.svg +28 -0
- package/agent-plugin/mcp.json +1 -1
- package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +28 -0
- package/dist/interact/android.js +10 -4
- package/dist/interact/conditional-action.js +68 -0
- package/dist/interact/index.js +54 -14
- package/dist/interact/ios.js +14 -8
- package/dist/manage/android.js +4 -3
- package/dist/manage/ios.js +3 -2
- package/dist/observe/android.js +25 -4
- package/dist/observe/evidence.js +273 -0
- package/dist/observe/index.js +33 -6
- package/dist/observe/ios.js +25 -6
- package/dist/observe/readiness-service.js +115 -0
- package/dist/observe/readiness.js +81 -0
- package/dist/observe/recording.js +165 -0
- package/dist/observe/snapshot-metadata.js +10 -2
- package/dist/server/common.js +9 -0
- package/dist/server/condition-schema.js +32 -0
- package/dist/server/journey-schema.js +3 -2
- package/dist/server/readiness-schema.js +12 -0
- package/dist/server/tool-definitions.js +46 -0
- package/dist/server/tool-handlers.js +99 -17
- package/dist/server-core.js +1 -1
- package/dist/utils/ios/utils.js +4 -4
- package/dist/utils/operation-budget.js +24 -2
- package/dist/utils/resolve-device.js +7 -0
- package/docs/CHANGELOG.md +11 -0
- package/docs/specs/020-conditional-actions/intent.md +62 -0
- package/docs/specs/020-conditional-actions/plan.md +214 -0
- package/docs/specs/020-conditional-actions/spec.md +301 -0
- package/docs/specs/021-app-readiness-signals/intent.md +40 -0
- package/docs/specs/021-app-readiness-signals/plan.md +102 -0
- package/docs/specs/021-app-readiness-signals/spec.md +144 -0
- package/docs/specs/022-correlated-evidence/device-evidence.json +60 -0
- package/docs/specs/022-correlated-evidence/intent.md +37 -0
- package/docs/specs/022-correlated-evidence/modul8-evidence.json +1062 -0
- package/docs/specs/022-correlated-evidence/modul8-selector-fix-evidence.json +336 -0
- package/docs/specs/022-correlated-evidence/plan.md +171 -0
- package/docs/specs/022-correlated-evidence/spec.md +123 -0
- package/docs/specs/process.md +213 -0
- package/docs/tools/TOOLS.md +3 -0
- package/docs/tools/evidence.md +79 -0
- package/docs/tools/interact.md +125 -2
- package/docs/tools/observe.md +63 -0
- package/package.json +1 -1
- package/scripts/agent-plugin.mjs +68 -8
- package/src/interact/android.ts +10 -4
- package/src/interact/conditional-action.ts +75 -0
- package/src/interact/index.ts +42 -14
- package/src/interact/ios.ts +14 -8
- package/src/manage/android.ts +4 -3
- package/src/manage/ios.ts +3 -2
- package/src/observe/android.ts +33 -6
- package/src/observe/evidence.ts +255 -0
- package/src/observe/index.ts +32 -8
- package/src/observe/ios.ts +32 -8
- package/src/observe/readiness-service.ts +89 -0
- package/src/observe/readiness.ts +78 -0
- package/src/observe/snapshot-metadata.ts +12 -1
- package/src/server/common.ts +7 -0
- package/src/server/condition-schema.ts +32 -0
- package/src/server/journey-schema.ts +3 -2
- package/src/server/readiness-schema.ts +12 -0
- package/src/server/tool-definitions.ts +44 -0
- package/src/server/tool-handlers.ts +93 -17
- package/src/server-core.ts +1 -1
- package/src/types.ts +30 -1
- package/src/utils/ios/utils.ts +4 -4
- package/src/utils/operation-budget.ts +22 -2
- package/src/utils/resolve-device.ts +8 -0
- package/test/device/automated/observe/correlated-evidence.smoke.ts +70 -0
- package/test/device/manual/interact/conditional_actions.manual.ts +95 -0
- package/test/device/manual/interact/modul8_selector_tap.manual.ts +63 -0
- package/test/device/manual/interact/readiness.manual.ts +99 -0
- package/test/fixtures/readiness-app/MainActivity.java +23 -0
- package/test/fixtures/readiness-app/README.md +12 -0
- package/test/fixtures/readiness-app/build.ts +37 -0
- package/test/fixtures/readiness-app/main.m +27 -0
- package/test/unit/observe/android_tree_acquisition.test.ts +2 -0
- package/test/unit/observe/capture_debug_snapshot.test.ts +42 -32
- package/test/unit/observe/compact_tree_error.test.ts +24 -0
- package/test/unit/observe/conditional_snapshots.test.ts +33 -0
- package/test/unit/observe/correlated-evidence.test.ts +128 -0
- package/test/unit/observe/ios_tree_acquisition.test.ts +2 -0
- package/test/unit/observe/readiness.test.ts +39 -0
- package/test/unit/observe/readiness_native.test.ts +68 -0
- package/test/unit/server/conditional_actions.test.ts +299 -0
- package/test/unit/server/conditional_schema.test.ts +59 -0
- package/test/unit/server/correlated-evidence.test.ts +194 -0
- package/test/unit/server/readiness.test.ts +170 -0
- package/test/unit/server/selector_tap_target.test.ts +79 -0
- package/test/unit/server/tap_selector.test.ts +2 -0
- package/skills/mobile-debug-tools-usage/SKILL.md +0 -96
- /package/{skills → .agents/skills}/README.md +0 -0
- /package/{skills → .agents/skills}/mcp-builder/SKILL.md +0 -0
- /package/{skills → .agents/skills}/mcp-builder/references/build-flags.md +0 -0
- /package/{skills → .agents/skills}/mcp-builder/references/diagnostics-schema.md +0 -0
- /package/{skills → .agents/skills}/mcp-builder/references/toolchain-details.md +0 -0
- /package/{skills → .agents/skills}/spec-review/SKILL.md +0 -0
- /package/{skills → .agents/skills}/spec-review/references/spec-review-checklist.md +0 -0
- /package/{skills → .agents/skills}/spec-review/references/spec-review-template.md +0 -0
- /package/{skills → .agents/skills}/test-authoring/SKILL.md +0 -0
- /package/{skills → .agents/skills}/test-authoring/references/repo-test-layout.md +0 -0
- /package/{skills → .agents/skills}/test-authoring/references/test-authoring-workflow.md +0 -0
- /package/{skills → .agents/skills}/test-authoring/references/test-quality-checklist.md +0 -0
package/src/interact/android.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { evidenceDispatch } from '../observe/evidence.js'
|
|
2
|
+
import { beforeInputDispatch } from './conditional-action.js'
|
|
1
3
|
import { markDispatchStarted } from '../utils/operation-budget.js'
|
|
2
4
|
import { TapResponse, SwipeResponse, TypeTextResponse, PressBackResponse } from "../types.js"
|
|
3
5
|
import { execAdb, getAndroidDeviceMetadata, getDeviceInfo } from "../utils/android/utils.js"
|
|
@@ -13,8 +15,9 @@ export class AndroidInteract {
|
|
|
13
15
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
14
16
|
|
|
15
17
|
try {
|
|
18
|
+
await beforeInputDispatch('android', deviceInfo.id)
|
|
16
19
|
markDispatchStarted()
|
|
17
|
-
await execAdb(['shell', 'input', 'tap', x.toString(), y.toString()], deviceId, { timeout: timeoutMs })
|
|
20
|
+
await evidenceDispatch({ platform: 'android', id: deviceInfo.id }, () => execAdb(['shell', 'input', 'tap', x.toString(), y.toString()], deviceId, { timeout: timeoutMs }))
|
|
18
21
|
return { device: deviceInfo, success: true, x, y }
|
|
19
22
|
} catch (e) {
|
|
20
23
|
return { device: deviceInfo, success: false, x, y, error: e instanceof Error ? e.message : String(e) }
|
|
@@ -26,8 +29,9 @@ export class AndroidInteract {
|
|
|
26
29
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
27
30
|
|
|
28
31
|
try {
|
|
32
|
+
await beforeInputDispatch('android', deviceInfo.id)
|
|
29
33
|
markDispatchStarted()
|
|
30
|
-
await execAdb(['shell', 'input', 'swipe', x1.toString(), y1.toString(), x2.toString(), y2.toString(), duration.toString()], deviceId, { timeout: timeoutMs })
|
|
34
|
+
await evidenceDispatch({ platform: 'android', id: deviceInfo.id }, () => execAdb(['shell', 'input', 'swipe', x1.toString(), y1.toString(), x2.toString(), y2.toString(), duration.toString()], deviceId, { timeout: timeoutMs }))
|
|
31
35
|
return { device: deviceInfo, success: true, start: [x1, y1], end: [x2, y2], duration }
|
|
32
36
|
} catch (e) {
|
|
33
37
|
return { device: deviceInfo, success: false, start: [x1, y1], end: [x2, y2], duration, error: e instanceof Error ? e.message : String(e) }
|
|
@@ -42,8 +46,9 @@ export class AndroidInteract {
|
|
|
42
46
|
// Encode spaces as %s to ensure proper input handling by adb shell input text
|
|
43
47
|
const encodedText = text.replace(/\s/g, '%s')
|
|
44
48
|
// Note: 'input text' might fail with some characters or if keyboard isn't ready, but it's the standard ADB way.
|
|
49
|
+
await beforeInputDispatch('android', deviceInfo.id)
|
|
45
50
|
markDispatchStarted()
|
|
46
|
-
await execAdb(['shell', 'input', 'text', encodedText], deviceId, { timeout: timeoutMs })
|
|
51
|
+
await evidenceDispatch({ platform: 'android', id: deviceInfo.id }, () => execAdb(['shell', 'input', 'text', encodedText], deviceId, { timeout: timeoutMs }))
|
|
47
52
|
return { device: deviceInfo, success: true, text }
|
|
48
53
|
} catch (e) {
|
|
49
54
|
return { device: deviceInfo, success: false, text, error: e instanceof Error ? e.message : String(e) }
|
|
@@ -55,8 +60,9 @@ export class AndroidInteract {
|
|
|
55
60
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
56
61
|
|
|
57
62
|
try {
|
|
63
|
+
await beforeInputDispatch('android', deviceInfo.id)
|
|
58
64
|
markDispatchStarted()
|
|
59
|
-
await execAdb(['shell', 'input', 'keyevent', '4'], deviceId, { timeout: timeoutMs })
|
|
65
|
+
await evidenceDispatch({ platform: 'android', id: deviceInfo.id }, () => execAdb(['shell', 'input', 'keyevent', '4'], deviceId, { timeout: timeoutMs }))
|
|
60
66
|
return { device: deviceInfo, success: true }
|
|
61
67
|
} catch (e) {
|
|
62
68
|
return { device: deviceInfo, success: false, error: e instanceof Error ? e.message : String(e) }
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
2
|
+
import type { ActionConditionResult } from '../types.js'
|
|
3
|
+
import type { ExpectedState } from '../server/condition-schema.js'
|
|
4
|
+
import { ToolsObserve } from '../observe/index.js'
|
|
5
|
+
import { getSnapshotSessionId, hasSnapshotBaseline } from '../observe/snapshot-metadata.js'
|
|
6
|
+
import { runWithSeparateBudget } from '../utils/operation-budget.js'
|
|
7
|
+
import { ToolsNetwork } from '../network/index.js'
|
|
8
|
+
|
|
9
|
+
export interface ConditionalAction {
|
|
10
|
+
platform: 'android' | 'ios'
|
|
11
|
+
deviceId: string
|
|
12
|
+
expected: ExpectedState
|
|
13
|
+
result: ActionConditionResult
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const context = new AsyncLocalStorage<ConditionalAction>()
|
|
17
|
+
|
|
18
|
+
export function createConditionalAction(platform: 'android' | 'ios', deviceId: string, expected: ExpectedState): ConditionalAction {
|
|
19
|
+
return { platform, deviceId, expected, result: {
|
|
20
|
+
status: 'not_checked',
|
|
21
|
+
expected: { snapshot_session_id: expected.snapshotSessionId, snapshot_revision: expected.snapshotRevision },
|
|
22
|
+
observed: null, elapsed_ms: 0, reason: 'ACTION_NOT_REACHED'
|
|
23
|
+
} }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function withConditionalAction<T>(condition: ConditionalAction, operation: () => Promise<T>) {
|
|
27
|
+
return context.run(condition, operation)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function isConditionBlocked() {
|
|
31
|
+
const status = context.getStore()?.result.status
|
|
32
|
+
return status === 'mismatch' || status === 'unavailable'
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Guarded calls reset the observation window only when input is about to spawn.
|
|
36
|
+
export function notifyActionStart() {
|
|
37
|
+
if (!context.getStore()) ToolsNetwork.notifyActionStart()
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function conditionFailureCode(condition: ConditionalAction) {
|
|
41
|
+
return condition.result.status === 'mismatch' ? 'PRECONDITION_FAILED' as const : 'PRECONDITION_UNAVAILABLE' as const
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export async function beforeInputDispatch(platform: 'android' | 'ios', deviceId: string) {
|
|
45
|
+
const condition = context.getStore()
|
|
46
|
+
if (!condition) return
|
|
47
|
+
const started = performance.now()
|
|
48
|
+
const result = condition.result
|
|
49
|
+
result.status = 'unavailable'
|
|
50
|
+
delete result.reason
|
|
51
|
+
try {
|
|
52
|
+
if (platform !== condition.platform || deviceId !== condition.deviceId) result.reason = 'DEVICE_MISMATCH'
|
|
53
|
+
else if (condition.expected.snapshotSessionId !== getSnapshotSessionId()) result.reason = 'SESSION_MISMATCH'
|
|
54
|
+
else if (!hasSnapshotBaseline(`${platform}:${deviceId}`, condition.expected.snapshotRevision)) result.reason = 'BASELINE_UNAVAILABLE'
|
|
55
|
+
else {
|
|
56
|
+
const tree = await runWithSeparateBudget(condition.expected.timeoutMs ?? 15000, () =>
|
|
57
|
+
ToolsObserve.getUITreeHandler({ platform, deviceId })) as any
|
|
58
|
+
if (tree?.error) result.reason = 'OBSERVATION_FAILED'
|
|
59
|
+
else if (!tree || !Array.isArray(tree.elements) || !Number.isSafeInteger(tree.snapshot_revision) || tree.snapshot_revision < 1 ||
|
|
60
|
+
tree.snapshot_session_id !== getSnapshotSessionId() || !Number.isFinite(tree.captured_at_ms) || tree.captured_at_ms <= 0 ||
|
|
61
|
+
!tree.resolution || !Number.isFinite(tree.resolution.width) || !Number.isFinite(tree.resolution.height) ||
|
|
62
|
+
!(tree.resolution.width > 0) || !(tree.resolution.height > 0)) result.reason = 'OBSERVATION_INVALID'
|
|
63
|
+
else if (tree.device?.platform !== platform || tree.device?.id !== deviceId) result.reason = 'DEVICE_MISMATCH'
|
|
64
|
+
else {
|
|
65
|
+
result.observed = tree
|
|
66
|
+
result.status = tree.snapshot_revision === condition.expected.snapshotRevision ? 'matched' : 'mismatch'
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
} catch (error) {
|
|
70
|
+
result.reason = /TIMEOUT|timed out/i.test(String(error)) ? 'CONDITION_TIMEOUT' : 'OBSERVATION_FAILED'
|
|
71
|
+
}
|
|
72
|
+
result.elapsed_ms = performance.now() - started
|
|
73
|
+
if (result.status !== 'matched') throw new Error(conditionFailureCode(condition))
|
|
74
|
+
ToolsNetwork.notifyActionStart()
|
|
75
|
+
}
|
package/src/interact/index.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { currentReadinessWait, observeReadinessWait, isDeviceUnavailable } from '../observe/readiness.js'
|
|
2
|
+
import { readinessDelay } from '../observe/readiness-service.js'
|
|
3
|
+
import { isConditionBlocked } from './conditional-action.js'
|
|
1
4
|
import { remainingBudget } from '../utils/operation-budget.js'
|
|
2
5
|
import { createHash } from 'crypto'
|
|
3
6
|
import { AndroidInteract } from './android.js';
|
|
@@ -605,6 +608,7 @@ export class ToolsInteract {
|
|
|
605
608
|
}
|
|
606
609
|
|
|
607
610
|
private static async _captureFingerprint(platform: 'android' | 'ios', deviceId?: string): Promise<string | null> {
|
|
611
|
+
if (isConditionBlocked()) return null
|
|
608
612
|
try {
|
|
609
613
|
const fingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as ScreenFingerprintResponse | null
|
|
610
614
|
return fingerprint?.fingerprint ?? null
|
|
@@ -2491,7 +2495,8 @@ export class ToolsInteract {
|
|
|
2491
2495
|
return { found: true, element: outEl, score: scoreVal, confidence: scoreVal, resolution }
|
|
2492
2496
|
}
|
|
2493
2497
|
|
|
2494
|
-
static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId, singleObservation = false, singleAttempt = false, rejectAmbiguous = false }: { selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }, condition?: 'exists'|'not_exists'|'visible'|'clickable', timeout_ms?: number, poll_interval_ms?: number, match?: { index?: number }, retry?: { max_attempts?: number, backoff_ms?: number }, platform?: 'android'|'ios', deviceId?: string, singleObservation?: boolean, singleAttempt?: boolean, rejectAmbiguous?: boolean }) {
|
|
2498
|
+
static async waitForUIHandler({ selector, condition = 'exists', timeout_ms = 60000, poll_interval_ms = 300, match, retry = { max_attempts: 1, backoff_ms: 0 }, platform, deviceId, singleObservation = false, singleAttempt = false, rejectAmbiguous = false, resolveTapTarget = false }: { selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }, condition?: 'exists'|'not_exists'|'visible'|'clickable', timeout_ms?: number, poll_interval_ms?: number, match?: { index?: number }, retry?: { max_attempts?: number, backoff_ms?: number }, platform?: 'android'|'ios', deviceId?: string, singleObservation?: boolean, singleAttempt?: boolean, rejectAmbiguous?: boolean, resolveTapTarget?: boolean }) {
|
|
2499
|
+
const readinessTracker = currentReadinessWait()
|
|
2495
2500
|
const overallStart = Date.now()
|
|
2496
2501
|
const requestedIndex = typeof match?.index === 'number' ? match.index : null
|
|
2497
2502
|
const requested = {
|
|
@@ -2559,7 +2564,11 @@ export class ToolsInteract {
|
|
|
2559
2564
|
remainingBudget()
|
|
2560
2565
|
totalPollCount++
|
|
2561
2566
|
try {
|
|
2567
|
+
const acquisitionStarted = Date.now()
|
|
2562
2568
|
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
|
|
2569
|
+
observeReadinessWait(tree, acquisitionStarted)
|
|
2570
|
+
// A failed acquisition cannot establish that the requested tap target is absent.
|
|
2571
|
+
if (resolveTapTarget && tree?.error && /ACTION_TIMEOUT|timed out/i.test(String(tree.error))) throw new Error('ACTION_TIMEOUT')
|
|
2563
2572
|
const elements = (tree && Array.isArray(tree.elements)) ? tree.elements as any[] : []
|
|
2564
2573
|
|
|
2565
2574
|
const matches: { el: any, idx: number }[] = []
|
|
@@ -2633,7 +2642,8 @@ export class ToolsInteract {
|
|
|
2633
2642
|
conditionMet = visibleFlag
|
|
2634
2643
|
} else conditionMet = false
|
|
2635
2644
|
} else if (condition === 'clickable') {
|
|
2636
|
-
matchedElement = chosen ? (ToolsInteract.
|
|
2645
|
+
matchedElement = chosen ? (resolveTapTarget && (!ToolsInteract._isVisibleElement(chosen.el) || chosen.el.enabled === false)
|
|
2646
|
+
? chosen : ToolsInteract._resolveActionableAncestor(elements, chosen as { el: UiElement, idx: number }) || chosen) : null
|
|
2637
2647
|
if (matchedElement) {
|
|
2638
2648
|
const b = matchedElement.el.bounds
|
|
2639
2649
|
const visibleFlag = !!matchedElement.el.visible && Array.isArray(b) && b.length >= 4 && (b[2] > b[0] && b[3] > b[1])
|
|
@@ -2643,6 +2653,13 @@ export class ToolsInteract {
|
|
|
2643
2653
|
} else conditionMet = false
|
|
2644
2654
|
}
|
|
2645
2655
|
|
|
2656
|
+
// Waiting for a label to exist/be visible is distinct from resolving its
|
|
2657
|
+
// executable control. Reuse the clickable-wait ancestor policy without
|
|
2658
|
+
// promoting hidden or disabled labels into an unrelated enabled target.
|
|
2659
|
+
if (resolveTapTarget && conditionMet && matchedElement &&
|
|
2660
|
+
ToolsInteract._isVisibleElement(matchedElement.el) && matchedElement.el.enabled !== false) {
|
|
2661
|
+
matchedElement = ToolsInteract._resolveActionableAncestor(elements, matchedElement) ?? matchedElement
|
|
2662
|
+
}
|
|
2646
2663
|
const resolvedPlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
|
|
2647
2664
|
const resolvedDeviceId = tree?.device?.id || deviceId
|
|
2648
2665
|
lastMatchedCount = matchedCount
|
|
@@ -2656,7 +2673,7 @@ export class ToolsInteract {
|
|
|
2656
2673
|
if (conditionMet && snapshotFresh) {
|
|
2657
2674
|
if (matchedAt === null) matchedAt = now
|
|
2658
2675
|
stableMatchCount++
|
|
2659
|
-
if (stableMatchCount >= stableObservationCount) {
|
|
2676
|
+
if (stableMatchCount >= stableObservationCount && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
|
|
2660
2677
|
const latency_ms = now - overallStart
|
|
2661
2678
|
const outEl = lastMatchedElement
|
|
2662
2679
|
|
|
@@ -2681,19 +2698,22 @@ export class ToolsInteract {
|
|
|
2681
2698
|
}
|
|
2682
2699
|
|
|
2683
2700
|
} catch (e) {
|
|
2701
|
+
// Deadline expiry does not invalidate the last successfully observed state.
|
|
2684
2702
|
if (e instanceof Error && e.message === 'ACTION_TIMEOUT') throw e
|
|
2703
|
+
if (readinessTracker && !String(e).includes('READINESS_')) readinessTracker.unavailable()
|
|
2704
|
+
if (readinessTracker && isDeviceUnavailable(e)) throw e
|
|
2685
2705
|
// Non-fatal per-poll error; record and continue
|
|
2686
|
-
console.warn('waitForUI: poll error (non-fatal):', e instanceof Error ? e.message : String(e))
|
|
2706
|
+
if (!readinessTracker) console.warn('waitForUI: poll error (non-fatal):', e instanceof Error ? e.message : String(e))
|
|
2687
2707
|
}
|
|
2688
2708
|
|
|
2689
2709
|
if (singleAttempt) break
|
|
2690
2710
|
// Sleep until next poll
|
|
2691
|
-
await new Promise(r => setTimeout(r, effectivePoll || 50))
|
|
2711
|
+
await (readinessTracker ? readinessDelay(effectivePoll || 50) : new Promise(r => setTimeout(r, effectivePoll || 50)))
|
|
2692
2712
|
}
|
|
2693
2713
|
|
|
2694
2714
|
// Attempt timed out; if more attempts allowed, backoff then retry
|
|
2695
2715
|
if (attempts < maxAttempts) {
|
|
2696
|
-
if (backoff > 0) await new Promise(r => setTimeout(r, backoff))
|
|
2716
|
+
if (backoff > 0) await (readinessTracker ? readinessDelay(backoff) : new Promise(r => setTimeout(r, backoff)))
|
|
2697
2717
|
continue
|
|
2698
2718
|
}
|
|
2699
2719
|
|
|
@@ -2710,7 +2730,7 @@ export class ToolsInteract {
|
|
|
2710
2730
|
: ` observed ${lastMatchedCount} match(es)`
|
|
2711
2731
|
return {
|
|
2712
2732
|
status: 'timeout',
|
|
2713
|
-
error: { code: 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout;${matchNote}` },
|
|
2733
|
+
error: { code: readinessTracker ? 'TIMEOUT' : 'ELEMENT_NOT_FOUND', message: `Condition ${condition} not satisfied within timeout;${matchNote}` },
|
|
2714
2734
|
metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
|
|
2715
2735
|
requested,
|
|
2716
2736
|
observed
|
|
@@ -2718,16 +2738,17 @@ export class ToolsInteract {
|
|
|
2718
2738
|
}
|
|
2719
2739
|
|
|
2720
2740
|
} catch (err) {
|
|
2721
|
-
if (err instanceof Error && err.message === 'ACTION_TIMEOUT') throw err
|
|
2741
|
+
if (!readinessTracker && err instanceof Error && err.message === 'ACTION_TIMEOUT') throw err
|
|
2742
|
+
if (readinessTracker && isDeviceUnavailable(err)) throw err
|
|
2722
2743
|
const elapsed = Date.now() - overallStart
|
|
2723
2744
|
return {
|
|
2724
2745
|
status: 'timeout',
|
|
2725
|
-
error: { code: 'INTERNAL_ERROR', message: err instanceof Error ? err.message : String(err) },
|
|
2746
|
+
error: { code: readinessTracker && String(err).includes('ACTION_TIMEOUT') ? 'TIMEOUT' : 'INTERNAL_ERROR', message: readinessTracker ? 'Readiness predicate wait did not complete' : err instanceof Error ? err.message : String(err) },
|
|
2726
2747
|
metrics: { latency_ms: elapsed, poll_count: totalPollCount, attempts },
|
|
2727
2748
|
requested,
|
|
2728
2749
|
observed: {
|
|
2729
2750
|
matched_count: lastMatchedCount,
|
|
2730
|
-
condition_satisfied: false,
|
|
2751
|
+
condition_satisfied: readinessTracker ? lastConditionSatisfied : false,
|
|
2731
2752
|
selected_index: lastMatchedElement?.index ?? null,
|
|
2732
2753
|
last_matched_element: lastMatchedElement
|
|
2733
2754
|
}
|
|
@@ -2821,6 +2842,7 @@ export class ToolsInteract {
|
|
|
2821
2842
|
scope?: 'screen' | 'subtree',
|
|
2822
2843
|
target?: string | null
|
|
2823
2844
|
}): Promise<WaitForUIChangeResponse> {
|
|
2845
|
+
const readinessTracker = currentReadinessWait()
|
|
2824
2846
|
const start = Date.now()
|
|
2825
2847
|
const pollIntervalMs = 300
|
|
2826
2848
|
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 300)
|
|
@@ -2844,7 +2866,10 @@ export class ToolsInteract {
|
|
|
2844
2866
|
|
|
2845
2867
|
while (Date.now() - start < timeout_ms) {
|
|
2846
2868
|
try {
|
|
2869
|
+
if (readinessTracker) remainingBudget()
|
|
2870
|
+
const acquisitionStarted = Date.now()
|
|
2847
2871
|
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
|
|
2872
|
+
observeReadinessWait(tree, acquisitionStarted)
|
|
2848
2873
|
const scopedTree = ToolsInteract._resolveUiChangeScope(tree, scope, target)
|
|
2849
2874
|
if (scopedTree.error) {
|
|
2850
2875
|
lastScopeResolution = scopedTree.resolution
|
|
@@ -2892,7 +2917,7 @@ export class ToolsInteract {
|
|
|
2892
2917
|
}
|
|
2893
2918
|
|
|
2894
2919
|
const stableForMs = candidateSinceMs === null ? 0 : Date.now() - candidateSinceMs
|
|
2895
|
-
if (stabilityWindow === 0 || stableForMs >= stabilityWindow) {
|
|
2920
|
+
if ((stabilityWindow === 0 || stableForMs >= stabilityWindow) && (!readinessTracker || readinessTracker.summary.state === 'settled')) {
|
|
2896
2921
|
return {
|
|
2897
2922
|
success: true,
|
|
2898
2923
|
observed_change: candidateObservedChange ?? observedChange,
|
|
@@ -2915,11 +2940,14 @@ export class ToolsInteract {
|
|
|
2915
2940
|
candidateSinceMs = null
|
|
2916
2941
|
}
|
|
2917
2942
|
}
|
|
2918
|
-
} catch {
|
|
2919
|
-
|
|
2943
|
+
} catch (error) {
|
|
2944
|
+
if (readinessTracker && !String(error).includes('READINESS_')) readinessTracker.unavailable()
|
|
2945
|
+
if (readinessTracker && isDeviceUnavailable(error)) throw error
|
|
2946
|
+
if (readinessTracker && String(error).includes('ACTION_TIMEOUT')) { readinessTracker.deadline(); break }
|
|
2947
|
+
// Default waits retain best-effort polling.
|
|
2920
2948
|
}
|
|
2921
2949
|
|
|
2922
|
-
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
|
|
2950
|
+
await (readinessTracker ? readinessDelay(pollIntervalMs) : new Promise(resolve => setTimeout(resolve, pollIntervalMs)))
|
|
2923
2951
|
}
|
|
2924
2952
|
|
|
2925
2953
|
return {
|
package/src/interact/ios.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { evidenceDispatch } from '../observe/evidence.js'
|
|
2
|
+
import { beforeInputDispatch } from './conditional-action.js'
|
|
1
3
|
import { remainingBudget, markDispatchStarted } from '../utils/operation-budget.js'
|
|
2
4
|
import { spawn } from "child_process"
|
|
3
5
|
import { TapResponse, SwipeResponse } from "../types.js"
|
|
@@ -31,9 +33,11 @@ export class iOSInteract {
|
|
|
31
33
|
args.push('--udid', targetUdid);
|
|
32
34
|
}
|
|
33
35
|
|
|
34
|
-
await
|
|
35
|
-
|
|
36
|
-
|
|
36
|
+
await beforeInputDispatch('ios', device.id)
|
|
37
|
+
const timeout = remainingBudget() || undefined
|
|
38
|
+
markDispatchStarted()
|
|
39
|
+
await evidenceDispatch({ platform: 'ios', id: device.id }, () => new Promise<void>((resolve, reject) => {
|
|
40
|
+
const proc = spawn(getIdbCmd(), args, { timeout });
|
|
37
41
|
let stderr = '';
|
|
38
42
|
proc.stderr.on('data', d => stderr += d.toString());
|
|
39
43
|
proc.on('close', code => {
|
|
@@ -41,7 +45,7 @@ export class iOSInteract {
|
|
|
41
45
|
else reject(new Error(`idb ui tap failed: ${stderr}`));
|
|
42
46
|
});
|
|
43
47
|
proc.on('error', err => reject(err));
|
|
44
|
-
});
|
|
48
|
+
}));
|
|
45
49
|
|
|
46
50
|
return { device, success: true, x, y };
|
|
47
51
|
} catch (e) {
|
|
@@ -73,9 +77,11 @@ export class iOSInteract {
|
|
|
73
77
|
args.push('--udid', targetUdid);
|
|
74
78
|
}
|
|
75
79
|
|
|
76
|
-
await
|
|
77
|
-
|
|
78
|
-
|
|
80
|
+
await beforeInputDispatch('ios', device.id)
|
|
81
|
+
const timeout = remainingBudget() || undefined
|
|
82
|
+
markDispatchStarted()
|
|
83
|
+
await evidenceDispatch({ platform: 'ios', id: device.id }, () => new Promise<void>((resolve, reject) => {
|
|
84
|
+
const proc = spawn(getIdbCmd(), args, { timeout });
|
|
79
85
|
let stderr = '';
|
|
80
86
|
proc.stderr.on('data', d => stderr += d.toString());
|
|
81
87
|
proc.on('close', code => {
|
|
@@ -83,7 +89,7 @@ export class iOSInteract {
|
|
|
83
89
|
else reject(new Error(`idb ui swipe failed: ${stderr}`));
|
|
84
90
|
});
|
|
85
91
|
proc.on('error', err => reject(err));
|
|
86
|
-
});
|
|
92
|
+
}));
|
|
87
93
|
|
|
88
94
|
return { device, success: true, start: [x1, y1], end: [x2, y2], duration };
|
|
89
95
|
} catch (e) {
|
package/src/manage/android.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { evidenceDispatch } from '../observe/evidence.js'
|
|
1
2
|
import { hasFiniteBudget, markDispatchStarted, remainingBudget, runWithBudget } from '../utils/operation-budget.js'
|
|
2
3
|
import { promises as fs } from 'fs'
|
|
3
4
|
import { spawn } from 'child_process'
|
|
@@ -156,7 +157,7 @@ export class AndroidManage {
|
|
|
156
157
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
157
158
|
try {
|
|
158
159
|
markDispatchStarted()
|
|
159
|
-
const output = await execAdb(['shell', 'monkey', '-p', appId, '-c', 'android.intent.category.LAUNCHER', '1'], deviceId, { timeout: timeoutMs })
|
|
160
|
+
const output = await evidenceDispatch({ platform: 'android', id: deviceInfo.id }, () => execAdb(['shell', 'monkey', '-p', appId, '-c', 'android.intent.category.LAUNCHER', '1'], deviceId, { timeout: timeoutMs }))
|
|
160
161
|
const current = await new AndroidObserve().getCurrentScreen(deviceId).catch(() => null)
|
|
161
162
|
return {
|
|
162
163
|
device: deviceInfo,
|
|
@@ -182,7 +183,7 @@ export class AndroidManage {
|
|
|
182
183
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
183
184
|
try {
|
|
184
185
|
markDispatchStarted()
|
|
185
|
-
await execAdb(['shell', 'am', 'force-stop', appId], deviceId)
|
|
186
|
+
await evidenceDispatch({ platform: 'android', id: deviceInfo.id }, () => execAdb(['shell', 'am', 'force-stop', appId], deviceId))
|
|
186
187
|
return { device: deviceInfo, appTerminated: true }
|
|
187
188
|
} catch (e: unknown) {
|
|
188
189
|
const diagnostics = hasFiniteBudget() ? undefined : execAdbWithDiagnostics(['shell', 'am', 'force-stop', appId], deviceId)
|
|
@@ -215,7 +216,7 @@ export class AndroidManage {
|
|
|
215
216
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
216
217
|
try {
|
|
217
218
|
markDispatchStarted()
|
|
218
|
-
const output = await execAdb(['shell', 'pm', 'clear', appId], deviceId)
|
|
219
|
+
const output = await evidenceDispatch({ platform: 'android', id: deviceInfo.id }, () => execAdb(['shell', 'pm', 'clear', appId], deviceId))
|
|
219
220
|
return { device: deviceInfo, dataCleared: output === 'Success' }
|
|
220
221
|
} catch (e: unknown) {
|
|
221
222
|
const diag = execAdbWithDiagnostics(['shell', 'pm', 'clear', appId], deviceId)
|
package/src/manage/ios.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { evidenceDispatch } from '../observe/evidence.js'
|
|
1
2
|
import { hasFiniteBudget, markDispatchStarted, remainingBudget, runWithBudget } from '../utils/operation-budget.js'
|
|
2
3
|
import { promises as fs } from "fs"
|
|
3
4
|
import { spawn, spawnSync } from "child_process"
|
|
@@ -316,7 +317,7 @@ export class iOSManage {
|
|
|
316
317
|
validateBundleId(bundleId)
|
|
317
318
|
try {
|
|
318
319
|
markDispatchStarted()
|
|
319
|
-
const result = await execCommand(['simctl', 'launch', deviceId, bundleId], deviceId)
|
|
320
|
+
const result = await evidenceDispatch({ platform: 'ios', id: deviceId }, () => execCommand(['simctl', 'launch', deviceId, bundleId], deviceId))
|
|
320
321
|
const device = await getIOSDeviceMetadata(deviceId)
|
|
321
322
|
const fingerprint = observe ? await new iOSObserve().getScreenFingerprint(deviceId).catch(() => null) : null
|
|
322
323
|
const pidMatch = result.output.match(/:\s*(\d+)\s*$/)
|
|
@@ -343,7 +344,7 @@ export class iOSManage {
|
|
|
343
344
|
validateBundleId(bundleId)
|
|
344
345
|
try {
|
|
345
346
|
markDispatchStarted()
|
|
346
|
-
await execCommand(['simctl', 'terminate', deviceId, bundleId], deviceId)
|
|
347
|
+
await evidenceDispatch({ platform: 'ios', id: deviceId }, () => execCommand(['simctl', 'terminate', deviceId, bundleId], deviceId))
|
|
347
348
|
const device = await getIOSDeviceMetadata(deviceId)
|
|
348
349
|
return { device, appTerminated: true }
|
|
349
350
|
} catch (e: unknown) {
|
package/src/observe/android.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { captureEvidence } from './evidence.js'
|
|
1
2
|
import { spawn } from "child_process"
|
|
2
3
|
import { XMLParser } from "fast-xml-parser"
|
|
3
4
|
import { GetLogsResponse, CaptureAndroidScreenResponse, GetUITreeResponse, GetCurrentScreenResponse, UIElement, DeviceInfo } from "../types.js"
|
|
@@ -11,14 +12,20 @@ import { deriveSnapshotMetadata } from "./snapshot-metadata.js"
|
|
|
11
12
|
import { remainingBudget } from '../utils/operation-budget.js'
|
|
12
13
|
import { createObservationMeasurement } from './performance-metrics.js'
|
|
13
14
|
|
|
14
|
-
const activeLogStreams: Map<string, { proc: any, file: string }> = new Map()
|
|
15
|
+
const activeLogStreams: Map<string, { proc: any, file: string, device?: { platform: 'android' | 'ios'; id: string } }> = new Map()
|
|
15
16
|
|
|
16
17
|
export class AndroidObserve {
|
|
18
|
+
getLogStreamDevice(sessionId: string = 'default') { return activeLogStreams.get(sessionId)?.device ?? null }
|
|
19
|
+
|
|
17
20
|
async getDeviceMetadata(appId: string, deviceId?: string): Promise<DeviceInfo> {
|
|
18
21
|
return getAndroidDeviceMetadata(appId, deviceId);
|
|
19
22
|
}
|
|
20
23
|
|
|
21
24
|
async getUITree(deviceId?: string): Promise<GetUITreeResponse> {
|
|
25
|
+
return captureEvidence('ui_tree', { platform: 'android', id: deviceId ?? '' }, () => this.collectGetUITree(deviceId))
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
private async collectGetUITree(deviceId?: string): Promise<GetUITreeResponse> {
|
|
22
29
|
const measurement = createObservationMeasurement('android')
|
|
23
30
|
let metadata
|
|
24
31
|
try {
|
|
@@ -122,6 +129,10 @@ export class AndroidObserve {
|
|
|
122
129
|
}
|
|
123
130
|
|
|
124
131
|
async getLogs(filters: { appId?: string, deviceId?: string, pid?: number, tag?: string, level?: string, contains?: string, since_seconds?: number, limit?: number } = {}): Promise<GetLogsResponse> {
|
|
132
|
+
return captureEvidence('logs', { platform: 'android', id: filters.deviceId ?? '' }, () => this.collectGetLogs(filters))
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
private async collectGetLogs(filters: { appId?: string, deviceId?: string, pid?: number, tag?: string, level?: string, contains?: string, since_seconds?: number, limit?: number } = {}): Promise<GetLogsResponse> {
|
|
125
136
|
const { appId, deviceId, pid, tag, level, contains, since_seconds, limit } = filters
|
|
126
137
|
const metadata = await getAndroidDeviceMetadata(appId || "", deviceId)
|
|
127
138
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
@@ -221,6 +232,10 @@ export class AndroidObserve {
|
|
|
221
232
|
}
|
|
222
233
|
|
|
223
234
|
async captureScreen(deviceId?: string): Promise<CaptureAndroidScreenResponse> {
|
|
235
|
+
return captureEvidence('screenshot', { platform: 'android', id: deviceId ?? '' }, () => this.collectCaptureScreen(deviceId))
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
private async collectCaptureScreen(deviceId?: string): Promise<CaptureAndroidScreenResponse> {
|
|
224
239
|
const metadata = await getAndroidDeviceMetadata("", deviceId)
|
|
225
240
|
const deviceInfo: DeviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
226
241
|
|
|
@@ -349,6 +364,10 @@ export class AndroidObserve {
|
|
|
349
364
|
}
|
|
350
365
|
|
|
351
366
|
async getCurrentScreen(deviceId?: string): Promise<GetCurrentScreenResponse> {
|
|
367
|
+
return captureEvidence('activity', { platform: 'android', id: deviceId ?? '' }, () => this.collectGetCurrentScreen(deviceId))
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
private async collectGetCurrentScreen(deviceId?: string): Promise<GetCurrentScreenResponse> {
|
|
352
371
|
const metadata = await getAndroidDeviceMetadata("", deviceId)
|
|
353
372
|
const deviceInfo = getDeviceInfo(deviceId || 'default', metadata)
|
|
354
373
|
|
|
@@ -410,13 +429,17 @@ export class AndroidObserve {
|
|
|
410
429
|
}
|
|
411
430
|
}
|
|
412
431
|
|
|
413
|
-
async getScreenFingerprint(deviceId?: string): Promise<{ fingerprint: string | null; activity?: string; error?: string }> {
|
|
432
|
+
async getScreenFingerprint(deviceId?: string): Promise<{ fingerprint: string | null; activity?: string; error?: string; captured_at_ms?: number }> {
|
|
433
|
+
return captureEvidence('fingerprint', { platform: 'android', id: deviceId ?? '' }, () => this.collectGetScreenFingerprint(deviceId))
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
private async collectGetScreenFingerprint(deviceId?: string): Promise<{ fingerprint: string | null; activity?: string; error?: string; captured_at_ms?: number }> {
|
|
414
437
|
try {
|
|
415
438
|
const tree = await this.getUITree(deviceId)
|
|
416
439
|
if (!tree || (tree as any).error) return { fingerprint: null, error: (tree as any).error }
|
|
417
440
|
|
|
418
441
|
const current = await this.getCurrentScreen(deviceId).catch(() => null)
|
|
419
|
-
return computeScreenFingerprint(tree, current, 'android', 50)
|
|
442
|
+
return { ...computeScreenFingerprint(tree, current, 'android', 50), captured_at_ms: tree.captured_at_ms }
|
|
420
443
|
} catch (e) {
|
|
421
444
|
return { fingerprint: null, error: e instanceof Error ? e.message : String(e) }
|
|
422
445
|
}
|
|
@@ -436,7 +459,7 @@ export class AndroidObserve {
|
|
|
436
459
|
activeLogStreams.delete(sessionId)
|
|
437
460
|
}
|
|
438
461
|
|
|
439
|
-
const args = ['logcat', `--pid=${pid}`, filter]
|
|
462
|
+
const args = [...(deviceId ? ['-s', deviceId] : []), 'logcat', `--pid=${pid}`, filter]
|
|
440
463
|
const proc = spawn(getAdbCmd(), args)
|
|
441
464
|
|
|
442
465
|
const tmpDir = process.env.TMPDIR || '/tmp'
|
|
@@ -463,10 +486,10 @@ export class AndroidObserve {
|
|
|
463
486
|
|
|
464
487
|
proc.on('close', () => {
|
|
465
488
|
stream.end()
|
|
466
|
-
activeLogStreams.delete(sessionId)
|
|
489
|
+
if (activeLogStreams.get(sessionId)?.proc === proc) activeLogStreams.delete(sessionId)
|
|
467
490
|
})
|
|
468
491
|
|
|
469
|
-
activeLogStreams.set(sessionId, { proc, file })
|
|
492
|
+
activeLogStreams.set(sessionId, { proc, file, device: { platform: 'android', id: deviceId ?? '' } })
|
|
470
493
|
return { success: true, stream_started: true }
|
|
471
494
|
} catch (e) {
|
|
472
495
|
return { success: false, error: e instanceof Error ? e.message : String(e) }
|
|
@@ -482,6 +505,10 @@ export class AndroidObserve {
|
|
|
482
505
|
}
|
|
483
506
|
|
|
484
507
|
async readLogStream(sessionId: string = 'default', limit: number = 100, since?: string) {
|
|
508
|
+
return captureEvidence('logs', this.getLogStreamDevice(sessionId), () => this.collectReadLogStream(sessionId, limit, since))
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
private async collectReadLogStream(sessionId: string = 'default', limit: number = 100, since?: string) {
|
|
485
512
|
const entry = activeLogStreams.get(sessionId)
|
|
486
513
|
let file: string | undefined
|
|
487
514
|
if (entry && entry.file) file = entry.file
|