mobile-debug-mcp 0.29.0 → 0.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +13 -0
- package/README.md +44 -21
- package/dist/interact/index.js +359 -46
- package/dist/observe/index.js +1 -0
- package/dist/observe/snapshot-metadata.js +62 -1
- package/dist/server/tool-definitions.js +8 -3
- package/dist/server/tool-handlers.js +4 -2
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +11 -0
- package/docs/ROADMAP.md +18 -7
- package/docs/rfcs/013-wait-and-synchronization-reliability.md +870 -0
- package/docs/rfcs/014-actionability-resolution.md +394 -0
- package/docs/specs/mcp-tooling-spec-v1.md +28 -0
- package/docs/tools/interact.md +6 -0
- package/package.json +1 -1
- package/src/interact/index.ts +444 -45
- package/src/observe/index.ts +1 -0
- package/src/observe/snapshot-metadata.ts +69 -2
- package/src/server/tool-definitions.ts +8 -3
- package/src/server/tool-handlers.ts +4 -2
- package/src/server-core.ts +1 -1
- package/src/types.ts +24 -0
- package/test/unit/interact/adjust_control.test.ts +104 -0
- package/test/unit/interact/subtree_collection.test.ts +24 -0
- package/test/unit/interact/tap_element.test.ts +71 -0
- package/test/unit/interact/wait_for_ui_change.test.ts +189 -45
- package/test/unit/observe/snapshot_metadata.test.ts +67 -0
|
@@ -2,74 +2,218 @@ import assert from 'assert'
|
|
|
2
2
|
import { ToolsInteract } from '../../../src/interact/index.js'
|
|
3
3
|
import { ToolsObserve } from '../../../src/observe/index.js'
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
type UiTree = {
|
|
6
|
+
device: { platform: 'android', id: string, osVersion: string, model: string, simulator: boolean }
|
|
7
|
+
screen: string
|
|
8
|
+
resolution: { width: number, height: number }
|
|
9
|
+
elements: Array<{
|
|
10
|
+
text: string
|
|
11
|
+
type: string
|
|
12
|
+
bounds: number[]
|
|
13
|
+
visible: boolean
|
|
14
|
+
enabled?: boolean
|
|
15
|
+
clickable?: boolean
|
|
16
|
+
stable_id?: string
|
|
17
|
+
parentId?: number
|
|
18
|
+
children?: number[]
|
|
19
|
+
state?: Record<string, unknown> | null
|
|
20
|
+
}>
|
|
21
|
+
snapshot_revision: number
|
|
22
|
+
captured_at_ms: number
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function makeTree(screen: string, revision: number): UiTree {
|
|
26
|
+
return {
|
|
27
|
+
device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
|
|
28
|
+
screen,
|
|
29
|
+
resolution: { width: 1080, height: 2400 },
|
|
30
|
+
elements: [{ text: screen, type: 'TextView', bounds: [0, 0, 100, 40], visible: true }],
|
|
31
|
+
snapshot_revision: revision,
|
|
32
|
+
captured_at_ms: 1000 + revision
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function makeScopedTree(title: string, status: string, revision: number): UiTree {
|
|
37
|
+
return {
|
|
38
|
+
device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
|
|
39
|
+
screen: 'Scoped',
|
|
40
|
+
resolution: { width: 1080, height: 2400 },
|
|
41
|
+
elements: [
|
|
42
|
+
{ text: 'Root', type: 'FrameLayout', bounds: [0, 0, 1080, 2400], visible: true, enabled: true, stable_id: 'root', children: [1, 2] },
|
|
43
|
+
{ text: title, type: 'TextView', bounds: [0, 0, 1080, 200], visible: true, enabled: true, stable_id: 'title', parentId: 0, state: { text_value: title } },
|
|
44
|
+
{ text: status, type: 'TextView', bounds: [0, 200, 1080, 260], visible: true, enabled: true, stable_id: 'status', parentId: 0, state: { text_value: status } }
|
|
45
|
+
],
|
|
46
|
+
snapshot_revision: revision,
|
|
47
|
+
captured_at_ms: 1000 + revision
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function runScenario({
|
|
52
|
+
snapshots,
|
|
53
|
+
expectedChange,
|
|
54
|
+
timeoutMs,
|
|
55
|
+
stabilityWindowMs,
|
|
56
|
+
stepMs
|
|
57
|
+
}: {
|
|
58
|
+
snapshots: UiTree[]
|
|
59
|
+
expectedChange: 'hierarchy_diff' | 'text_change' | 'state_change'
|
|
60
|
+
timeoutMs: number
|
|
61
|
+
stabilityWindowMs?: number
|
|
62
|
+
stepMs: number
|
|
63
|
+
}) {
|
|
6
64
|
const originalGetUITreeHandler = (ToolsObserve as any).getUITreeHandler
|
|
65
|
+
const originalSetTimeout = globalThis.setTimeout
|
|
66
|
+
const originalDateNow = Date.now
|
|
67
|
+
let now = 0
|
|
68
|
+
let calls = 0
|
|
69
|
+
const delays: number[] = []
|
|
7
70
|
|
|
8
71
|
try {
|
|
9
|
-
|
|
72
|
+
;(Date as any).now = () => now
|
|
73
|
+
;(globalThis as any).setTimeout = (callback: (...args: any[]) => void, delay?: number) => {
|
|
74
|
+
delays.push(typeof delay === 'number' ? delay : 0)
|
|
75
|
+
now += stepMs
|
|
76
|
+
callback()
|
|
77
|
+
return 0
|
|
78
|
+
}
|
|
79
|
+
|
|
10
80
|
;(ToolsObserve as any).getUITreeHandler = async () => {
|
|
11
81
|
calls++
|
|
12
|
-
|
|
13
|
-
return {
|
|
14
|
-
device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
|
|
15
|
-
screen: 'Loading',
|
|
16
|
-
resolution: { width: 1080, height: 2400 },
|
|
17
|
-
elements: [{ text: 'Loading', type: 'TextView', bounds: [0, 0, 100, 40], visible: true }],
|
|
18
|
-
snapshot_revision: 1,
|
|
19
|
-
captured_at_ms: 1000
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
return {
|
|
24
|
-
device: { platform: 'android', id: 'mock', osVersion: '14', model: 'Pixel', simulator: true },
|
|
25
|
-
screen: 'Loaded',
|
|
26
|
-
resolution: { width: 1080, height: 2400 },
|
|
27
|
-
elements: [{ text: 'Loaded', type: 'TextView', bounds: [0, 0, 100, 40], visible: true }],
|
|
28
|
-
snapshot_revision: 2,
|
|
29
|
-
captured_at_ms: 2000
|
|
30
|
-
}
|
|
82
|
+
return snapshots[Math.min(calls - 1, snapshots.length - 1)]
|
|
31
83
|
}
|
|
32
84
|
|
|
33
|
-
const
|
|
85
|
+
const result = await ToolsInteract.waitForUIChangeHandler({
|
|
34
86
|
platform: 'android',
|
|
35
87
|
deviceId: 'mock',
|
|
36
|
-
expected_change:
|
|
37
|
-
timeout_ms:
|
|
38
|
-
stability_window_ms:
|
|
88
|
+
expected_change: expectedChange,
|
|
89
|
+
timeout_ms: timeoutMs,
|
|
90
|
+
stability_window_ms: stabilityWindowMs
|
|
39
91
|
})
|
|
40
92
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
93
|
+
return { result, calls, delays }
|
|
94
|
+
} finally {
|
|
95
|
+
;(ToolsObserve as any).getUITreeHandler = originalGetUITreeHandler
|
|
96
|
+
;(globalThis as any).setTimeout = originalSetTimeout
|
|
97
|
+
;(Date as any).now = originalDateNow
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function runScopedScenario() {
|
|
102
|
+
const snapshots = [
|
|
103
|
+
makeScopedTree('Title', 'Status 1', 1),
|
|
104
|
+
makeScopedTree('Title', 'Status 2', 2),
|
|
105
|
+
makeScopedTree('Title Ready', 'Status 2', 3),
|
|
106
|
+
makeScopedTree('Title Ready', 'Status 2', 4)
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
const originalGetUITreeHandler = (ToolsObserve as any).getUITreeHandler
|
|
110
|
+
const originalSetTimeout = globalThis.setTimeout
|
|
111
|
+
const originalDateNow = Date.now
|
|
112
|
+
let now = 0
|
|
113
|
+
let calls = 0
|
|
114
|
+
|
|
115
|
+
try {
|
|
116
|
+
;(Date as any).now = () => now
|
|
117
|
+
;(globalThis as any).setTimeout = (callback: (...args: any[]) => void) => {
|
|
118
|
+
now += 1
|
|
119
|
+
callback()
|
|
120
|
+
return 0
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
;(ToolsObserve as any).getUITreeHandler = async () => snapshots[Math.min(calls++, snapshots.length - 1)]
|
|
124
|
+
|
|
125
|
+
const resolution = await ToolsInteract.waitForUIHandler({
|
|
126
|
+
selector: { text: 'Title' },
|
|
127
|
+
condition: 'exists',
|
|
128
|
+
timeout_ms: 1000,
|
|
129
|
+
poll_interval_ms: 1,
|
|
130
|
+
platform: 'android',
|
|
131
|
+
deviceId: 'mock'
|
|
53
132
|
})
|
|
54
133
|
|
|
55
|
-
const
|
|
134
|
+
const targetId = resolution?.element?.elementId
|
|
135
|
+
assert.ok(targetId)
|
|
136
|
+
|
|
137
|
+
calls = 0
|
|
138
|
+
now = 0
|
|
139
|
+
const result = await ToolsInteract.waitForUIChangeHandler({
|
|
56
140
|
platform: 'android',
|
|
57
141
|
deviceId: 'mock',
|
|
58
|
-
expected_change: '
|
|
59
|
-
|
|
142
|
+
expected_change: 'text_change',
|
|
143
|
+
scope: 'subtree',
|
|
144
|
+
target: targetId,
|
|
145
|
+
timeout_ms: 2000,
|
|
60
146
|
stability_window_ms: 1
|
|
61
147
|
})
|
|
62
148
|
|
|
63
|
-
|
|
64
|
-
assert.strictEqual(timeout.observed_change, null)
|
|
65
|
-
assert.strictEqual(timeout.timeout, true)
|
|
66
|
-
|
|
67
|
-
console.log('wait_for_ui_change tests passed')
|
|
149
|
+
return { result, targetId, calls }
|
|
68
150
|
} finally {
|
|
69
151
|
;(ToolsObserve as any).getUITreeHandler = originalGetUITreeHandler
|
|
152
|
+
;(globalThis as any).setTimeout = originalSetTimeout
|
|
153
|
+
;(Date as any).now = originalDateNow
|
|
70
154
|
}
|
|
71
155
|
}
|
|
72
156
|
|
|
157
|
+
async function run() {
|
|
158
|
+
const success = await runScenario({
|
|
159
|
+
snapshots: [makeTree('Loading', 1), makeTree('Loaded', 2)],
|
|
160
|
+
expectedChange: 'text_change',
|
|
161
|
+
timeoutMs: 1500,
|
|
162
|
+
stabilityWindowMs: 1,
|
|
163
|
+
stepMs: 1
|
|
164
|
+
})
|
|
165
|
+
|
|
166
|
+
assert.strictEqual(success.result.success, true)
|
|
167
|
+
assert.strictEqual(success.result.observed_change, 'text_change')
|
|
168
|
+
assert.strictEqual(success.result.snapshot_revision, 2)
|
|
169
|
+
assert.strictEqual(success.result.timeout, false)
|
|
170
|
+
|
|
171
|
+
const timeout = await runScenario({
|
|
172
|
+
snapshots: [makeTree('Static', 9)],
|
|
173
|
+
expectedChange: 'state_change',
|
|
174
|
+
timeoutMs: 5,
|
|
175
|
+
stabilityWindowMs: 1,
|
|
176
|
+
stepMs: 1
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
assert.strictEqual(timeout.result.success, false)
|
|
180
|
+
assert.strictEqual(timeout.result.observed_change, null)
|
|
181
|
+
assert.strictEqual(timeout.result.timeout, true)
|
|
182
|
+
|
|
183
|
+
const defaultWindow = await runScenario({
|
|
184
|
+
snapshots: [makeTree('Loading', 1), makeTree('Loaded', 2), makeTree('Loaded', 3), makeTree('Loaded', 4)],
|
|
185
|
+
expectedChange: 'text_change',
|
|
186
|
+
timeoutMs: 2000,
|
|
187
|
+
stepMs: 260
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
assert.strictEqual(defaultWindow.result.success, true)
|
|
191
|
+
assert.strictEqual(defaultWindow.calls, 4)
|
|
192
|
+
assert.deepStrictEqual(defaultWindow.delays, [300, 300, 300])
|
|
193
|
+
|
|
194
|
+
const scoped = await runScopedScenario()
|
|
195
|
+
assert.strictEqual(scoped.result.success, true)
|
|
196
|
+
assert.strictEqual(scoped.result.scope, 'subtree')
|
|
197
|
+
assert.strictEqual(scoped.result.target, scoped.targetId)
|
|
198
|
+
assert.strictEqual(scoped.result.observed_change, 'text_change')
|
|
199
|
+
assert.strictEqual(scoped.result.change_summary?.added_elements, 0)
|
|
200
|
+
assert.strictEqual(scoped.result.change_summary?.removed_elements, 0)
|
|
201
|
+
assert.strictEqual(scoped.result.stability_state, 'stable')
|
|
202
|
+
|
|
203
|
+
const resetWindow = await runScenario({
|
|
204
|
+
snapshots: [makeTree('Loading', 1), makeTree('Loaded', 2), makeTree('Loaded-again', 3), makeTree('Loaded-again', 4), makeTree('Loaded-again', 5)],
|
|
205
|
+
expectedChange: 'text_change',
|
|
206
|
+
timeoutMs: 2000,
|
|
207
|
+
stabilityWindowMs: 300,
|
|
208
|
+
stepMs: 150
|
|
209
|
+
})
|
|
210
|
+
|
|
211
|
+
assert.strictEqual(resetWindow.result.success, true)
|
|
212
|
+
assert.strictEqual(resetWindow.calls, 5)
|
|
213
|
+
|
|
214
|
+
console.log('wait_for_ui_change tests passed')
|
|
215
|
+
}
|
|
216
|
+
|
|
73
217
|
run().catch((error) => {
|
|
74
218
|
console.error(error)
|
|
75
219
|
process.exit(1)
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import assert from 'assert'
|
|
2
|
+
import { deriveSnapshotMetadata, resetSnapshotMetadataForTests } from '../../../src/observe/snapshot-metadata.js'
|
|
3
|
+
|
|
4
|
+
async function run() {
|
|
5
|
+
console.log('Starting snapshot_metadata unit tests...')
|
|
6
|
+
|
|
7
|
+
resetSnapshotMetadataForTests()
|
|
8
|
+
|
|
9
|
+
const deviceKey = 'android:mock'
|
|
10
|
+
const first = deriveSnapshotMetadata(deviceKey, {
|
|
11
|
+
screen: 'Home',
|
|
12
|
+
resolution: { width: 100, height: 200 },
|
|
13
|
+
elements: [
|
|
14
|
+
{
|
|
15
|
+
text: 'Alpha',
|
|
16
|
+
contentDescription: null,
|
|
17
|
+
resourceId: 'row_1',
|
|
18
|
+
type: 'TextView',
|
|
19
|
+
clickable: false,
|
|
20
|
+
enabled: true,
|
|
21
|
+
visible: true,
|
|
22
|
+
bounds: [0, 0, 10, 10],
|
|
23
|
+
state: null,
|
|
24
|
+
stable_id: 'stable-row'
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}, 'ui_tree')
|
|
28
|
+
|
|
29
|
+
assert.strictEqual(first.snapshot_revision, 1)
|
|
30
|
+
assert.strictEqual(first.snapshot_delta, null)
|
|
31
|
+
|
|
32
|
+
const second = deriveSnapshotMetadata(deviceKey, {
|
|
33
|
+
screen: 'Home',
|
|
34
|
+
resolution: { width: 100, height: 200 },
|
|
35
|
+
elements: [
|
|
36
|
+
{
|
|
37
|
+
text: 'Beta',
|
|
38
|
+
contentDescription: null,
|
|
39
|
+
resourceId: 'row_1',
|
|
40
|
+
type: 'TextView',
|
|
41
|
+
clickable: false,
|
|
42
|
+
enabled: true,
|
|
43
|
+
visible: true,
|
|
44
|
+
bounds: [0, 0, 10, 10],
|
|
45
|
+
state: null,
|
|
46
|
+
stable_id: 'stable-row'
|
|
47
|
+
}
|
|
48
|
+
]
|
|
49
|
+
}, 'ui_tree')
|
|
50
|
+
|
|
51
|
+
assert.strictEqual(second.snapshot_revision, 2)
|
|
52
|
+
assert.deepStrictEqual(second.snapshot_delta, {
|
|
53
|
+
previous_snapshot_revision: 1,
|
|
54
|
+
added_elements: 0,
|
|
55
|
+
removed_elements: 0,
|
|
56
|
+
mutated_elements: 1,
|
|
57
|
+
total_elements: 1
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
resetSnapshotMetadataForTests()
|
|
61
|
+
console.log('snapshot_metadata unit tests passed')
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
run().catch((error) => {
|
|
65
|
+
console.error(error)
|
|
66
|
+
process.exit(1)
|
|
67
|
+
})
|