mobile-debug-mcp 0.31.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/android.js +13 -8
- package/dist/interact/index.js +33 -22
- package/dist/interact/ios.js +5 -2
- package/dist/manage/android.js +20 -10
- package/dist/manage/ios.js +19 -10
- package/dist/observe/android.js +6 -1
- package/dist/observe/index.js +22 -5
- package/dist/observe/ios.js +7 -2
- package/dist/observe/snapshot-metadata.js +94 -15
- package/dist/server/common.js +14 -7
- package/dist/server/journey-schema.js +60 -0
- package/dist/server/tool-definitions.js +59 -6
- package/dist/server/tool-handlers.js +358 -55
- package/dist/server-core.js +1 -1
- package/dist/utils/exec.js +4 -2
- package/dist/utils/ios/utils.js +5 -2
- package/dist/utils/operation-budget.js +25 -0
- package/dist/utils/resolve-device.js +12 -1
- package/dist/utils/timing.js +19 -0
- package/dist/utils/ui/index.js +2 -0
- package/docs/CHANGELOG.md +3 -0
- package/docs/benchmarks/interaction-latency/README.md +26 -0
- package/docs/benchmarks/interaction-latency/final.json +1810 -0
- package/docs/benchmarks/interaction-latency/final.md +17 -0
- package/docs/benchmarks/interaction-latency/ios-validation.md +11 -0
- package/docs/specs-deltas/018-interaction-latency-and-batched-journeys.md +301 -0
- package/docs/tools/interact.md +30 -0
- package/package.json +1 -1
- package/src/interact/android.ts +13 -8
- package/src/interact/index.ts +32 -23
- package/src/interact/ios.ts +5 -2
- package/src/manage/android.ts +19 -10
- package/src/manage/ios.ts +18 -10
- package/src/observe/android.ts +5 -1
- package/src/observe/index.ts +21 -5
- package/src/observe/ios.ts +6 -2
- package/src/observe/snapshot-metadata.ts +94 -15
- package/src/server/common.ts +13 -6
- package/src/server/journey-schema.ts +63 -0
- package/src/server/tool-definitions.ts +60 -7
- package/src/server/tool-handlers.ts +324 -53
- package/src/server-core.ts +1 -1
- package/src/types.ts +31 -2
- package/src/utils/exec.ts +4 -2
- package/src/utils/ios/utils.ts +5 -2
- package/src/utils/operation-budget.ts +22 -0
- package/src/utils/resolve-device.ts +13 -1
- package/src/utils/timing.ts +15 -0
- package/src/utils/ui/index.ts +2 -0
- package/test/device/automated/latency/benchmark.ts +136 -0
- package/test/device/automated/latency/report.ts +6 -0
- package/test/fixtures/latency-journey-app/AndroidManifest.xml +6 -0
- package/test/fixtures/latency-journey-app/README.md +10 -0
- package/test/fixtures/latency-journey-app/build.ts +26 -0
- package/test/fixtures/latency-journey-app/src/MainActivity.java +48 -0
- package/test/unit/manage/restart_budget.test.ts +37 -0
- package/test/unit/observe/capture_debug_snapshot.test.ts +6 -0
- package/test/unit/observe/snapshot_metadata.test.ts +20 -1
- package/test/unit/observe/state_delta.test.ts +15 -0
- package/test/unit/server/benchmark_report.test.ts +4 -0
- package/test/unit/server/contract.test.ts +1 -1
- package/test/unit/server/interaction_modes.test.ts +60 -0
- package/test/unit/server/operation_budget.test.ts +32 -0
- package/test/unit/server/response_shapes.test.ts +3 -3
- package/test/unit/server/run_journey.test.ts +90 -0
- package/test/unit/server/tap_selector.test.ts +68 -0
package/src/utils/ios/utils.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { remainingBudget } from '../operation-budget.js'
|
|
1
2
|
import { execFile, spawn, execSync, spawnSync } from "child_process"
|
|
2
3
|
import { DeviceInfo } from "../../types.js"
|
|
3
4
|
import { promises as fsPromises } from 'fs'
|
|
@@ -86,6 +87,7 @@ export function validateBundleId(bundleId: string) {
|
|
|
86
87
|
export function execCommand(args: string[], deviceId: string = "booted"): Promise<IOSResult> {
|
|
87
88
|
return new Promise((resolve, reject) => {
|
|
88
89
|
// Use spawn for better stream control and consistency with Android implementation
|
|
90
|
+
const budgetMs = remainingBudget()
|
|
89
91
|
const child = spawn(getXcrunCmd(), args)
|
|
90
92
|
|
|
91
93
|
let stdout = ''
|
|
@@ -105,7 +107,7 @@ export function execCommand(args: string[], deviceId: string = "booted"): Promis
|
|
|
105
107
|
|
|
106
108
|
const DEFAULT_XCRUN_LOG_TIMEOUT = parseInt(process.env.MCP_XCRUN_LOG_TIMEOUT || '', 10) || 30000 // env (ms) or default 30s
|
|
107
109
|
const DEFAULT_XCRUN_CMD_TIMEOUT = parseInt(process.env.MCP_XCRUN_TIMEOUT || '', 10) || 60000 // env (ms) or default 60s
|
|
108
|
-
const timeoutMs = args.includes('log') ? DEFAULT_XCRUN_LOG_TIMEOUT : DEFAULT_XCRUN_CMD_TIMEOUT // choose appropriate timeout
|
|
110
|
+
const timeoutMs = budgetMs || (args.includes('log') ? DEFAULT_XCRUN_LOG_TIMEOUT : DEFAULT_XCRUN_CMD_TIMEOUT) // choose appropriate timeout
|
|
109
111
|
const timeout = setTimeout(() => {
|
|
110
112
|
child.kill()
|
|
111
113
|
reject(new Error(`Command timed out after ${timeoutMs}ms: ${getXcrunCmd()} ${args.join(' ')}`))
|
|
@@ -128,10 +130,11 @@ export function execCommand(args: string[], deviceId: string = "booted"): Promis
|
|
|
128
130
|
}
|
|
129
131
|
|
|
130
132
|
export function execCommandWithDiagnostics(args: string[], deviceId: string = "booted") {
|
|
133
|
+
const budgetMs = remainingBudget()
|
|
131
134
|
// Run synchronously to capture stdout/stderr and exitCode reliably for diagnostics
|
|
132
135
|
const DEFAULT_XCRUN_LOG_TIMEOUT = parseInt(process.env.MCP_XCRUN_LOG_TIMEOUT || '', 10) || 30000
|
|
133
136
|
const DEFAULT_XCRUN_CMD_TIMEOUT = parseInt(process.env.MCP_XCRUN_TIMEOUT || '', 10) || 60000
|
|
134
|
-
const timeoutMs = args.includes('log') ? DEFAULT_XCRUN_LOG_TIMEOUT : DEFAULT_XCRUN_CMD_TIMEOUT
|
|
137
|
+
const timeoutMs = budgetMs || (args.includes('log') ? DEFAULT_XCRUN_LOG_TIMEOUT : DEFAULT_XCRUN_CMD_TIMEOUT)
|
|
135
138
|
const res = spawnSync(getXcrunCmd(), args, { encoding: 'utf8', timeout: timeoutMs }) as any
|
|
136
139
|
const runResult = {
|
|
137
140
|
exitCode: typeof res.status === 'number' ? res.status : null,
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
2
|
+
|
|
3
|
+
export interface OperationBudget { deadline: number; dispatchStarted: boolean; parent?: OperationBudget }
|
|
4
|
+
const budgets = new AsyncLocalStorage<OperationBudget>()
|
|
5
|
+
export function hasFiniteBudget() { return Number.isFinite(budgets.getStore()?.deadline) }
|
|
6
|
+
export function runWithBudget<T>(timeoutMs: number, operation: (budget: OperationBudget) => Promise<T>) {
|
|
7
|
+
const parent = budgets.getStore()
|
|
8
|
+
const budget = { deadline: Math.min(parent?.deadline ?? Infinity, performance.now() + timeoutMs), dispatchStarted: false, parent }
|
|
9
|
+
return budgets.run(budget, () => operation(budget))
|
|
10
|
+
}
|
|
11
|
+
export function remainingBudget(fallback = 0): number {
|
|
12
|
+
const budget = budgets.getStore()
|
|
13
|
+
if (!budget || !Number.isFinite(budget.deadline)) return fallback
|
|
14
|
+
const remaining = Math.floor(budget.deadline - performance.now())
|
|
15
|
+
if (remaining <= 0) throw new Error('ACTION_TIMEOUT')
|
|
16
|
+
return fallback > 0 ? Math.min(fallback, remaining) : remaining
|
|
17
|
+
}
|
|
18
|
+
export function markDispatchStarted() {
|
|
19
|
+
remainingBudget()
|
|
20
|
+
let budget = budgets.getStore()
|
|
21
|
+
while (budget) { budget.dispatchStarted = true; budget = budget.parent }
|
|
22
|
+
}
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { retainConnectedDeviceSnapshots } from '../observe/snapshot-metadata.js'
|
|
2
|
+
import { measure } from './timing.js'
|
|
3
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
1
4
|
import { DeviceInfo } from "../types.js"
|
|
2
5
|
import { listAndroidDevices } from "./android/utils.js"
|
|
3
6
|
import { listIOSDevices } from "./ios/utils.js"
|
|
@@ -44,9 +47,18 @@ export async function listDevices(platform?: "android" | "ios", appId?: string):
|
|
|
44
47
|
return iosDeviceLister(appId)
|
|
45
48
|
}
|
|
46
49
|
|
|
50
|
+
const deviceContext = new AsyncLocalStorage<DeviceInfo>()
|
|
51
|
+
|
|
52
|
+
export function withResolvedDevice<T>(device: DeviceInfo, operation: () => Promise<T>): Promise<T> {
|
|
53
|
+
return deviceContext.run(device, operation)
|
|
54
|
+
}
|
|
55
|
+
|
|
47
56
|
export async function resolveTargetDevice(opts: ResolveOptions): Promise<DeviceInfo> {
|
|
48
57
|
const { platform, appId, prefer, deviceId } = opts
|
|
49
|
-
const
|
|
58
|
+
const current = deviceContext.getStore()
|
|
59
|
+
if (current && current.platform === platform && (!deviceId || current.id === deviceId)) return current
|
|
60
|
+
const devices = await measure('device_resolution_ms', () => listDevices(platform, appId))
|
|
61
|
+
retainConnectedDeviceSnapshots(platform, devices.map(device => device.id))
|
|
50
62
|
|
|
51
63
|
// During unit tests (no adb/xcrun available), provide a lightweight mock device so
|
|
52
64
|
// the observe/interact unit tests can run without real devices.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
2
|
+
const timings = new AsyncLocalStorage<Record<string, number>>()
|
|
3
|
+
export function withTiming<T>(values: Record<string, number>, operation: () => Promise<T>) { return timings.run(values, operation) }
|
|
4
|
+
export function recordTiming(component: string, milliseconds: number) {
|
|
5
|
+
const values = timings.getStore()
|
|
6
|
+
if (values) values[component] = (values[component] ?? 0) + milliseconds
|
|
7
|
+
}
|
|
8
|
+
export async function measure<T>(component: string, operation: () => Promise<T>): Promise<T> {
|
|
9
|
+
const start = performance.now()
|
|
10
|
+
try { return await operation() }
|
|
11
|
+
finally {
|
|
12
|
+
const values = timings.getStore()
|
|
13
|
+
if (values) values[component] = (values[component] ?? 0) + performance.now() - start
|
|
14
|
+
}
|
|
15
|
+
}
|
package/src/utils/ui/index.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { remainingBudget } from '../operation-budget.js'
|
|
1
2
|
import crypto from 'crypto'
|
|
2
3
|
import { GetUITreeResponse, GetCurrentScreenResponse, UIElement, SwipeResponse } from '../../types.js'
|
|
3
4
|
|
|
@@ -152,6 +153,7 @@ export async function scrollToElementShared(opts: {
|
|
|
152
153
|
let scrollsPerformed = 0
|
|
153
154
|
|
|
154
155
|
for (let i = 0; i < maxScrolls; i++) {
|
|
156
|
+
remainingBudget()
|
|
155
157
|
const { x1, y1, x2, y2 } = computeCoords()
|
|
156
158
|
try {
|
|
157
159
|
await swipe(x1, y1, x2, y2, duration, deviceId)
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { execFileSync } from 'node:child_process'
|
|
2
|
+
import { createHash } from 'node:crypto'
|
|
3
|
+
import { readFileSync, writeFileSync, mkdirSync } from 'node:fs'
|
|
4
|
+
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
|
5
|
+
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'
|
|
6
|
+
import { buildFixture } from '../../../fixtures/latency-journey-app/build.js'
|
|
7
|
+
import { summarize } from './report.js'
|
|
8
|
+
|
|
9
|
+
const appId = 'dev.mobiledebugmcp.latencyfixture'
|
|
10
|
+
const stepTimeoutMs = 30000
|
|
11
|
+
const targetWaitTimeoutMs = 10000
|
|
12
|
+
const verificationTimeoutMs = 60000
|
|
13
|
+
const deviceId = process.env.LATENCY_DEVICE_ID
|
|
14
|
+
if (!deviceId) throw new Error('Set LATENCY_DEVICE_ID to the named Android emulator serial')
|
|
15
|
+
const adb = (...args: string[]) => execFileSync('adb', ['-s', deviceId, ...args], { encoding: 'utf8', timeout: 30000 }).trim()
|
|
16
|
+
const apk = buildFixture()
|
|
17
|
+
const selectorMap = {
|
|
18
|
+
savedSession: 'saved-session', idleDetail: 'idle-detail', activeDetail: 'active-detail',
|
|
19
|
+
activate: 'activate', deactivate: 'deactivate', home: 'home', homeScreen: 'home-screen',
|
|
20
|
+
stop: 'stop', stopped: 'stopped'
|
|
21
|
+
}
|
|
22
|
+
const selector = (id: string) => ({ accessibility_id: id })
|
|
23
|
+
const flow = [
|
|
24
|
+
[selectorMap.savedSession, selectorMap.idleDetail], [selectorMap.activate, selectorMap.activeDetail],
|
|
25
|
+
[selectorMap.home, selectorMap.homeScreen], [selectorMap.savedSession, selectorMap.activeDetail],
|
|
26
|
+
[selectorMap.deactivate, selectorMap.idleDetail], [selectorMap.stop, selectorMap.stopped]
|
|
27
|
+
]
|
|
28
|
+
const transport = new StdioClientTransport({ command: process.execPath, args: ['dist/server.js'], stderr: 'pipe' })
|
|
29
|
+
transport.stderr?.on('data', chunk => process.stderr.write(`[server] ${chunk.toString()}`))
|
|
30
|
+
const client = new Client({ name: 'latency-benchmark', version: '1.0' })
|
|
31
|
+
const raw: any[] = []
|
|
32
|
+
const requestSequence: Record<string, any[]> = {}
|
|
33
|
+
let sequence: any[] = []
|
|
34
|
+
let failedSelector: string | null = null
|
|
35
|
+
let failedTool: string | null = null
|
|
36
|
+
async function call(name: string, args: any) {
|
|
37
|
+
failedTool = name
|
|
38
|
+
sequence.push({ tool: name, ...args, appId: args.appId ? '[fixture]' : undefined, deviceId: undefined, steps: args.steps?.map((step: any) => ({ ...step, appId: step.appId ? '[fixture]' : undefined })) })
|
|
39
|
+
const start = performance.now()
|
|
40
|
+
let response
|
|
41
|
+
try {
|
|
42
|
+
response = await client.callTool({ name, arguments: { platform: 'android', deviceId, ...args } }, undefined, { timeout: 300_000 })
|
|
43
|
+
} catch (error) {
|
|
44
|
+
console.error(`MCP ${name} rejected: ${error instanceof Error ? error.stack : String(error)}`)
|
|
45
|
+
throw error
|
|
46
|
+
}
|
|
47
|
+
const elapsed = performance.now() - start
|
|
48
|
+
const text = (response.content as any[]).find(item => item.type === 'text')?.text
|
|
49
|
+
let data: any
|
|
50
|
+
try { data = JSON.parse(text) } catch { data = {} }
|
|
51
|
+
if (response.isError || data.success === false || data.status === 'timeout' || data.error) {
|
|
52
|
+
const failed = args.steps?.find((step: any) => step.id === data.stopped_at_step_id)
|
|
53
|
+
if (failed) { failedSelector = failed.selector?.accessibility_id ?? failed.assertion?.selector?.accessibility_id ?? null; failedTool = `run_journey:${failed.id}` }
|
|
54
|
+
throw new Error('Fixture step failed')
|
|
55
|
+
}
|
|
56
|
+
return { data, elapsed }
|
|
57
|
+
}
|
|
58
|
+
async function run(mode: string) {
|
|
59
|
+
sequence = []
|
|
60
|
+
failedSelector = null
|
|
61
|
+
failedTool = null
|
|
62
|
+
adb('shell', 'am', 'force-stop', appId)
|
|
63
|
+
try { adb('shell', 'pm', 'clear', appId) } catch { /* First install has no app data. */ }
|
|
64
|
+
adb('install', '-r', apk)
|
|
65
|
+
// Check the reset state outside the measured MCP flow, then force-stop once more.
|
|
66
|
+
failedTool = 'reset-state'
|
|
67
|
+
failedSelector = selectorMap.savedSession
|
|
68
|
+
adb('shell', 'am', 'start', '-n', `${appId}/.MainActivity`)
|
|
69
|
+
adb('shell', 'uiautomator', 'dump', '/sdcard/ui.xml')
|
|
70
|
+
if (!adb('shell', 'cat', '/sdcard/ui.xml').includes(`content-desc="${selectorMap.savedSession}"`)) throw new Error('Reset selector missing')
|
|
71
|
+
adb('shell', 'am', 'force-stop', appId)
|
|
72
|
+
sequence = []
|
|
73
|
+
const requestTimes: number[] = []
|
|
74
|
+
const componentTimes: Record<string, number[]> = {}
|
|
75
|
+
const record = (response: any) => {
|
|
76
|
+
requestTimes.push(response.elapsed)
|
|
77
|
+
const timing = response.data.timing ?? {}
|
|
78
|
+
const components = Object.entries(timing).filter(([key, value]) => key !== 'total_ms' && typeof value === 'number')
|
|
79
|
+
const stepTimings = (response.data.steps ?? []).flatMap((step: any) => Object.entries(step.result?.timing ?? {}))
|
|
80
|
+
const selected = components.length ? components : stepTimings
|
|
81
|
+
for (const [key, value] of selected) if (typeof value === 'number') (componentTimes[key] ??= []).push(value)
|
|
82
|
+
}
|
|
83
|
+
if (mode === 'baseline') {
|
|
84
|
+
record(await call('start_app', { appId, actionTimeoutMs: stepTimeoutMs, verificationTimeoutMs }))
|
|
85
|
+
for (const [target, expected] of flow) {
|
|
86
|
+
failedSelector = target
|
|
87
|
+
const found = await call('wait_for_ui', { selector: selector(target), condition: 'clickable', timeout_ms: targetWaitTimeoutMs, poll_interval_ms: 100 })
|
|
88
|
+
record(found)
|
|
89
|
+
record(await call('tap_element', { elementId: found.data.element.elementId, actionTimeoutMs: stepTimeoutMs, verificationTimeoutMs }))
|
|
90
|
+
failedSelector = expected
|
|
91
|
+
record(await call('expect_element_visible', { selector: selector(expected), timeout_ms: stepTimeoutMs, poll_interval_ms: 100 }))
|
|
92
|
+
}
|
|
93
|
+
} else {
|
|
94
|
+
const steps: any[] = [{ id: 'launch', type: 'start_app', appId }]
|
|
95
|
+
flow.forEach(([target, expected], index) => {
|
|
96
|
+
steps.push({ id: `tap-${index}`, type: 'tap', selector: selector(target), waitFor: { condition: 'clickable', timeoutMs: targetWaitTimeoutMs, pollIntervalMs: 100 } })
|
|
97
|
+
steps.push({ id: `assert-${index}`, type: 'assert', assertion: { kind: 'element_visible', selector: selector(expected) }, timeoutMs: stepTimeoutMs, pollIntervalMs: 100 })
|
|
98
|
+
})
|
|
99
|
+
record(await call('run_journey', { defaults: { verificationMode: mode === 'full' ? 'full' : 'light', actionTimeoutMs: stepTimeoutMs, verificationTimeoutMs }, responseMode: mode === 'full' ? 'debug' : 'compact', steps }))
|
|
100
|
+
}
|
|
101
|
+
requestSequence[mode] ??= sequence
|
|
102
|
+
const screenshot = await call('capture_screenshot', {})
|
|
103
|
+
const snapshot = await call('capture_debug_snapshot', { includeScreenshot: false, includeLogs: false })
|
|
104
|
+
return { total_ms: requestTimes.reduce((a, b) => a + b, 0), componentTimes, screenshot_ms: screenshot.elapsed, snapshot_ms: snapshot.elapsed }
|
|
105
|
+
}
|
|
106
|
+
async function main() {
|
|
107
|
+
await client.connect(transport)
|
|
108
|
+
try {
|
|
109
|
+
for (const mode of ['baseline', 'full', 'light']) {
|
|
110
|
+
for (let iteration = 0; iteration < 13; iteration++) {
|
|
111
|
+
console.log(`${mode} ${iteration + 1}/13`)
|
|
112
|
+
try { raw.push({ mode, warmup: iteration < 3, success: true, ...await run(mode) }) }
|
|
113
|
+
catch (error) { const message = error instanceof Error ? error.message : String(error); console.error(`${mode} run failed at ${failedTool}/${failedSelector}: ${message}`); raw.push({ mode, warmup: iteration < 3, success: false, failed_selector: failedSelector, failed_tool: failedTool, error: message }) }
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
} finally { await client.close() }
|
|
117
|
+
const modes = Object.fromEntries(['baseline', 'full', 'light'].map(mode => {
|
|
118
|
+
const samples = raw.filter(item => item.mode === mode && !item.warmup && item.success)
|
|
119
|
+
const failures = raw.filter(item => item.mode === mode && !item.success)
|
|
120
|
+
return [mode, { failures: failures.length, first_failure: failures[0] ? { selector: failures[0].failed_selector, tool_or_step: failures[0].failed_tool } : null, total_ms: summarize(samples.map(item => item.total_ms)), screenshot_ms: summarize(samples.map(item => item.screenshot_ms)), snapshot_ms: summarize(samples.map(item => item.snapshot_ms)), components: Object.fromEntries([...new Set(samples.flatMap(item => Object.keys(item.componentTimes)))].map(key => [key, summarize(samples.flatMap(item => item.componentTimes[key] ?? []))])) }]
|
|
121
|
+
})) as any
|
|
122
|
+
const report = {
|
|
123
|
+
commit: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), working_tree: 'candidate',
|
|
124
|
+
server_version: JSON.parse(readFileSync('package.json', 'utf8')).version,
|
|
125
|
+
platform: 'android', model: adb('shell', 'getprop', 'ro.product.model'), os: adb('shell', 'getprop', 'ro.build.version.release'), api: adb('shell', 'getprop', 'ro.build.version.sdk'), emulator_image: adb('emu', 'avd', 'name').split('\n')[0].trim(), display_size: adb('shell', 'wm', 'size'), display_density: adb('shell', 'wm', 'density'), adb_timeout_ms: Number(process.env.MCP_ADB_TIMEOUT ?? 20000),
|
|
126
|
+
fixture_version: '1.0', fixture_sha256: createHash('sha256').update(readFileSync(apk)).digest('hex'), warmup_runs: 3, measured_runs: 10,
|
|
127
|
+
modes, requests: requestSequence, raw,
|
|
128
|
+
gates: { full_p95: !!modes.baseline.total_ms && !!modes.full.total_ms && modes.full.total_ms.p95 <= modes.baseline.total_ms.p95 * 1.1, light_p50: !!modes.full.total_ms && !!modes.light.total_ms && modes.light.total_ms.p50 <= modes.full.total_ms.p50 * .7 }
|
|
129
|
+
}
|
|
130
|
+
const root = 'docs/benchmarks/interaction-latency'
|
|
131
|
+
mkdirSync(root, { recursive: true })
|
|
132
|
+
writeFileSync(`${root}/final.json`, JSON.stringify(report, null, 2))
|
|
133
|
+
writeFileSync(`${root}/final.md`, `# Interaction latency benchmark\n\nCommit: ${report.commit} (working tree candidate). Fixture: ${report.fixture_version}, SHA-256 ${report.fixture_sha256}.\n\nEnvironment: ${report.emulator_image}, Android ${report.os}, API ${report.api}. Three warm-ups and ten measured runs per mode. Failed runs excluded.\n\n| Mode | p50 ms | p95 ms | Failures |\n|---|---:|---:|---:|\n${Object.entries(modes).map(([mode, value]: [string, any]) => `| ${mode} | ${value.total_ms?.p50 ?? 'n/a'} | ${value.total_ms?.p95 ?? 'n/a'} | ${value.failures} |`).join('\n')}\n\nFirst failure by mode: ${Object.entries(modes).map(([mode, value]: [string, any]) => `${mode}: ${value.first_failure ? `${value.first_failure.tool_or_step} / ${value.first_failure.selector ?? 'no selector'}` : 'none'}`).join('; ')}.\n\nFull p95 gate: ${report.gates.full_p95}. Light p50 gate: ${report.gates.light_p50}.\n\nSee final.json for requests, timing components, and failure selectors.\n`)
|
|
134
|
+
if (!report.gates.full_p95 || !report.gates.light_p50) process.exitCode = 1
|
|
135
|
+
}
|
|
136
|
+
main().catch(error => { console.error(error); process.exitCode = 1 })
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export function summarize(samples: number[]) {
|
|
2
|
+
if (!samples.length) return null
|
|
3
|
+
const sorted = [...samples].sort((a, b) => a - b)
|
|
4
|
+
const percentile = (p: number) => sorted[Math.max(0, Math.ceil(p * sorted.length) - 1)]
|
|
5
|
+
return { count: samples.length, p50: percentile(.5), p95: percentile(.95), min: sorted[0], max: sorted[sorted.length - 1] }
|
|
6
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="dev.mobiledebugmcp.latencyfixture" android:versionCode="1" android:versionName="1.0">
|
|
2
|
+
<uses-sdk android:minSdkVersion="23" android:targetSdkVersion="35"/>
|
|
3
|
+
<application android:theme="@android:style/Theme.Material.Light.NoActionBar" android:label="Latency fixture" android:allowBackup="false">
|
|
4
|
+
<activity android:name=".MainActivity" android:exported="true"><intent-filter><action android:name="android.intent.action.MAIN"/><category android:name="android.intent.category.LAUNCHER"/></intent-filter></activity>
|
|
5
|
+
</application>
|
|
6
|
+
</manifest>
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Interaction latency fixture
|
|
2
|
+
|
|
3
|
+
An offline native Android fixture, version 1.0. It requires Android SDK platform 36,
|
|
4
|
+
build-tools 36.0.0 and JDK 21. The harness builds and signs it using those installed
|
|
5
|
+
tools; no Gradle dependency download is needed.
|
|
6
|
+
|
|
7
|
+
Selectors are accessibility IDs: `saved-session`, `idle-detail`, `activate`,
|
|
8
|
+
`active-detail`, `home`, `home-screen`, `deactivate`, `stop`, and `stopped`.
|
|
9
|
+
The app keeps control state only in memory. Clearing data and relaunching restores
|
|
10
|
+
Home with an inactive control. It requests no permissions and has no network code.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { execFileSync } from 'node:child_process'
|
|
2
|
+
import { mkdirSync } from 'node:fs'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import { fileURLToPath } from 'node:url'
|
|
5
|
+
|
|
6
|
+
export function buildFixture() {
|
|
7
|
+
const root = path.dirname(fileURLToPath(import.meta.url))
|
|
8
|
+
const out = path.join(root, 'build')
|
|
9
|
+
const sdk = process.env.ANDROID_HOME ?? path.join(process.env.HOME!, 'Library/Android/sdk')
|
|
10
|
+
const buildTools = path.join(sdk, 'build-tools', '36.0.0')
|
|
11
|
+
const androidJar = path.join(sdk, 'platforms', 'android-36', 'android.jar')
|
|
12
|
+
const java = process.env.JAVA_HOME
|
|
13
|
+
const tool = (name: string) => java ? path.join(java, 'bin', name) : name
|
|
14
|
+
const run = (cmd: string, args: string[]) => execFileSync(cmd, args, { cwd: out, stdio: 'pipe' })
|
|
15
|
+
mkdirSync(path.join(out, 'classes'), { recursive: true })
|
|
16
|
+
run(tool('javac'), ['--release', '8', '-classpath', androidJar, '-d', 'classes', path.join(root, 'src/MainActivity.java')])
|
|
17
|
+
run(tool('jar'), ['cf', 'classes.jar', '-C', 'classes', '.'])
|
|
18
|
+
run(path.join(buildTools, 'd8'), ['--lib', androidJar, '--output', out, 'classes.jar'])
|
|
19
|
+
run(path.join(buildTools, 'aapt2'), ['link', '-I', androidJar, '--manifest', path.join(root, 'AndroidManifest.xml'), '-o', 'unsigned.apk'])
|
|
20
|
+
run('zip', ['-q', 'unsigned.apk', 'classes.dex'])
|
|
21
|
+
run(path.join(buildTools, 'zipalign'), ['-f', '4', 'unsigned.apk', 'aligned.apk'])
|
|
22
|
+
// Ephemeral signing material is generated outside version control.
|
|
23
|
+
try { run(tool('keytool'), ['-genkeypair', '-keystore', 'fixture.jks', '-storepass', 'android', '-keypass', 'android', '-alias', 'fixture', '-dname', 'CN=Latency Fixture', '-keyalg', 'RSA', '-validity', '3650']) } catch { /* Existing local fixture key. */ }
|
|
24
|
+
run(path.join(buildTools, 'apksigner'), ['sign', '--ks', 'fixture.jks', '--ks-pass', 'pass:android', '--out', 'fixture.apk', 'aligned.apk'])
|
|
25
|
+
return path.join(out, 'fixture.apk')
|
|
26
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
package dev.mobiledebugmcp.latencyfixture;
|
|
2
|
+
|
|
3
|
+
import android.app.Activity;
|
|
4
|
+
import android.os.Bundle;
|
|
5
|
+
import android.widget.Button;
|
|
6
|
+
import android.widget.LinearLayout;
|
|
7
|
+
import android.widget.TextView;
|
|
8
|
+
|
|
9
|
+
public final class MainActivity extends Activity {
|
|
10
|
+
private static final class Selectors {
|
|
11
|
+
static final String SAVED_SESSION = "saved-session";
|
|
12
|
+
static final String IDLE_DETAIL = "idle-detail";
|
|
13
|
+
static final String ACTIVE_DETAIL = "active-detail";
|
|
14
|
+
static final String ACTIVATE = "activate";
|
|
15
|
+
static final String DEACTIVATE = "deactivate";
|
|
16
|
+
static final String HOME = "home";
|
|
17
|
+
static final String HOME_SCREEN = "home-screen";
|
|
18
|
+
static final String STOP = "stop";
|
|
19
|
+
static final String STOPPED = "stopped";
|
|
20
|
+
}
|
|
21
|
+
private boolean active = false;
|
|
22
|
+
private LinearLayout layout;
|
|
23
|
+
@Override public void onCreate(Bundle state) { super.onCreate(state); home(); }
|
|
24
|
+
private void screen(String id) {
|
|
25
|
+
layout = new LinearLayout(this);
|
|
26
|
+
layout.setOrientation(LinearLayout.VERTICAL);
|
|
27
|
+
layout.setPadding(32, 100, 32, 32);
|
|
28
|
+
setContentView(layout);
|
|
29
|
+
TextView title = new TextView(this);
|
|
30
|
+
title.setText(id);
|
|
31
|
+
title.setContentDescription(id);
|
|
32
|
+
layout.addView(title);
|
|
33
|
+
}
|
|
34
|
+
private void button(String id, Runnable action) {
|
|
35
|
+
Button button = new Button(this);
|
|
36
|
+
button.setText(id);
|
|
37
|
+
button.setContentDescription(id);
|
|
38
|
+
button.setOnClickListener(v -> action.run());
|
|
39
|
+
layout.addView(button, new LinearLayout.LayoutParams(-1, 120));
|
|
40
|
+
}
|
|
41
|
+
private void home() { screen(Selectors.HOME_SCREEN); button(Selectors.SAVED_SESSION, () -> detail()); }
|
|
42
|
+
private void detail() {
|
|
43
|
+
screen(active ? Selectors.ACTIVE_DETAIL : Selectors.IDLE_DETAIL);
|
|
44
|
+
button(active ? Selectors.DEACTIVATE : Selectors.ACTIVATE, () -> { active = !active; detail(); });
|
|
45
|
+
button(Selectors.HOME, () -> home());
|
|
46
|
+
button(Selectors.STOP, () -> { active = false; screen(Selectors.STOPPED); });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import assert from 'node:assert/strict'
|
|
2
|
+
import { AndroidManage } from '../../../src/manage/android.js'
|
|
3
|
+
import { iOSManage } from '../../../src/manage/ios.js'
|
|
4
|
+
import { remainingBudget, runWithBudget } from '../../../src/utils/operation-budget.js'
|
|
5
|
+
|
|
6
|
+
async function run() {
|
|
7
|
+
const androidTerminate = AndroidManage.prototype.terminateApp
|
|
8
|
+
const androidStart = AndroidManage.prototype.startApp
|
|
9
|
+
const iosTerminate = iOSManage.prototype.terminateApp
|
|
10
|
+
const iosStart = iOSManage.prototype.startApp
|
|
11
|
+
const device = { platform: 'android', id: 'fixture' } as any
|
|
12
|
+
let androidLaunchBudget = 0
|
|
13
|
+
let iosLaunchBudget = 0
|
|
14
|
+
try {
|
|
15
|
+
AndroidManage.prototype.terminateApp = async () => ({ device, appTerminated: true })
|
|
16
|
+
AndroidManage.prototype.startApp = async (_appId, _deviceId, timeoutMs) => {
|
|
17
|
+
androidLaunchBudget = timeoutMs ?? 0
|
|
18
|
+
return { device, appStarted: true } as any
|
|
19
|
+
}
|
|
20
|
+
iOSManage.prototype.terminateApp = async () => ({ device, appTerminated: true })
|
|
21
|
+
iOSManage.prototype.startApp = async () => {
|
|
22
|
+
iosLaunchBudget = remainingBudget()
|
|
23
|
+
return { device, appStarted: true } as any
|
|
24
|
+
}
|
|
25
|
+
await runWithBudget(30000, () => new AndroidManage().restartApp('fixture', 'fixture', true, 30000))
|
|
26
|
+
await runWithBudget(30000, () => new iOSManage().restartApp('fixture', 'fixture', false, true, 30000))
|
|
27
|
+
assert.ok(androidLaunchBudget > 15000, `Android launch received ${androidLaunchBudget}ms`)
|
|
28
|
+
assert.ok(iosLaunchBudget > 15000, `iOS launch received ${iosLaunchBudget}ms`)
|
|
29
|
+
} finally {
|
|
30
|
+
AndroidManage.prototype.terminateApp = androidTerminate
|
|
31
|
+
AndroidManage.prototype.startApp = androidStart
|
|
32
|
+
iOSManage.prototype.terminateApp = iosTerminate
|
|
33
|
+
iOSManage.prototype.startApp = iosStart
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
run().catch(error => { console.error(error); process.exitCode = 1 })
|
|
@@ -83,6 +83,12 @@ async function run() {
|
|
|
83
83
|
assert.ok(pass3, 'captureDebugSnapshot should return an empty logs array when includeLogs is false')
|
|
84
84
|
console.log('Test 3:', pass3 ? 'PASS' : 'FAIL')
|
|
85
85
|
|
|
86
|
+
let screenshotCalls = 0
|
|
87
|
+
;(ToolsObserve as any).captureScreenshotHandler = async function() { screenshotCalls++; return { screenshot: 'unexpected' } }
|
|
88
|
+
const res4: any = await ToolsObserve.captureDebugSnapshotHandler({ platform: 'android', includeLogs: false, includeScreenshot: false })
|
|
89
|
+
assert.strictEqual(screenshotCalls, 0, 'includeScreenshot=false must not invoke screenshot capture')
|
|
90
|
+
assert.strictEqual(res4.raw.screenshot, null)
|
|
91
|
+
|
|
86
92
|
} finally {
|
|
87
93
|
;(ToolsObserve as any).captureScreenshotHandler = origCaptureHandler
|
|
88
94
|
;(ToolsObserve as any).getCurrentScreenHandler = origGetCurrentHandler
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import assert from 'assert'
|
|
2
|
-
import { deriveSnapshotMetadata, resetSnapshotMetadataForTests } from '../../../src/observe/snapshot-metadata.js'
|
|
2
|
+
import { deriveSnapshotMetadata, getStateDelta, resetSnapshotMetadataForTests } from '../../../src/observe/snapshot-metadata.js'
|
|
3
3
|
|
|
4
4
|
async function run() {
|
|
5
5
|
console.log('Starting snapshot_metadata unit tests...')
|
|
@@ -57,6 +57,25 @@ async function run() {
|
|
|
57
57
|
total_elements: 1
|
|
58
58
|
})
|
|
59
59
|
|
|
60
|
+
const delta = getStateDelta(deviceKey, 1, 2)
|
|
61
|
+
assert.ok(delta)
|
|
62
|
+
assert.strictEqual(delta!.added.length, 0)
|
|
63
|
+
assert.strictEqual(delta!.removed.length, 0)
|
|
64
|
+
assert.strictEqual(delta!.changed.length, 1)
|
|
65
|
+
assert.strictEqual(delta!.changed[0].text, 'Beta')
|
|
66
|
+
assert.strictEqual(delta!.truncated, false)
|
|
67
|
+
assert.strictEqual(getStateDelta(deviceKey, 99, 2), null)
|
|
68
|
+
|
|
69
|
+
for (let index = 3; index <= 10; index++) {
|
|
70
|
+
deriveSnapshotMetadata(deviceKey, {
|
|
71
|
+
screen: 'Home',
|
|
72
|
+
resolution: { width: 100, height: 200 },
|
|
73
|
+
elements: [{ text: `Revision ${index}`, stable_id: 'stable-row', visible: true }]
|
|
74
|
+
}, 'ui_tree')
|
|
75
|
+
}
|
|
76
|
+
assert.ok(getStateDelta(deviceKey, 3, 10), 'recent revisions remain available after pruning')
|
|
77
|
+
assert.strictEqual(getStateDelta(deviceKey, 2, 10), null, 'only the oldest revision is pruned')
|
|
78
|
+
|
|
60
79
|
resetSnapshotMetadataForTests()
|
|
61
80
|
console.log('snapshot_metadata unit tests passed')
|
|
62
81
|
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import assert from 'node:assert/strict'
|
|
2
|
+
import { deriveSnapshotMetadata, getStateDelta, getLatestStateDelta, resetSnapshotMetadataForTests } from '../../../src/observe/snapshot-metadata.js'
|
|
3
|
+
resetSnapshotMetadataForTests()
|
|
4
|
+
const tree = (text: string) => ({ elements: [{ stable_id: 'one', text }, { stable_id: 'unchanged', text: 'fixed' }] }) as any
|
|
5
|
+
const first = deriveSnapshotMetadata('android:test', tree('before'), 'test')
|
|
6
|
+
const second = deriveSnapshotMetadata('android:test', tree('after'), 'test')
|
|
7
|
+
const delta = getStateDelta('android:test', first.snapshot_revision, second.snapshot_revision)!
|
|
8
|
+
assert.equal(delta.changed.length, 1)
|
|
9
|
+
assert.equal(delta.changed[0].text, 'after')
|
|
10
|
+
deriveSnapshotMetadata('android:test', tree('after'), 'test')
|
|
11
|
+
assert.equal(getLatestStateDelta('android:test')!.changed.length, 0)
|
|
12
|
+
const large = { elements: Array.from({ length: 501 }, (_, i) => ({ stable_id: String(i), text: String(i) })) } as any
|
|
13
|
+
const third = deriveSnapshotMetadata('android:test', large, 'test')
|
|
14
|
+
assert.equal(getStateDelta('android:test', second.snapshot_revision, third.snapshot_revision)!.truncated, true)
|
|
15
|
+
assert.equal(getStateDelta('android:test', -1, third.snapshot_revision), null)
|
|
@@ -49,7 +49,7 @@ async function run() {
|
|
|
49
49
|
|
|
50
50
|
const tapElement = toolDefinitions.find((tool) => tool.name === 'tap_element')
|
|
51
51
|
assert(tapElement, 'tap_element should be registered')
|
|
52
|
-
assert.deepStrictEqual((tapElement as any).inputSchema.required, ['elementId'])
|
|
52
|
+
assert.deepStrictEqual((tapElement as any).inputSchema.oneOf, [{ required: ['elementId'], not: { required: ['selector'] } }, { required: ['selector'], not: { required: ['elementId'] } }])
|
|
53
53
|
assert.match((tapElement as any).description, /RESOLVE → ACT → WAIT \(if needed\) → EXPECT/)
|
|
54
54
|
assert.match((tapElement as any).description, /If needed, wait for transition using wait_for_\*/)
|
|
55
55
|
assert.match((tapElement as any).description, /Verify outcome using expect_\*/)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import assert from 'node:assert/strict'
|
|
2
|
+
import { handleToolCall } from '../../../src/server-core.js'
|
|
3
|
+
import { ToolsInteract } from '../../../src/interact/index.js'
|
|
4
|
+
import { ToolsObserve } from '../../../src/observe/index.js'
|
|
5
|
+
import { AndroidManage } from '../../../src/manage/android.js'
|
|
6
|
+
import { markDispatchStarted } from '../../../src/utils/operation-budget.js'
|
|
7
|
+
|
|
8
|
+
async function run() {
|
|
9
|
+
const treeOriginal = ToolsObserve.getUITreeHandler
|
|
10
|
+
const fingerprintOriginal = ToolsObserve.getScreenFingerprintHandler
|
|
11
|
+
const tapOriginal = ToolsInteract.tapHandler
|
|
12
|
+
const launchOriginal = AndroidManage.prototype.startApp
|
|
13
|
+
const device: any = { platform: 'android', id: 'fixture' }
|
|
14
|
+
let dispatched = false
|
|
15
|
+
let postReads = 0
|
|
16
|
+
let fingerprints = 0
|
|
17
|
+
ToolsObserve.getUITreeHandler = async () => {
|
|
18
|
+
if (dispatched) postReads++
|
|
19
|
+
return { device, elements: [{ text: 'Go', stable_id: 'go', bounds: [0, 0, 40, 40], visible: true, enabled: true, clickable: true }] } as any
|
|
20
|
+
}
|
|
21
|
+
ToolsObserve.getScreenFingerprintHandler = async () => { fingerprints++; return { fingerprint: 'fixture' } as any }
|
|
22
|
+
ToolsInteract.tapHandler = async () => { dispatched = true; return { success: true, device } as any }
|
|
23
|
+
AndroidManage.prototype.startApp = async () => { dispatched = true; return { appStarted: true, device } as any }
|
|
24
|
+
try {
|
|
25
|
+
for (const tool of ['tap_element', 'start_app']) {
|
|
26
|
+
for (const mode of ['none', 'light', 'full']) {
|
|
27
|
+
dispatched = false; postReads = 0; fingerprints = 0
|
|
28
|
+
const response = await handleToolCall(tool, { platform: 'android', deviceId: 'fixture', ...(tool === 'tap_element' ? { selector: { text: 'Go' } } : { appId: 'fixture' }), verificationMode: mode, responseMode: 'compact' })
|
|
29
|
+
const result = JSON.parse(response.content[0].text!)
|
|
30
|
+
assert.equal(result.success, true, JSON.stringify(result))
|
|
31
|
+
assert.equal(postReads, mode === 'light' ? 1 : 0)
|
|
32
|
+
assert.equal(fingerprints, mode === 'full' ? 2 : 0)
|
|
33
|
+
assert.equal(result.device, undefined)
|
|
34
|
+
assert.equal(result.trace, undefined)
|
|
35
|
+
assert.equal(result.dispatch_started, true)
|
|
36
|
+
assert.deepEqual(Object.keys(result.timing), ['total_ms'])
|
|
37
|
+
if (mode !== 'full') assert.equal(result.lifecycle_state, 'pending_verification')
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
ToolsObserve.getUITreeHandler = async () => { throw new Error('offline') }
|
|
41
|
+
const result = JSON.parse((await handleToolCall('start_app', { platform: 'android', appId: 'fixture', verificationMode: 'light', responseMode: 'compact' })).content[0].text!)
|
|
42
|
+
assert.equal(result.success, true)
|
|
43
|
+
assert.equal(result.verification.status, 'unavailable')
|
|
44
|
+
assert.equal(result.verification_diagnostic.retryable, true)
|
|
45
|
+
ToolsInteract.tapHandler = async () => { markDispatchStarted(); dispatched = true; return { success: false, error: 'ACTION_TIMEOUT', device } as any }
|
|
46
|
+
const timedOut = JSON.parse((await handleToolCall('tap', { platform: 'android', deviceId: 'fixture', x: 1, y: 1, verificationMode: 'none', responseMode: 'compact' })).content[0].text!)
|
|
47
|
+
assert.equal(timedOut.failure_code, 'ACTION_TIMEOUT')
|
|
48
|
+
assert.equal(timedOut.delivery_status, 'unknown')
|
|
49
|
+
assert.equal(timedOut.retryable, false)
|
|
50
|
+
AndroidManage.prototype.startApp = async () => ({ appStarted: false, device, error: 'launch failed', diagnostics: { runResult: { exitCode: 1, stdout: '', stderr: 'launch failed' } } } as any)
|
|
51
|
+
const failedLaunch = JSON.parse((await handleToolCall('start_app', { platform: 'android', appId: 'fixture' })).content[0].text!)
|
|
52
|
+
assert.equal(failedLaunch.details.diagnostics.runResult.exitCode, 1)
|
|
53
|
+
} finally {
|
|
54
|
+
ToolsObserve.getUITreeHandler = treeOriginal
|
|
55
|
+
ToolsObserve.getScreenFingerprintHandler = fingerprintOriginal
|
|
56
|
+
ToolsInteract.tapHandler = tapOriginal
|
|
57
|
+
AndroidManage.prototype.startApp = launchOriginal
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
run().catch(error => { console.error(error); process.exitCode = 1 })
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import assert from 'node:assert/strict'
|
|
2
|
+
import { runWithBudget, remainingBudget, markDispatchStarted } from '../../../src/utils/operation-budget.js'
|
|
3
|
+
import { execCmd } from '../../../src/utils/exec.js'
|
|
4
|
+
import { ToolsInteract } from '../../../src/interact/index.js'
|
|
5
|
+
import { ToolsObserve } from '../../../src/observe/index.js'
|
|
6
|
+
|
|
7
|
+
async function run() {
|
|
8
|
+
await runWithBudget(100, async budget => {
|
|
9
|
+
assert.equal(budget.dispatchStarted, false)
|
|
10
|
+
markDispatchStarted()
|
|
11
|
+
assert.equal(budget.dispatchStarted, true)
|
|
12
|
+
const response = await execCmd(process.execPath, ['-e', 'setTimeout(() => {}, 10000)'])
|
|
13
|
+
assert.equal(response.stderr, 'ACTION_TIMEOUT')
|
|
14
|
+
assert.throws(() => remainingBudget(), /ACTION_TIMEOUT/)
|
|
15
|
+
})
|
|
16
|
+
await runWithBudget(1, async budget => {
|
|
17
|
+
await new Promise(resolve => setTimeout(resolve, 5))
|
|
18
|
+
assert.throws(() => markDispatchStarted(), /ACTION_TIMEOUT/)
|
|
19
|
+
assert.equal(budget.dispatchStarted, false)
|
|
20
|
+
})
|
|
21
|
+
const originalTree = ToolsObserve.getUITreeHandler
|
|
22
|
+
ToolsObserve.getUITreeHandler = async () => { throw new Error('transient observation error') }
|
|
23
|
+
try {
|
|
24
|
+
await assert.rejects(
|
|
25
|
+
() => runWithBudget(30, () => ToolsInteract.waitForUIHandler({ selector: { text: 'missing' }, timeout_ms: 1000, poll_interval_ms: 5, platform: 'android', deviceId: 'fixture' })),
|
|
26
|
+
/ACTION_TIMEOUT/
|
|
27
|
+
)
|
|
28
|
+
} finally {
|
|
29
|
+
ToolsObserve.getUITreeHandler = originalTree
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
run().catch(error => { console.error(error); process.exitCode = 1 })
|
|
@@ -37,7 +37,7 @@ async function run() {
|
|
|
37
37
|
|
|
38
38
|
const missingBuildResponse = await handleToolCall('build_app', { projectPath: '/tmp/project' })
|
|
39
39
|
const missingBuildPayload = JSON.parse((missingBuildResponse as any).content[0].text)
|
|
40
|
-
assert.deepStrictEqual(missingBuildPayload, {
|
|
40
|
+
assert.deepStrictEqual({ error: missingBuildPayload.error }, {
|
|
41
41
|
error: {
|
|
42
42
|
tool: 'build_app',
|
|
43
43
|
message: 'Missing or invalid string argument: platform'
|
|
@@ -236,7 +236,7 @@ async function run() {
|
|
|
236
236
|
const failingTapResponse = await handleToolCall('tap', { platform: 'android', x: 1, y: 2 })
|
|
237
237
|
assert.strictEqual((failingTapResponse as any).content.length, 1)
|
|
238
238
|
const failingTapPayload = JSON.parse((failingTapResponse as any).content[0].text)
|
|
239
|
-
assert.deepStrictEqual(failingTapPayload, {
|
|
239
|
+
assert.deepStrictEqual({ error: failingTapPayload.error }, {
|
|
240
240
|
error: {
|
|
241
241
|
tool: 'tap',
|
|
242
242
|
message: 'boom'
|
|
@@ -255,7 +255,7 @@ async function run() {
|
|
|
255
255
|
|
|
256
256
|
const missingArgResponse = await handleToolCall('tap', { platform: 'android', x: 1 })
|
|
257
257
|
const missingArgPayload = JSON.parse((missingArgResponse as any).content[0].text)
|
|
258
|
-
assert.deepStrictEqual(missingArgPayload, {
|
|
258
|
+
assert.deepStrictEqual({ error: missingArgPayload.error }, {
|
|
259
259
|
error: {
|
|
260
260
|
tool: 'tap',
|
|
261
261
|
message: 'Missing or invalid number argument: y'
|