mobai-mcp 1.5.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -3
- package/dist/index.js +293 -1452
- package/dist/resources.d.ts +7 -0
- package/dist/resources.js +497 -0
- package/package.json +1 -1
- package/server.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,61 +1,66 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* MobAI MCP Server
|
|
3
|
+
* MobAI MCP Server (stdio)
|
|
4
4
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
5
|
+
* Mirrors the Go HTTP-based MCP server as a stdio transport.
|
|
6
|
+
* Proxies tool calls to the MobAI HTTP API at 127.0.0.1:8686.
|
|
7
7
|
*/
|
|
8
8
|
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
9
9
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
10
|
-
import { CallToolRequestSchema, ListToolsRequestSchema,
|
|
10
|
+
import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
|
|
11
11
|
import * as fs from "fs";
|
|
12
|
+
import * as os from "os";
|
|
12
13
|
import * as path from "path";
|
|
13
14
|
const API_BASE_URL = "http://127.0.0.1:8686/api/v1";
|
|
14
|
-
const DEFAULT_TIMEOUT_MS =
|
|
15
|
-
const SCREENSHOT_DIR = "
|
|
16
|
-
//
|
|
15
|
+
const DEFAULT_TIMEOUT_MS = 300000; // 5 minutes (matches Go httpClient timeout)
|
|
16
|
+
const SCREENSHOT_DIR = path.join(os.tmpdir(), "mobai", "screenshots");
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// Screenshot helpers
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
17
20
|
function ensureScreenshotDir() {
|
|
18
21
|
if (!fs.existsSync(SCREENSHOT_DIR)) {
|
|
19
22
|
fs.mkdirSync(SCREENSHOT_DIR, { recursive: true });
|
|
20
23
|
}
|
|
21
24
|
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
if (!base64Data || base64Data.length <= 200 || base64Data.startsWith("/")) {
|
|
25
|
+
function saveBase64ToTemp(base64Data, prefix) {
|
|
26
|
+
if (!base64Data || base64Data.length <= 200)
|
|
25
27
|
return null;
|
|
26
|
-
}
|
|
27
28
|
ensureScreenshotDir();
|
|
28
|
-
const filename = `${prefix}
|
|
29
|
+
const filename = `${prefix}_${Date.now()}.png`;
|
|
29
30
|
const filePath = path.join(SCREENSHOT_DIR, filename);
|
|
30
31
|
fs.writeFileSync(filePath, Buffer.from(base64Data, "base64"));
|
|
31
32
|
return filePath;
|
|
32
33
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
function screenshotToFile(body) {
|
|
35
|
+
if (body?.path) {
|
|
36
|
+
return `Screenshot saved to ${body.path}`;
|
|
37
|
+
}
|
|
38
|
+
// Fallback: base64 mode
|
|
39
|
+
if (body?.data) {
|
|
40
|
+
const imgData = Buffer.from(body.data, "base64");
|
|
36
41
|
ensureScreenshotDir();
|
|
37
|
-
const
|
|
42
|
+
const ext = body.format || "png";
|
|
43
|
+
const filename = `screenshot_${Date.now()}.${ext}`;
|
|
38
44
|
const filePath = path.join(SCREENSHOT_DIR, filename);
|
|
39
|
-
fs.writeFileSync(filePath,
|
|
40
|
-
return
|
|
45
|
+
fs.writeFileSync(filePath, imgData);
|
|
46
|
+
return `Screenshot saved to ${filePath}`;
|
|
41
47
|
}
|
|
42
|
-
return body;
|
|
48
|
+
return JSON.stringify(body, null, 2);
|
|
43
49
|
}
|
|
44
|
-
|
|
45
|
-
function processDslResponse(body) {
|
|
50
|
+
function extractDSLScreenshots(body) {
|
|
46
51
|
if (!body?.step_results)
|
|
47
52
|
return body;
|
|
48
53
|
for (const step of body.step_results) {
|
|
49
54
|
const native = step.result?.observations?.native;
|
|
50
|
-
if (native?.screenshot &&
|
|
51
|
-
const filePath =
|
|
55
|
+
if (native?.screenshot && typeof native.screenshot === "string" && native.screenshot.length > 200) {
|
|
56
|
+
const filePath = saveBase64ToTemp(native.screenshot, "observe");
|
|
52
57
|
if (filePath) {
|
|
53
58
|
native.screenshot = filePath;
|
|
54
59
|
native.screenshot_saved = true;
|
|
55
60
|
}
|
|
56
61
|
}
|
|
57
|
-
if (step.debug?.screenshot &&
|
|
58
|
-
const filePath =
|
|
62
|
+
if (step.debug?.screenshot && typeof step.debug.screenshot === "string" && step.debug.screenshot.length > 200) {
|
|
63
|
+
const filePath = saveBase64ToTemp(step.debug.screenshot, "debug");
|
|
59
64
|
if (filePath) {
|
|
60
65
|
step.debug.screenshot = filePath;
|
|
61
66
|
step.debug.screenshot_saved = true;
|
|
@@ -64,1633 +69,469 @@ function processDslResponse(body) {
|
|
|
64
69
|
}
|
|
65
70
|
return body;
|
|
66
71
|
}
|
|
67
|
-
//
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
}
|
|
72
|
-
if (url.includes("/dsl/execute")) {
|
|
73
|
-
return processDslResponse(body);
|
|
74
|
-
}
|
|
75
|
-
return body;
|
|
76
|
-
}
|
|
77
|
-
// Make HTTP request to MobAI API
|
|
78
|
-
async function makeRequest(method, endpoint, body, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
79
|
-
const url = endpoint.startsWith("http") ? endpoint : `${API_BASE_URL}${endpoint}`;
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
// HTTP helpers
|
|
74
|
+
// ---------------------------------------------------------------------------
|
|
75
|
+
async function doRequest(method, urlPath, payload, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
76
|
+
const url = urlPath.startsWith("http") ? urlPath : `${API_BASE_URL}${urlPath}`;
|
|
80
77
|
const controller = new AbortController();
|
|
81
78
|
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
|
82
79
|
try {
|
|
83
|
-
const
|
|
80
|
+
const opts = {
|
|
84
81
|
method,
|
|
85
82
|
headers: { "Content-Type": "application/json" },
|
|
86
83
|
signal: controller.signal,
|
|
87
84
|
};
|
|
88
|
-
if (
|
|
89
|
-
|
|
85
|
+
if (payload !== undefined && ["POST", "PUT", "PATCH"].includes(method)) {
|
|
86
|
+
opts.body = typeof payload === "string" ? payload : JSON.stringify(payload);
|
|
90
87
|
}
|
|
91
|
-
const response = await fetch(url,
|
|
88
|
+
const response = await fetch(url, opts);
|
|
92
89
|
clearTimeout(timeoutId);
|
|
93
|
-
const
|
|
94
|
-
let
|
|
90
|
+
const text = await response.text();
|
|
91
|
+
let body;
|
|
95
92
|
try {
|
|
96
|
-
|
|
97
|
-
responseBody = processResponseBody(responseBody, url);
|
|
93
|
+
body = JSON.parse(text);
|
|
98
94
|
}
|
|
99
95
|
catch {
|
|
100
|
-
|
|
96
|
+
body = text;
|
|
101
97
|
}
|
|
102
|
-
|
|
103
|
-
status:
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
};
|
|
98
|
+
if (response.status >= 400) {
|
|
99
|
+
throw new Error(`HTTP ${response.status}: ${typeof body === "string" ? body : JSON.stringify(body)}`);
|
|
100
|
+
}
|
|
101
|
+
return body;
|
|
107
102
|
}
|
|
108
103
|
finally {
|
|
109
104
|
clearTimeout(timeoutId);
|
|
110
105
|
}
|
|
111
106
|
}
|
|
112
|
-
|
|
113
|
-
const
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
}
|
|
107
|
+
const doGet = (p) => doRequest("GET", p);
|
|
108
|
+
const doPost = (p, body) => doRequest("POST", p, body);
|
|
109
|
+
const doDelete = (p) => doRequest("DELETE", p);
|
|
110
|
+
const doPut = (p, body) => doRequest("PUT", p, body);
|
|
111
|
+
const doPatch = (p, body) => doRequest("PATCH", p, body);
|
|
112
|
+
function textResult(data) {
|
|
113
|
+
return {
|
|
114
|
+
content: [{ type: "text", text: typeof data === "string" ? data : JSON.stringify(data, null, 2) }],
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
function errResult(err) {
|
|
118
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
119
|
+
return {
|
|
120
|
+
content: [{ type: "text", text: message }],
|
|
121
|
+
isError: true,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
// ---------------------------------------------------------------------------
|
|
125
|
+
// Server
|
|
126
|
+
// ---------------------------------------------------------------------------
|
|
127
|
+
const server = new Server({ name: "mobai", version: "1.0.0" }, {
|
|
128
|
+
capabilities: { tools: {}, resources: {} },
|
|
129
|
+
instructions: `MobAI controls Android and iOS devices. Before starting any device task, read the relevant MCP resources:
|
|
130
|
+
- mobai://reference/device-automation — how to control devices
|
|
131
|
+
- mobai://reference/testing — testing workflow, rules, and .mob script syntax
|
|
132
|
+
Check available skills in current work directory and load any relevant to the user's request.`,
|
|
121
133
|
});
|
|
122
|
-
//
|
|
134
|
+
// ---------------------------------------------------------------------------
|
|
135
|
+
// Tool definitions — exactly matches Go registerTools()
|
|
136
|
+
// ---------------------------------------------------------------------------
|
|
123
137
|
const TOOLS = [
|
|
138
|
+
// Device management
|
|
124
139
|
{
|
|
125
140
|
name: "list_devices",
|
|
126
|
-
description: "List all connected Android and iOS devices
|
|
127
|
-
inputSchema: {
|
|
128
|
-
type: "object",
|
|
129
|
-
properties: {},
|
|
130
|
-
required: [],
|
|
131
|
-
},
|
|
141
|
+
description: "List all connected Android and iOS devices",
|
|
142
|
+
inputSchema: { type: "object", properties: {}, required: [] },
|
|
132
143
|
},
|
|
133
144
|
{
|
|
134
145
|
name: "get_device",
|
|
135
|
-
description: "Get
|
|
146
|
+
description: "Get details about a specific device",
|
|
136
147
|
inputSchema: {
|
|
137
148
|
type: "object",
|
|
138
|
-
properties: {
|
|
139
|
-
device_id: {
|
|
140
|
-
type: "string",
|
|
141
|
-
description: "Device ID (serial for Android, UDID for iOS)",
|
|
142
|
-
},
|
|
143
|
-
},
|
|
149
|
+
properties: { device_id: { type: "string", description: "Device ID" } },
|
|
144
150
|
required: ["device_id"],
|
|
145
151
|
},
|
|
146
152
|
},
|
|
147
153
|
{
|
|
148
154
|
name: "start_bridge",
|
|
149
|
-
description: "Start the
|
|
155
|
+
description: "Start the automation bridge on a device. Required before interacting with the device.",
|
|
150
156
|
inputSchema: {
|
|
151
157
|
type: "object",
|
|
152
|
-
properties: {
|
|
153
|
-
device_id: {
|
|
154
|
-
type: "string",
|
|
155
|
-
description: "Device ID",
|
|
156
|
-
},
|
|
157
|
-
},
|
|
158
|
+
properties: { device_id: { type: "string", description: "Device ID" } },
|
|
158
159
|
required: ["device_id"],
|
|
159
160
|
},
|
|
160
161
|
},
|
|
161
162
|
{
|
|
162
163
|
name: "stop_bridge",
|
|
163
|
-
description: "Stop the on
|
|
164
|
+
description: "Stop the automation bridge on a device",
|
|
164
165
|
inputSchema: {
|
|
165
166
|
type: "object",
|
|
166
|
-
properties: {
|
|
167
|
-
device_id: {
|
|
168
|
-
type: "string",
|
|
169
|
-
description: "Device ID",
|
|
170
|
-
},
|
|
171
|
-
},
|
|
167
|
+
properties: { device_id: { type: "string", description: "Device ID" } },
|
|
172
168
|
required: ["device_id"],
|
|
173
169
|
},
|
|
174
170
|
},
|
|
171
|
+
// Screenshot
|
|
175
172
|
{
|
|
176
173
|
name: "get_screenshot",
|
|
177
|
-
description: "Capture a screenshot
|
|
178
|
-
inputSchema: {
|
|
179
|
-
type: "object",
|
|
180
|
-
properties: {
|
|
181
|
-
device_id: {
|
|
182
|
-
type: "string",
|
|
183
|
-
description: "Device ID",
|
|
184
|
-
},
|
|
185
|
-
path: {
|
|
186
|
-
type: "string",
|
|
187
|
-
description: "Custom directory to save the screenshot (supports ~/). Example: ~/Downloads",
|
|
188
|
-
},
|
|
189
|
-
name: {
|
|
190
|
-
type: "string",
|
|
191
|
-
description: "Custom filename without .png extension. Defaults to timestamp-based name.",
|
|
192
|
-
},
|
|
193
|
-
},
|
|
194
|
-
required: ["device_id"],
|
|
195
|
-
},
|
|
196
|
-
},
|
|
197
|
-
{
|
|
198
|
-
name: "get_ui_tree",
|
|
199
|
-
description: "Get the UI accessibility tree showing all visible elements with indices for tapping",
|
|
200
|
-
inputSchema: {
|
|
201
|
-
type: "object",
|
|
202
|
-
properties: {
|
|
203
|
-
device_id: {
|
|
204
|
-
type: "string",
|
|
205
|
-
description: "Device ID",
|
|
206
|
-
},
|
|
207
|
-
verbose: {
|
|
208
|
-
type: "boolean",
|
|
209
|
-
description: "Include detailed elements array with bounds (default: false)",
|
|
210
|
-
},
|
|
211
|
-
only_visible: {
|
|
212
|
-
type: "boolean",
|
|
213
|
-
description: "Filter to only visible elements (default: true)",
|
|
214
|
-
},
|
|
215
|
-
include_keyboard: {
|
|
216
|
-
type: "boolean",
|
|
217
|
-
description: "Include keyboard elements in the tree (default: false). Useful for interacting with on-screen keyboards.",
|
|
218
|
-
},
|
|
219
|
-
text_regex: {
|
|
220
|
-
type: "string",
|
|
221
|
-
description: "Regex to filter elements by text/value/contentDesc. Only matching elements are returned.",
|
|
222
|
-
},
|
|
223
|
-
bounds: {
|
|
224
|
-
type: "object",
|
|
225
|
-
description: "Filter to elements within a bounding rectangle",
|
|
226
|
-
properties: {
|
|
227
|
-
x: { type: "number", description: "Left X coordinate" },
|
|
228
|
-
y: { type: "number", description: "Top Y coordinate" },
|
|
229
|
-
w: { type: "number", description: "Width" },
|
|
230
|
-
h: { type: "number", description: "Height" },
|
|
231
|
-
},
|
|
232
|
-
required: ["x", "y", "w", "h"],
|
|
233
|
-
},
|
|
234
|
-
},
|
|
235
|
-
required: ["device_id"],
|
|
236
|
-
},
|
|
237
|
-
},
|
|
238
|
-
{
|
|
239
|
-
name: "tap",
|
|
240
|
-
description: "Tap an element by index (from UI tree) or coordinates",
|
|
241
|
-
inputSchema: {
|
|
242
|
-
type: "object",
|
|
243
|
-
properties: {
|
|
244
|
-
device_id: {
|
|
245
|
-
type: "string",
|
|
246
|
-
description: "Device ID",
|
|
247
|
-
},
|
|
248
|
-
index: {
|
|
249
|
-
type: "number",
|
|
250
|
-
description: "Element index from UI tree (preferred)",
|
|
251
|
-
},
|
|
252
|
-
x: {
|
|
253
|
-
type: "number",
|
|
254
|
-
description: "X coordinate (use with y instead of index)",
|
|
255
|
-
},
|
|
256
|
-
y: {
|
|
257
|
-
type: "number",
|
|
258
|
-
description: "Y coordinate (use with x instead of index)",
|
|
259
|
-
},
|
|
260
|
-
},
|
|
261
|
-
required: ["device_id"],
|
|
262
|
-
},
|
|
263
|
-
},
|
|
264
|
-
{
|
|
265
|
-
name: "double_tap",
|
|
266
|
-
description: "Double tap an element by index (from UI tree) or coordinates",
|
|
174
|
+
description: "Capture a fast, low-quality screenshot for LLM visual analysis. Returns the file path to the saved image. Use this for AI/LLM processing only — for full-quality screenshots use save_screenshot instead.",
|
|
267
175
|
inputSchema: {
|
|
268
176
|
type: "object",
|
|
269
|
-
properties: {
|
|
270
|
-
device_id: {
|
|
271
|
-
type: "string",
|
|
272
|
-
description: "Device ID",
|
|
273
|
-
},
|
|
274
|
-
index: {
|
|
275
|
-
type: "number",
|
|
276
|
-
description: "Element index from UI tree (preferred)",
|
|
277
|
-
},
|
|
278
|
-
x: {
|
|
279
|
-
type: "number",
|
|
280
|
-
description: "X coordinate (use with y instead of index)",
|
|
281
|
-
},
|
|
282
|
-
y: {
|
|
283
|
-
type: "number",
|
|
284
|
-
description: "Y coordinate (use with x instead of index)",
|
|
285
|
-
},
|
|
286
|
-
},
|
|
177
|
+
properties: { device_id: { type: "string", description: "Device ID" } },
|
|
287
178
|
required: ["device_id"],
|
|
288
179
|
},
|
|
289
180
|
},
|
|
290
181
|
{
|
|
291
|
-
name: "
|
|
292
|
-
description: "
|
|
182
|
+
name: "save_screenshot",
|
|
183
|
+
description: "Save a full-quality PNG screenshot to disk. Use this when you need a high-quality image for reporting, debugging, or sharing — not for LLM processing (use get_screenshot instead).",
|
|
293
184
|
inputSchema: {
|
|
294
185
|
type: "object",
|
|
295
186
|
properties: {
|
|
296
|
-
device_id: {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
},
|
|
300
|
-
index: {
|
|
301
|
-
type: "number",
|
|
302
|
-
description: "Element index from UI tree (preferred)",
|
|
303
|
-
},
|
|
304
|
-
x: {
|
|
305
|
-
type: "number",
|
|
306
|
-
description: "X coordinate (use with y instead of index)",
|
|
307
|
-
},
|
|
308
|
-
y: {
|
|
309
|
-
type: "number",
|
|
310
|
-
description: "Y coordinate (use with x instead of index)",
|
|
311
|
-
},
|
|
187
|
+
device_id: { type: "string", description: "Device ID" },
|
|
188
|
+
path: { type: "string", description: "Directory to save screenshot to (supports ~/). Defaults to OS temp directory." },
|
|
189
|
+
name: { type: "string", description: "Optional filename (without .png extension)" },
|
|
312
190
|
},
|
|
313
191
|
required: ["device_id"],
|
|
314
192
|
},
|
|
315
193
|
},
|
|
194
|
+
// App management
|
|
316
195
|
{
|
|
317
|
-
name: "
|
|
318
|
-
description: "
|
|
319
|
-
inputSchema: {
|
|
320
|
-
type: "object",
|
|
321
|
-
properties: {
|
|
322
|
-
device_id: {
|
|
323
|
-
type: "string",
|
|
324
|
-
description: "Device ID",
|
|
325
|
-
},
|
|
326
|
-
index: {
|
|
327
|
-
type: "number",
|
|
328
|
-
description: "Element index from UI tree (preferred)",
|
|
329
|
-
},
|
|
330
|
-
x: {
|
|
331
|
-
type: "number",
|
|
332
|
-
description: "X coordinate (use with y instead of index)",
|
|
333
|
-
},
|
|
334
|
-
y: {
|
|
335
|
-
type: "number",
|
|
336
|
-
description: "Y coordinate (use with x instead of index)",
|
|
337
|
-
},
|
|
338
|
-
},
|
|
339
|
-
required: ["device_id"],
|
|
340
|
-
},
|
|
341
|
-
},
|
|
342
|
-
{
|
|
343
|
-
name: "drag",
|
|
344
|
-
description: "Drag from one point to another (press, hold, move, release)",
|
|
345
|
-
inputSchema: {
|
|
346
|
-
type: "object",
|
|
347
|
-
properties: {
|
|
348
|
-
device_id: {
|
|
349
|
-
type: "string",
|
|
350
|
-
description: "Device ID",
|
|
351
|
-
},
|
|
352
|
-
from_x: {
|
|
353
|
-
type: "number",
|
|
354
|
-
description: "Starting X coordinate",
|
|
355
|
-
},
|
|
356
|
-
from_y: {
|
|
357
|
-
type: "number",
|
|
358
|
-
description: "Starting Y coordinate",
|
|
359
|
-
},
|
|
360
|
-
to_x: {
|
|
361
|
-
type: "number",
|
|
362
|
-
description: "Ending X coordinate",
|
|
363
|
-
},
|
|
364
|
-
to_y: {
|
|
365
|
-
type: "number",
|
|
366
|
-
description: "Ending Y coordinate",
|
|
367
|
-
},
|
|
368
|
-
duration_ms: {
|
|
369
|
-
type: "number",
|
|
370
|
-
description: "Drag duration in milliseconds (default: 500)",
|
|
371
|
-
},
|
|
372
|
-
press_duration_ms: {
|
|
373
|
-
type: "number",
|
|
374
|
-
description: "Hold duration before dragging in milliseconds (0 = no hold). Use for press-and-drag gestures like moving app icons.",
|
|
375
|
-
},
|
|
376
|
-
},
|
|
377
|
-
required: ["device_id", "from_x", "from_y", "to_x", "to_y"],
|
|
378
|
-
},
|
|
379
|
-
},
|
|
380
|
-
{
|
|
381
|
-
name: "dismiss_keyboard",
|
|
382
|
-
description: "Dismiss the on-screen keyboard if visible",
|
|
196
|
+
name: "list_apps",
|
|
197
|
+
description: "List installed apps on the device",
|
|
383
198
|
inputSchema: {
|
|
384
199
|
type: "object",
|
|
385
|
-
properties: {
|
|
386
|
-
device_id: {
|
|
387
|
-
type: "string",
|
|
388
|
-
description: "Device ID",
|
|
389
|
-
},
|
|
390
|
-
},
|
|
200
|
+
properties: { device_id: { type: "string", description: "Device ID" } },
|
|
391
201
|
required: ["device_id"],
|
|
392
202
|
},
|
|
393
203
|
},
|
|
394
204
|
{
|
|
395
|
-
name: "
|
|
396
|
-
description: "
|
|
205
|
+
name: "install_app",
|
|
206
|
+
description: "Install an app on the device from a local file path (.apk for Android, .ipa for iOS)",
|
|
397
207
|
inputSchema: {
|
|
398
208
|
type: "object",
|
|
399
209
|
properties: {
|
|
400
|
-
device_id: {
|
|
401
|
-
|
|
402
|
-
description: "Device ID",
|
|
403
|
-
},
|
|
404
|
-
text: {
|
|
405
|
-
type: "string",
|
|
406
|
-
description: "Text to type",
|
|
407
|
-
},
|
|
210
|
+
device_id: { type: "string", description: "Device ID" },
|
|
211
|
+
path: { type: "string", description: "Local file path to the app (.apk or .ipa)" },
|
|
408
212
|
},
|
|
409
|
-
required: ["device_id", "
|
|
213
|
+
required: ["device_id", "path"],
|
|
410
214
|
},
|
|
411
215
|
},
|
|
412
216
|
{
|
|
413
|
-
name: "
|
|
414
|
-
description: "
|
|
415
|
-
inputSchema: {
|
|
416
|
-
type: "object",
|
|
417
|
-
properties: {
|
|
418
|
-
device_id: {
|
|
419
|
-
type: "string",
|
|
420
|
-
description: "Device ID",
|
|
421
|
-
},
|
|
422
|
-
from_x: {
|
|
423
|
-
type: "number",
|
|
424
|
-
description: "Starting X coordinate",
|
|
425
|
-
},
|
|
426
|
-
from_y: {
|
|
427
|
-
type: "number",
|
|
428
|
-
description: "Starting Y coordinate",
|
|
429
|
-
},
|
|
430
|
-
to_x: {
|
|
431
|
-
type: "number",
|
|
432
|
-
description: "Ending X coordinate",
|
|
433
|
-
},
|
|
434
|
-
to_y: {
|
|
435
|
-
type: "number",
|
|
436
|
-
description: "Ending Y coordinate",
|
|
437
|
-
},
|
|
438
|
-
duration_ms: {
|
|
439
|
-
type: "number",
|
|
440
|
-
description: "Duration in milliseconds (default: 300)",
|
|
441
|
-
},
|
|
442
|
-
},
|
|
443
|
-
required: ["device_id", "from_x", "from_y", "to_x", "to_y"],
|
|
444
|
-
},
|
|
445
|
-
},
|
|
446
|
-
{
|
|
447
|
-
name: "go_home",
|
|
448
|
-
description: "Navigate to device home screen",
|
|
449
|
-
inputSchema: {
|
|
450
|
-
type: "object",
|
|
451
|
-
properties: {
|
|
452
|
-
device_id: {
|
|
453
|
-
type: "string",
|
|
454
|
-
description: "Device ID",
|
|
455
|
-
},
|
|
456
|
-
},
|
|
457
|
-
required: ["device_id"],
|
|
458
|
-
},
|
|
459
|
-
},
|
|
460
|
-
{
|
|
461
|
-
name: "launch_app",
|
|
462
|
-
description: "Launch an application by bundle ID",
|
|
217
|
+
name: "uninstall_app",
|
|
218
|
+
description: "Uninstall an app from the device",
|
|
463
219
|
inputSchema: {
|
|
464
220
|
type: "object",
|
|
465
221
|
properties: {
|
|
466
|
-
device_id: {
|
|
467
|
-
|
|
468
|
-
description: "Device ID",
|
|
469
|
-
},
|
|
470
|
-
bundle_id: {
|
|
471
|
-
type: "string",
|
|
472
|
-
description: "App bundle ID (e.g., com.apple.Preferences, com.android.settings)",
|
|
473
|
-
},
|
|
222
|
+
device_id: { type: "string", description: "Device ID" },
|
|
223
|
+
bundle_id: { type: "string", description: "App bundle ID (iOS) or package name (Android)" },
|
|
474
224
|
},
|
|
475
225
|
required: ["device_id", "bundle_id"],
|
|
476
226
|
},
|
|
477
227
|
},
|
|
478
|
-
|
|
479
|
-
name: "list_apps",
|
|
480
|
-
description: "List installed applications on the device",
|
|
481
|
-
inputSchema: {
|
|
482
|
-
type: "object",
|
|
483
|
-
properties: {
|
|
484
|
-
device_id: {
|
|
485
|
-
type: "string",
|
|
486
|
-
description: "Device ID",
|
|
487
|
-
},
|
|
488
|
-
},
|
|
489
|
-
required: ["device_id"],
|
|
490
|
-
},
|
|
491
|
-
},
|
|
492
|
-
{
|
|
493
|
-
name: "get_ocr",
|
|
494
|
-
description: "Perform OCR text recognition on the current screen (iOS only). Returns detected text with screen coordinates for tapping (already adjusted for tapping).",
|
|
495
|
-
inputSchema: {
|
|
496
|
-
type: "object",
|
|
497
|
-
properties: {
|
|
498
|
-
device_id: {
|
|
499
|
-
type: "string",
|
|
500
|
-
description: "Device ID",
|
|
501
|
-
},
|
|
502
|
-
},
|
|
503
|
-
required: ["device_id"],
|
|
504
|
-
},
|
|
505
|
-
},
|
|
228
|
+
// DSL execution
|
|
506
229
|
{
|
|
507
230
|
name: "execute_dsl",
|
|
508
|
-
description: `Execute a batch of
|
|
509
|
-
This is the PREFERRED method for complex automation as it's more reliable than sequential API calls.
|
|
231
|
+
description: `Execute a batch of DSL commands on a device. This is the primary tool for all device interaction — tap, type, swipe, observe, launch apps, assertions, web automation, and more.
|
|
510
232
|
|
|
511
|
-
|
|
233
|
+
Read the MCP resource mobai://reference/device-automation to learn how to control devices before using this tool.
|
|
512
234
|
|
|
513
|
-
|
|
514
|
-
{
|
|
515
|
-
"
|
|
516
|
-
"
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
{"action": "delay", "duration_ms": 500},
|
|
520
|
-
{"action": "observe", "context": "native", "include": ["ui_tree"]}
|
|
521
|
-
],
|
|
522
|
-
"on_fail": {"strategy": "retry", "max_retries": 2}
|
|
523
|
-
}`,
|
|
235
|
+
Input: JSON string with "version": "0.2" and "steps" array. Example:
|
|
236
|
+
{"version":"0.2","steps":[
|
|
237
|
+
{"action":"open_app","bundle_id":"com.apple.Preferences"},
|
|
238
|
+
{"action":"tap","predicate":{"text_contains":"Wi-Fi"}},
|
|
239
|
+
{"action":"wait_for","predicate":{"type":"switch"},"timeout_ms":3000}
|
|
240
|
+
]}`,
|
|
524
241
|
inputSchema: {
|
|
525
242
|
type: "object",
|
|
526
243
|
properties: {
|
|
527
|
-
device_id: {
|
|
528
|
-
|
|
529
|
-
description: "Device ID",
|
|
530
|
-
},
|
|
531
|
-
script: {
|
|
532
|
-
type: "object",
|
|
533
|
-
description: "DSL script object with version, steps, and optional on_fail",
|
|
534
|
-
properties: {
|
|
535
|
-
version: {
|
|
536
|
-
type: "string",
|
|
537
|
-
description: "DSL version (use '0.2')",
|
|
538
|
-
},
|
|
539
|
-
steps: {
|
|
540
|
-
type: "array",
|
|
541
|
-
description: "Array of action steps",
|
|
542
|
-
items: { type: "object" },
|
|
543
|
-
},
|
|
544
|
-
on_fail: {
|
|
545
|
-
type: "object",
|
|
546
|
-
description: "Failure handling strategy",
|
|
547
|
-
},
|
|
548
|
-
},
|
|
549
|
-
required: ["version", "steps"],
|
|
550
|
-
},
|
|
244
|
+
device_id: { type: "string", description: "Device ID" },
|
|
245
|
+
commands: { type: "string", description: "DSL script as JSON string with version and steps" },
|
|
551
246
|
},
|
|
552
|
-
required: ["device_id", "
|
|
247
|
+
required: ["device_id", "commands"],
|
|
553
248
|
},
|
|
554
249
|
},
|
|
250
|
+
// Test management
|
|
555
251
|
{
|
|
556
|
-
name: "
|
|
557
|
-
description: "
|
|
558
|
-
inputSchema: {
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
},
|
|
565
|
-
task: {
|
|
566
|
-
type: "string",
|
|
567
|
-
description: "Task description (e.g., 'Open Settings and enable WiFi')",
|
|
568
|
-
},
|
|
569
|
-
agent_type: {
|
|
570
|
-
type: "string",
|
|
571
|
-
enum: ["toolagent", "hierarchical", "classic"],
|
|
572
|
-
description: "Agent type (default: toolagent)",
|
|
573
|
-
},
|
|
574
|
-
use_vision: {
|
|
575
|
-
type: "boolean",
|
|
576
|
-
description: "Enable vision/screenshots (default: from app settings)",
|
|
577
|
-
},
|
|
578
|
-
},
|
|
579
|
-
required: ["device_id", "task"],
|
|
580
|
-
},
|
|
252
|
+
name: "test_get_active",
|
|
253
|
+
description: "Get the currently active test project and its cases. Use this to discover which test cases are available.",
|
|
254
|
+
inputSchema: { type: "object", properties: {}, required: [] },
|
|
255
|
+
},
|
|
256
|
+
{
|
|
257
|
+
name: "test_list_projects",
|
|
258
|
+
description: "List all test projects with their test cases included inline",
|
|
259
|
+
inputSchema: { type: "object", properties: {}, required: [] },
|
|
581
260
|
},
|
|
582
261
|
{
|
|
583
|
-
name: "
|
|
584
|
-
description: "
|
|
262
|
+
name: "test_create_project",
|
|
263
|
+
description: "Create a new test project",
|
|
585
264
|
inputSchema: {
|
|
586
265
|
type: "object",
|
|
587
|
-
properties: {
|
|
588
|
-
|
|
589
|
-
type: "string",
|
|
590
|
-
description: "Device ID",
|
|
591
|
-
},
|
|
592
|
-
},
|
|
593
|
-
required: ["device_id"],
|
|
266
|
+
properties: { name: { type: "string", description: "Project name" } },
|
|
267
|
+
required: ["name"],
|
|
594
268
|
},
|
|
595
269
|
},
|
|
596
270
|
{
|
|
597
|
-
name: "
|
|
598
|
-
description: "
|
|
271
|
+
name: "test_rename_project",
|
|
272
|
+
description: "Rename an existing test project",
|
|
599
273
|
inputSchema: {
|
|
600
274
|
type: "object",
|
|
601
275
|
properties: {
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
description: "Device ID",
|
|
605
|
-
},
|
|
606
|
-
url: {
|
|
607
|
-
type: "string",
|
|
608
|
-
description: "URL to navigate to",
|
|
609
|
-
},
|
|
276
|
+
project_id: { type: "string", description: "Project ID" },
|
|
277
|
+
name: { type: "string", description: "New project name" },
|
|
610
278
|
},
|
|
611
|
-
required: ["
|
|
279
|
+
required: ["project_id", "name"],
|
|
612
280
|
},
|
|
613
281
|
},
|
|
614
282
|
{
|
|
615
|
-
name: "
|
|
616
|
-
description: "
|
|
283
|
+
name: "test_create_case",
|
|
284
|
+
description: "Create a new test case in a project",
|
|
617
285
|
inputSchema: {
|
|
618
286
|
type: "object",
|
|
619
287
|
properties: {
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
},
|
|
288
|
+
project_id: { type: "string", description: "Project ID" },
|
|
289
|
+
name: { type: "string", description: "Test case name" },
|
|
290
|
+
folder: { type: "string", description: "Optional folder path within the project" },
|
|
624
291
|
},
|
|
625
|
-
required: ["
|
|
292
|
+
required: ["project_id", "name"],
|
|
626
293
|
},
|
|
627
294
|
},
|
|
628
295
|
{
|
|
629
|
-
name: "
|
|
630
|
-
description: "
|
|
296
|
+
name: "test_rename_case",
|
|
297
|
+
description: "Rename an existing test case",
|
|
631
298
|
inputSchema: {
|
|
632
299
|
type: "object",
|
|
633
300
|
properties: {
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
},
|
|
638
|
-
selector: {
|
|
639
|
-
type: "string",
|
|
640
|
-
description: "CSS selector (e.g., 'button.submit', '#login-btn')",
|
|
641
|
-
},
|
|
301
|
+
project_id: { type: "string", description: "Project ID" },
|
|
302
|
+
case_id: { type: "string", description: "Test case ID" },
|
|
303
|
+
name: { type: "string", description: "New test case name" },
|
|
642
304
|
},
|
|
643
|
-
required: ["
|
|
305
|
+
required: ["project_id", "case_id", "name"],
|
|
644
306
|
},
|
|
645
307
|
},
|
|
646
308
|
{
|
|
647
|
-
name: "
|
|
648
|
-
description: "
|
|
309
|
+
name: "test_delete_case",
|
|
310
|
+
description: "Delete a test case from a project",
|
|
649
311
|
inputSchema: {
|
|
650
312
|
type: "object",
|
|
651
313
|
properties: {
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
description: "Device ID",
|
|
655
|
-
},
|
|
656
|
-
selector: {
|
|
657
|
-
type: "string",
|
|
658
|
-
description: "CSS selector for the input element",
|
|
659
|
-
},
|
|
660
|
-
text: {
|
|
661
|
-
type: "string",
|
|
662
|
-
description: "Text to type",
|
|
663
|
-
},
|
|
314
|
+
project_id: { type: "string", description: "Project ID" },
|
|
315
|
+
case_id: { type: "string", description: "Test case ID" },
|
|
664
316
|
},
|
|
665
|
-
required: ["
|
|
317
|
+
required: ["project_id", "case_id"],
|
|
666
318
|
},
|
|
667
319
|
},
|
|
668
320
|
{
|
|
669
|
-
name: "
|
|
670
|
-
description: "
|
|
321
|
+
name: "test_get_script",
|
|
322
|
+
description: "Get the .mob script content for a test case (with 1-based line numbers)",
|
|
671
323
|
inputSchema: {
|
|
672
324
|
type: "object",
|
|
673
325
|
properties: {
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
description: "Device ID",
|
|
677
|
-
},
|
|
678
|
-
script: {
|
|
679
|
-
type: "string",
|
|
680
|
-
description: "JavaScript code to execute (use 'return' for results)",
|
|
681
|
-
},
|
|
326
|
+
project_id: { type: "string", description: "Project ID" },
|
|
327
|
+
case_id: { type: "string", description: "Test case ID" },
|
|
682
328
|
},
|
|
683
|
-
required: ["
|
|
329
|
+
required: ["project_id", "case_id"],
|
|
684
330
|
},
|
|
685
331
|
},
|
|
686
332
|
{
|
|
687
|
-
name: "
|
|
688
|
-
description: "
|
|
333
|
+
name: "test_replace_script",
|
|
334
|
+
description: "Replace the entire .mob script for a test case",
|
|
689
335
|
inputSchema: {
|
|
690
336
|
type: "object",
|
|
691
337
|
properties: {
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
},
|
|
696
|
-
bundle_id: {
|
|
697
|
-
type: "string",
|
|
698
|
-
description: "App bundle ID (iOS) or package name (Android) to uninstall",
|
|
699
|
-
},
|
|
338
|
+
project_id: { type: "string", description: "Project ID" },
|
|
339
|
+
case_id: { type: "string", description: "Test case ID" },
|
|
340
|
+
script: { type: "string", description: "New script content (without line numbers)" },
|
|
700
341
|
},
|
|
701
|
-
required: ["
|
|
342
|
+
required: ["project_id", "case_id", "script"],
|
|
702
343
|
},
|
|
703
344
|
},
|
|
704
345
|
{
|
|
705
|
-
name: "
|
|
706
|
-
description: "
|
|
346
|
+
name: "test_update_line",
|
|
347
|
+
description: "Update a single line in the .mob script",
|
|
707
348
|
inputSchema: {
|
|
708
349
|
type: "object",
|
|
709
350
|
properties: {
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
},
|
|
714
|
-
bundle_id: {
|
|
715
|
-
type: "string",
|
|
716
|
-
description: "Bundle ID / package name of the app to kill",
|
|
717
|
-
},
|
|
351
|
+
project_id: { type: "string", description: "Project ID" },
|
|
352
|
+
case_id: { type: "string", description: "Test case ID" },
|
|
353
|
+
line_number: { type: "number", description: "1-based line number to update" },
|
|
354
|
+
content: { type: "string", description: "New line content" },
|
|
718
355
|
},
|
|
719
|
-
required: ["
|
|
356
|
+
required: ["project_id", "case_id", "line_number", "content"],
|
|
720
357
|
},
|
|
721
358
|
},
|
|
722
359
|
{
|
|
723
|
-
name: "
|
|
724
|
-
description: "
|
|
360
|
+
name: "test_insert_after",
|
|
361
|
+
description: "Insert a new line after the specified line number in the .mob script",
|
|
725
362
|
inputSchema: {
|
|
726
363
|
type: "object",
|
|
727
364
|
properties: {
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
},
|
|
732
|
-
lat: {
|
|
733
|
-
type: "number",
|
|
734
|
-
description: "Latitude (-90 to 90)",
|
|
735
|
-
},
|
|
736
|
-
lon: {
|
|
737
|
-
type: "number",
|
|
738
|
-
description: "Longitude (-180 to 180)",
|
|
739
|
-
},
|
|
365
|
+
project_id: { type: "string", description: "Project ID" },
|
|
366
|
+
case_id: { type: "string", description: "Test case ID" },
|
|
367
|
+
line_number: { type: "number", description: "1-based line number to insert after (0 = insert at beginning)" },
|
|
368
|
+
content: { type: "string", description: "Line content to insert" },
|
|
740
369
|
},
|
|
741
|
-
required: ["
|
|
370
|
+
required: ["project_id", "case_id", "line_number", "content"],
|
|
742
371
|
},
|
|
743
372
|
},
|
|
744
373
|
{
|
|
745
|
-
name: "
|
|
746
|
-
description: "
|
|
374
|
+
name: "test_delete_line",
|
|
375
|
+
description: "Delete a line from the .mob script",
|
|
747
376
|
inputSchema: {
|
|
748
377
|
type: "object",
|
|
749
378
|
properties: {
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
},
|
|
379
|
+
project_id: { type: "string", description: "Project ID" },
|
|
380
|
+
case_id: { type: "string", description: "Test case ID" },
|
|
381
|
+
line_number: { type: "number", description: "1-based line number to delete" },
|
|
754
382
|
},
|
|
755
|
-
required: ["
|
|
383
|
+
required: ["project_id", "case_id", "line_number"],
|
|
756
384
|
},
|
|
757
385
|
},
|
|
758
386
|
{
|
|
759
|
-
name: "
|
|
760
|
-
description:
|
|
761
|
-
|
|
762
|
-
Base URL: http://127.0.0.1:8686/api/v1
|
|
763
|
-
|
|
764
|
-
Common endpoints:
|
|
765
|
-
- GET /devices - List devices
|
|
766
|
-
- GET /devices/{id}/screenshot - Take screenshot
|
|
767
|
-
- GET /devices/{id}/ui-tree - Get UI tree
|
|
768
|
-
- POST /devices/{id}/dsl/execute - Execute DSL script
|
|
769
|
-
- POST /devices/{id}/agent/run - Run AI agent`,
|
|
387
|
+
name: "test_run",
|
|
388
|
+
description: "Run a test case on a device",
|
|
770
389
|
inputSchema: {
|
|
771
390
|
type: "object",
|
|
772
391
|
properties: {
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
description: "HTTP method",
|
|
777
|
-
},
|
|
778
|
-
url: {
|
|
779
|
-
type: "string",
|
|
780
|
-
description: "Full URL or endpoint path (e.g., /devices)",
|
|
781
|
-
},
|
|
782
|
-
body: {
|
|
783
|
-
type: "string",
|
|
784
|
-
description: "Request body as JSON string",
|
|
785
|
-
},
|
|
786
|
-
timeout_ms: {
|
|
787
|
-
type: "number",
|
|
788
|
-
description: "Timeout in milliseconds (default: 600000)",
|
|
789
|
-
},
|
|
392
|
+
project_id: { type: "string", description: "Project ID" },
|
|
393
|
+
case_id: { type: "string", description: "Test case ID" },
|
|
394
|
+
device_id: { type: "string", description: "Device ID to run the test on" },
|
|
790
395
|
},
|
|
791
|
-
required: ["
|
|
396
|
+
required: ["project_id", "case_id", "device_id"],
|
|
792
397
|
},
|
|
793
398
|
},
|
|
794
399
|
];
|
|
795
|
-
//
|
|
400
|
+
// ---------------------------------------------------------------------------
|
|
401
|
+
// List tools
|
|
402
|
+
// ---------------------------------------------------------------------------
|
|
796
403
|
server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
797
404
|
return { tools: TOOLS };
|
|
798
405
|
});
|
|
799
|
-
//
|
|
406
|
+
// ---------------------------------------------------------------------------
|
|
407
|
+
// Tool call handler
|
|
408
|
+
// ---------------------------------------------------------------------------
|
|
409
|
+
function testCasePath(args) {
|
|
410
|
+
const projectId = args?.project_id;
|
|
411
|
+
const caseId = args?.case_id;
|
|
412
|
+
if (!projectId || !caseId)
|
|
413
|
+
throw new Error("project_id and case_id are required");
|
|
414
|
+
return `/tests/projects/${projectId}/cases/${caseId}`;
|
|
415
|
+
}
|
|
800
416
|
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
801
417
|
const { name, arguments: args } = request.params;
|
|
802
418
|
try {
|
|
803
|
-
let result;
|
|
804
419
|
switch (name) {
|
|
420
|
+
// Device management
|
|
805
421
|
case "list_devices":
|
|
806
|
-
|
|
807
|
-
break;
|
|
422
|
+
return textResult(await doGet("/devices"));
|
|
808
423
|
case "get_device":
|
|
809
|
-
|
|
810
|
-
break;
|
|
424
|
+
return textResult(await doGet(`/devices/${args?.device_id}`));
|
|
811
425
|
case "start_bridge":
|
|
812
|
-
|
|
813
|
-
break;
|
|
426
|
+
return textResult(await doPost(`/devices/${args?.device_id}/bridge/start`));
|
|
814
427
|
case "stop_bridge":
|
|
815
|
-
|
|
816
|
-
|
|
428
|
+
return textResult(await doPost(`/devices/${args?.device_id}/bridge/stop`));
|
|
429
|
+
// Screenshots
|
|
817
430
|
case "get_screenshot": {
|
|
818
|
-
const
|
|
431
|
+
const body = await doGet(`/devices/${args?.device_id}/screenshot?low_quality=true`);
|
|
432
|
+
return textResult(screenshotToFile(body));
|
|
433
|
+
}
|
|
434
|
+
case "save_screenshot": {
|
|
435
|
+
const params = new URLSearchParams();
|
|
819
436
|
if (args?.path)
|
|
820
|
-
|
|
437
|
+
params.set("path", args.path);
|
|
821
438
|
if (args?.name)
|
|
822
|
-
|
|
823
|
-
const
|
|
824
|
-
|
|
825
|
-
|
|
439
|
+
params.set("name", args.name);
|
|
440
|
+
const query = params.toString();
|
|
441
|
+
const body = await doGet(`/devices/${args?.device_id}/screenshot${query ? "?" + query : ""}`);
|
|
442
|
+
return textResult(screenshotToFile(body));
|
|
826
443
|
}
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
if (
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
params.set("boundsH", String(b.h));
|
|
444
|
+
// App management
|
|
445
|
+
case "list_apps":
|
|
446
|
+
return textResult(await doGet(`/devices/${args?.device_id}/apps`));
|
|
447
|
+
case "install_app":
|
|
448
|
+
return textResult(await doPost(`/devices/${args?.device_id}/install-app`, { path: args?.path }));
|
|
449
|
+
case "uninstall_app":
|
|
450
|
+
return textResult(await doDelete(`/devices/${args?.device_id}/apps/${encodeURIComponent(args?.bundle_id)}`));
|
|
451
|
+
// DSL execution
|
|
452
|
+
case "execute_dsl": {
|
|
453
|
+
const commandsStr = args?.commands;
|
|
454
|
+
if (!commandsStr)
|
|
455
|
+
throw new Error("commands is required");
|
|
456
|
+
let script;
|
|
457
|
+
try {
|
|
458
|
+
script = JSON.parse(commandsStr);
|
|
843
459
|
}
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
result = await makeRequest("GET", endpoint);
|
|
847
|
-
break;
|
|
848
|
-
}
|
|
849
|
-
case "tap": {
|
|
850
|
-
const body = {};
|
|
851
|
-
if (args?.index !== undefined)
|
|
852
|
-
body.index = args.index;
|
|
853
|
-
if (args?.x !== undefined && args?.y !== undefined) {
|
|
854
|
-
body.x = args.x;
|
|
855
|
-
body.y = args.y;
|
|
460
|
+
catch {
|
|
461
|
+
throw new Error("invalid DSL JSON: " + commandsStr);
|
|
856
462
|
}
|
|
857
|
-
|
|
858
|
-
|
|
463
|
+
const body = await doPost(`/devices/${args?.device_id}/dsl/execute`, script);
|
|
464
|
+
return textResult(extractDSLScreenshots(body));
|
|
859
465
|
}
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
466
|
+
// Test management
|
|
467
|
+
case "test_get_active":
|
|
468
|
+
return textResult(await doGet("/tests/active"));
|
|
469
|
+
case "test_list_projects":
|
|
470
|
+
return textResult(await doGet("/tests/projects"));
|
|
471
|
+
case "test_create_project":
|
|
472
|
+
return textResult(await doPost("/tests/projects", { name: args?.name }));
|
|
473
|
+
case "test_rename_project":
|
|
474
|
+
return textResult(await doPatch(`/tests/projects/${args?.project_id}`, { name: args?.name }));
|
|
475
|
+
case "test_create_case": {
|
|
476
|
+
const body = { name: args?.name };
|
|
477
|
+
if (args?.folder)
|
|
478
|
+
body.folder = args.folder;
|
|
479
|
+
return textResult(await doPost(`/tests/projects/${args?.project_id}/cases`, body));
|
|
870
480
|
}
|
|
871
|
-
case "
|
|
872
|
-
const
|
|
873
|
-
|
|
874
|
-
body.index = args.index;
|
|
875
|
-
if (args?.x !== undefined && args?.y !== undefined) {
|
|
876
|
-
body.x = args.x;
|
|
877
|
-
body.y = args.y;
|
|
878
|
-
}
|
|
879
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/long-press`, body);
|
|
880
|
-
break;
|
|
481
|
+
case "test_rename_case": {
|
|
482
|
+
const p = testCasePath(args);
|
|
483
|
+
return textResult(await doPatch(p, { name: args?.name }));
|
|
881
484
|
}
|
|
882
|
-
case "
|
|
883
|
-
const
|
|
884
|
-
|
|
885
|
-
body.index = args.index;
|
|
886
|
-
if (args?.x !== undefined && args?.y !== undefined) {
|
|
887
|
-
body.x = args.x;
|
|
888
|
-
body.y = args.y;
|
|
889
|
-
}
|
|
890
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/two-finger-tap`, body);
|
|
891
|
-
break;
|
|
485
|
+
case "test_delete_case": {
|
|
486
|
+
const p = testCasePath(args);
|
|
487
|
+
return textResult(await doDelete(p));
|
|
892
488
|
}
|
|
893
|
-
case "
|
|
894
|
-
const
|
|
895
|
-
|
|
896
|
-
fromY: args?.from_y,
|
|
897
|
-
toX: args?.to_x,
|
|
898
|
-
toY: args?.to_y,
|
|
899
|
-
duration: args?.duration_ms ?? 500,
|
|
900
|
-
};
|
|
901
|
-
if (args?.press_duration_ms) {
|
|
902
|
-
dragBody.pressDuration = args.press_duration_ms;
|
|
903
|
-
}
|
|
904
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/drag`, dragBody);
|
|
905
|
-
break;
|
|
489
|
+
case "test_get_script": {
|
|
490
|
+
const p = testCasePath(args);
|
|
491
|
+
return textResult(await doGet(`${p}/script`));
|
|
906
492
|
}
|
|
907
|
-
case "
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
case "type_text":
|
|
911
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/type`, { text: args?.text });
|
|
912
|
-
break;
|
|
913
|
-
case "swipe":
|
|
914
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/swipe`, {
|
|
915
|
-
fromX: args?.from_x,
|
|
916
|
-
fromY: args?.from_y,
|
|
917
|
-
toX: args?.to_x,
|
|
918
|
-
toY: args?.to_y,
|
|
919
|
-
duration: args?.duration_ms ?? 300,
|
|
920
|
-
});
|
|
921
|
-
break;
|
|
922
|
-
case "go_home":
|
|
923
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/go-home`);
|
|
924
|
-
break;
|
|
925
|
-
case "launch_app":
|
|
926
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/launch-app`, {
|
|
927
|
-
bundleId: args?.bundle_id,
|
|
928
|
-
});
|
|
929
|
-
break;
|
|
930
|
-
case "list_apps":
|
|
931
|
-
result = await makeRequest("GET", `/devices/${args?.device_id}/apps`);
|
|
932
|
-
break;
|
|
933
|
-
case "get_ocr":
|
|
934
|
-
result = await makeRequest("GET", `/devices/${args?.device_id}/ocr`);
|
|
935
|
-
break;
|
|
936
|
-
case "uninstall_app":
|
|
937
|
-
result = await makeRequest("DELETE", `/devices/${args?.device_id}/apps/${encodeURIComponent(args?.bundle_id)}`);
|
|
938
|
-
break;
|
|
939
|
-
case "kill_app":
|
|
940
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/kill-app`, {
|
|
941
|
-
bundleId: args?.bundle_id,
|
|
942
|
-
});
|
|
943
|
-
break;
|
|
944
|
-
case "set_location":
|
|
945
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/location`, {
|
|
946
|
-
lat: args?.lat,
|
|
947
|
-
lon: args?.lon,
|
|
948
|
-
});
|
|
949
|
-
break;
|
|
950
|
-
case "reset_location":
|
|
951
|
-
result = await makeRequest("DELETE", `/devices/${args?.device_id}/location`);
|
|
952
|
-
break;
|
|
953
|
-
case "execute_dsl":
|
|
954
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/dsl/execute`, args?.script, 300000 // 5 minutes
|
|
955
|
-
);
|
|
956
|
-
break;
|
|
957
|
-
case "run_agent": {
|
|
958
|
-
const agentBody = { task: args?.task };
|
|
959
|
-
if (args?.agent_type)
|
|
960
|
-
agentBody.agentType = args.agent_type;
|
|
961
|
-
if (args?.use_vision !== undefined)
|
|
962
|
-
agentBody.useVision = args.use_vision;
|
|
963
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/agent/run`, agentBody, 600000 // 10 minutes
|
|
964
|
-
);
|
|
965
|
-
break;
|
|
493
|
+
case "test_replace_script": {
|
|
494
|
+
const p = testCasePath(args);
|
|
495
|
+
return textResult(await doPut(`${p}/script`, { script: args?.script }));
|
|
966
496
|
}
|
|
967
|
-
case "
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
result = await makeRequest("POST", `/devices/${args?.device_id}/web/execute`, {
|
|
991
|
-
script: args?.script,
|
|
992
|
-
});
|
|
993
|
-
break;
|
|
994
|
-
case "http_request": {
|
|
995
|
-
const url = (args?.url).startsWith("http")
|
|
996
|
-
? args?.url
|
|
997
|
-
: `${API_BASE_URL}${args?.url}`;
|
|
998
|
-
let body = undefined;
|
|
999
|
-
if (args?.body) {
|
|
1000
|
-
try {
|
|
1001
|
-
body = JSON.parse(args.body);
|
|
1002
|
-
}
|
|
1003
|
-
catch {
|
|
1004
|
-
body = args.body;
|
|
1005
|
-
}
|
|
1006
|
-
}
|
|
1007
|
-
result = await makeRequest(args?.method, url, body, args?.timeout_ms ?? DEFAULT_TIMEOUT_MS);
|
|
1008
|
-
break;
|
|
497
|
+
case "test_update_line": {
|
|
498
|
+
const p = testCasePath(args);
|
|
499
|
+
return textResult(await doPost(`${p}/script/update-line`, {
|
|
500
|
+
line_number: args?.line_number,
|
|
501
|
+
content: args?.content,
|
|
502
|
+
}));
|
|
503
|
+
}
|
|
504
|
+
case "test_insert_after": {
|
|
505
|
+
const p = testCasePath(args);
|
|
506
|
+
return textResult(await doPost(`${p}/script/insert-after`, {
|
|
507
|
+
line_number: args?.line_number,
|
|
508
|
+
content: args?.content,
|
|
509
|
+
}));
|
|
510
|
+
}
|
|
511
|
+
case "test_delete_line": {
|
|
512
|
+
const p = testCasePath(args);
|
|
513
|
+
return textResult(await doPost(`${p}/script/delete-line`, {
|
|
514
|
+
line_number: args?.line_number,
|
|
515
|
+
}));
|
|
516
|
+
}
|
|
517
|
+
case "test_run": {
|
|
518
|
+
const p = testCasePath(args);
|
|
519
|
+
return textResult(await doPost(`${p}/run`, { device_id: args?.device_id }));
|
|
1009
520
|
}
|
|
1010
521
|
default:
|
|
1011
|
-
return {
|
|
1012
|
-
content: [{ type: "text", text: `Unknown tool: ${name}` }],
|
|
1013
|
-
isError: true,
|
|
1014
|
-
};
|
|
522
|
+
return { content: [{ type: "text", text: `Unknown tool: ${name}` }], isError: true };
|
|
1015
523
|
}
|
|
1016
|
-
const formattedBody = typeof result.body === "string"
|
|
1017
|
-
? result.body
|
|
1018
|
-
: JSON.stringify(result.body, null, 2);
|
|
1019
|
-
return {
|
|
1020
|
-
content: [
|
|
1021
|
-
{
|
|
1022
|
-
type: "text",
|
|
1023
|
-
text: `Status: ${result.status} ${result.statusText}\n\n${formattedBody}`,
|
|
1024
|
-
},
|
|
1025
|
-
],
|
|
1026
|
-
isError: result.status >= 400,
|
|
1027
|
-
};
|
|
1028
524
|
}
|
|
1029
525
|
catch (error) {
|
|
1030
526
|
if (error instanceof Error && error.name === "AbortError") {
|
|
1031
|
-
return
|
|
1032
|
-
content: [{ type: "text", text: "Request timed out" }],
|
|
1033
|
-
isError: true,
|
|
1034
|
-
};
|
|
527
|
+
return errResult("Request timed out");
|
|
1035
528
|
}
|
|
1036
|
-
|
|
1037
|
-
return {
|
|
1038
|
-
content: [{ type: "text", text: `Error: ${message}` }],
|
|
1039
|
-
isError: true,
|
|
1040
|
-
};
|
|
529
|
+
return errResult(error);
|
|
1041
530
|
}
|
|
1042
531
|
});
|
|
1043
|
-
//
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
uri: "mobai://api-reference",
|
|
1047
|
-
name: "MobAI API Reference",
|
|
1048
|
-
description: "Complete API documentation for MobAI HTTP API",
|
|
1049
|
-
mimeType: "text/markdown",
|
|
1050
|
-
},
|
|
1051
|
-
{
|
|
1052
|
-
uri: "mobai://dsl-guide",
|
|
1053
|
-
name: "DSL Automation Guide",
|
|
1054
|
-
description: "Guide for using the DSL batch execution system",
|
|
1055
|
-
mimeType: "text/markdown",
|
|
1056
|
-
},
|
|
1057
|
-
{
|
|
1058
|
-
uri: "mobai://native-runner",
|
|
1059
|
-
name: "Native App Automation",
|
|
1060
|
-
description: "Guide for automating native mobile apps",
|
|
1061
|
-
mimeType: "text/markdown",
|
|
1062
|
-
},
|
|
1063
|
-
{
|
|
1064
|
-
uri: "mobai://web-runner",
|
|
1065
|
-
name: "Web Automation",
|
|
1066
|
-
description: "Guide for automating browsers and WebViews",
|
|
1067
|
-
mimeType: "text/markdown",
|
|
1068
|
-
},
|
|
1069
|
-
];
|
|
1070
|
-
// Handle list resources request
|
|
1071
|
-
server.setRequestHandler(ListResourcesRequestSchema, async () => {
|
|
1072
|
-
return { resources: RESOURCES };
|
|
1073
|
-
});
|
|
1074
|
-
// Handle read resource request
|
|
1075
|
-
server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
|
|
1076
|
-
const { uri } = request.params;
|
|
1077
|
-
const content = getResourceContent(uri);
|
|
1078
|
-
if (!content) {
|
|
1079
|
-
throw new Error(`Resource not found: ${uri}`);
|
|
1080
|
-
}
|
|
1081
|
-
return {
|
|
1082
|
-
contents: [
|
|
1083
|
-
{
|
|
1084
|
-
uri,
|
|
1085
|
-
mimeType: "text/markdown",
|
|
1086
|
-
text: content,
|
|
1087
|
-
},
|
|
1088
|
-
],
|
|
1089
|
-
};
|
|
1090
|
-
});
|
|
1091
|
-
function getResourceContent(uri) {
|
|
1092
|
-
switch (uri) {
|
|
1093
|
-
case "mobai://api-reference":
|
|
1094
|
-
return API_REFERENCE;
|
|
1095
|
-
case "mobai://dsl-guide":
|
|
1096
|
-
return DSL_GUIDE;
|
|
1097
|
-
case "mobai://native-runner":
|
|
1098
|
-
return NATIVE_RUNNER_GUIDE;
|
|
1099
|
-
case "mobai://web-runner":
|
|
1100
|
-
return WEB_RUNNER_GUIDE;
|
|
1101
|
-
default:
|
|
1102
|
-
return null;
|
|
1103
|
-
}
|
|
1104
|
-
}
|
|
1105
|
-
// Resource content
|
|
1106
|
-
const API_REFERENCE = `# MobAI API Reference
|
|
1107
|
-
|
|
1108
|
-
**Base URL:** \`http://127.0.0.1:8686/api/v1\`
|
|
1109
|
-
|
|
1110
|
-
## Device Management
|
|
1111
|
-
|
|
1112
|
-
| Endpoint | Method | Description |
|
|
1113
|
-
|----------|--------|-------------|
|
|
1114
|
-
| /devices | GET | List all connected devices |
|
|
1115
|
-
| /devices/{id} | GET | Get device info |
|
|
1116
|
-
| /devices/{id}/screenshot | GET | Capture screenshot (saved to /tmp/mobai/screenshots/) |
|
|
1117
|
-
| /devices/{id}/ui-tree | GET | Get UI accessibility tree |
|
|
1118
|
-
| /devices/{id}/apps | GET | List installed apps |
|
|
1119
|
-
| /devices/{id}/ocr | GET | OCR text recognition (iOS only) |
|
|
1120
|
-
|
|
1121
|
-
## Bridge Control
|
|
1122
|
-
|
|
1123
|
-
| Endpoint | Method | Description |
|
|
1124
|
-
|----------|--------|-------------|
|
|
1125
|
-
| /devices/{id}/bridge/start | POST | Start on-device bridge (required for automation) |
|
|
1126
|
-
| /devices/{id}/bridge/stop | POST | Stop bridge |
|
|
1127
|
-
|
|
1128
|
-
## UI Operations
|
|
1129
|
-
|
|
1130
|
-
| Endpoint | Method | Description |
|
|
1131
|
-
|----------|--------|-------------|
|
|
1132
|
-
| /devices/{id}/tap | POST | Tap element: {"index": N} or {"x": X, "y": Y} |
|
|
1133
|
-
| /devices/{id}/double-tap | POST | Double tap: {"index": N} or {"x": X, "y": Y} |
|
|
1134
|
-
| /devices/{id}/long-press | POST | Long press (0.5s): {"index": N} or {"x": X, "y": Y} |
|
|
1135
|
-
| /devices/{id}/two-finger-tap | POST | Two-finger tap (iOS): {"index": N} or {"x": X, "y": Y} |
|
|
1136
|
-
| /devices/{id}/swipe | POST | Swipe: {"fromX", "fromY", "toX", "toY", "duration"} |
|
|
1137
|
-
| /devices/{id}/drag | POST | Drag: {"fromX", "fromY", "toX", "toY", "duration", "pressDuration"} |
|
|
1138
|
-
| /devices/{id}/type | POST | Type text: {"text": "..."} |
|
|
1139
|
-
| /devices/{id}/dismiss-keyboard | POST | Dismiss on-screen keyboard |
|
|
1140
|
-
| /devices/{id}/go-home | POST | Go to home screen |
|
|
1141
|
-
| /devices/{id}/launch-app | POST | Launch app: {"bundleId": "..."} |
|
|
1142
|
-
| /devices/{id}/apps/{bundleId} | DELETE | Uninstall app by bundle ID |
|
|
1143
|
-
| /devices/{id}/kill-app | POST | Kill app: {"bundleId": "..."} |
|
|
1144
|
-
|
|
1145
|
-
## DSL Execution
|
|
1146
|
-
|
|
1147
|
-
| Endpoint | Method | Description |
|
|
1148
|
-
|----------|--------|-------------|
|
|
1149
|
-
| /devices/{id}/dsl/execute | POST | Execute DSL batch script |
|
|
1150
|
-
|
|
1151
|
-
## AI Agent
|
|
1152
|
-
|
|
1153
|
-
| Endpoint | Method | Description |
|
|
1154
|
-
|----------|--------|-------------|
|
|
1155
|
-
| /devices/{id}/agent/run | POST | Run AI agent: {"task": "..."} |
|
|
1156
|
-
|
|
1157
|
-
## Performance Metrics
|
|
1158
|
-
|
|
1159
|
-
| Endpoint | Method | Description |
|
|
1160
|
-
|----------|--------|-------------|
|
|
1161
|
-
| /devices/{id}/metrics/start | POST | Start metrics collection |
|
|
1162
|
-
| /devices/{id}/metrics/stop | POST | Stop collection, return summary |
|
|
1163
|
-
| /devices/{id}/metrics | GET | Get raw metrics buffer |
|
|
1164
|
-
| /devices/{id}/metrics/summary | GET | Get current summary without stopping |
|
|
1165
|
-
|
|
1166
|
-
## Web Automation
|
|
1167
|
-
|
|
1168
|
-
| Endpoint | Method | Description |
|
|
1169
|
-
|----------|--------|-------------|
|
|
1170
|
-
| /devices/{id}/web/pages | GET | List browser tabs/WebViews |
|
|
1171
|
-
| /devices/{id}/web/navigate | POST | Navigate to URL: {"url": "..."} |
|
|
1172
|
-
| /devices/{id}/web/dom | GET | Get DOM tree |
|
|
1173
|
-
| /devices/{id}/web/click | POST | Click element: {"selector": "..."} |
|
|
1174
|
-
| /devices/{id}/web/type | POST | Type text: {"selector": "...", "text": "..."} |
|
|
1175
|
-
| /devices/{id}/web/execute | POST | Execute JS: {"script": "..."} |
|
|
1176
|
-
|
|
1177
|
-
## Response Format
|
|
1178
|
-
|
|
1179
|
-
**Success:**
|
|
1180
|
-
\`\`\`json
|
|
1181
|
-
{"success": true, "data": {...}}
|
|
1182
|
-
\`\`\`
|
|
1183
|
-
|
|
1184
|
-
**Error:**
|
|
1185
|
-
\`\`\`json
|
|
1186
|
-
{"error": "message", "code": "ERROR_CODE"}
|
|
1187
|
-
\`\`\`
|
|
1188
|
-
|
|
1189
|
-
## DSL Action Reference
|
|
1190
|
-
|
|
1191
|
-
### type Action
|
|
1192
|
-
- **predicate**: Required if keyboard not already open (auto-taps the element first)
|
|
1193
|
-
- **dismiss_keyboard**: Default \`false\` (keyboard stays open after typing)
|
|
1194
|
-
- **clear_first**: Optional, clears field before typing
|
|
1195
|
-
|
|
1196
|
-
\`\`\`json
|
|
1197
|
-
{"action": "type", "text": "hello", "predicate": {"type": "input"}}
|
|
1198
|
-
\`\`\`
|
|
1199
|
-
|
|
1200
|
-
### press_key Action
|
|
1201
|
-
- **key**: Keyboard key to press (return, tab, delete, escape, etc.)
|
|
1202
|
-
- **context**: Optional, "web" for web context (supports enter, tab, delete, escape)
|
|
1203
|
-
|
|
1204
|
-
\`\`\`json
|
|
1205
|
-
{"action": "press_key", "key": "return"}
|
|
1206
|
-
{"action": "press_key", "key": "tab", "context": "web"}
|
|
1207
|
-
\`\`\`
|
|
1208
|
-
|
|
1209
|
-
### select_web_context Action
|
|
1210
|
-
- **url_contains**: Filter by URL substring
|
|
1211
|
-
- **title_contains**: Filter by page title substring
|
|
1212
|
-
|
|
1213
|
-
\`\`\`json
|
|
1214
|
-
{"action": "select_web_context"}
|
|
1215
|
-
{"action": "select_web_context", "url_contains": "example.com"}
|
|
1216
|
-
{"action": "select_web_context", "title_contains": "Login"}
|
|
1217
|
-
\`\`\`
|
|
1218
|
-
`;
|
|
1219
|
-
const DSL_GUIDE = `# MobAI DSL Guide
|
|
1220
|
-
|
|
1221
|
-
The DSL (Domain Specific Language) enables batch execution of multiple automation steps in a single request.
|
|
1222
|
-
|
|
1223
|
-
## Basic Structure
|
|
1224
|
-
|
|
1225
|
-
\`\`\`json
|
|
1226
|
-
{
|
|
1227
|
-
"version": "0.2",
|
|
1228
|
-
"steps": [
|
|
1229
|
-
{"action": "observe", "context": "native", "include": ["ui_tree"]},
|
|
1230
|
-
{"action": "tap", "predicate": {"text_contains": "Settings"}}
|
|
1231
|
-
],
|
|
1232
|
-
"on_fail": {"strategy": "retry", "max_retries": 2}
|
|
1233
|
-
}
|
|
1234
|
-
\`\`\`
|
|
1235
|
-
|
|
1236
|
-
## Available Actions
|
|
1237
|
-
|
|
1238
|
-
| Action | Description | Key Fields |
|
|
1239
|
-
|--------|-------------|------------|
|
|
1240
|
-
| observe | Get UI tree/screenshot/OCR | context, include (ui_tree, screenshot, installed_apps, ocr), filter ({text_regex, bounds}) |
|
|
1241
|
-
| tap | Tap element | predicate or coords |
|
|
1242
|
-
| double_tap | Double-tap element | predicate or coords |
|
|
1243
|
-
| long_press | Long-press element | predicate or coords, duration_ms (default: 1000) |
|
|
1244
|
-
| two_finger_tap | Two-finger tap element | predicate or coords |
|
|
1245
|
-
| type | Type text | text, predicate (if keyboard not open), clear_first, dismiss_keyboard (default: false) |
|
|
1246
|
-
| press_key | Press keyboard key | key (return, tab, delete, etc.), context (optional: "web") |
|
|
1247
|
-
| toggle | Set switch state | predicate, state ("on"/"off") |
|
|
1248
|
-
| swipe | Swipe gesture | direction, distance, duration_ms, or from_coords/to_coords |
|
|
1249
|
-
| scroll | Scroll in container | direction, predicate (container), to_element, max_scrolls |
|
|
1250
|
-
| drag | Drag element to target | from (predicate), to_element (predicate), or from_coords/to_coords, duration_ms, press_duration_ms |
|
|
1251
|
-
| open_app | Launch app | bundle_id |
|
|
1252
|
-
| kill_app | Force-kill running app | bundle_id |
|
|
1253
|
-
| navigate | Go home/back | target ("home", "back") |
|
|
1254
|
-
| wait_for | Wait for element or UI stability | predicate, timeout_ms, poll_interval_ms, stable (wait for UI to stop changing) |
|
|
1255
|
-
| delay | Wait fixed time | duration_ms |
|
|
1256
|
-
| screenshot | Save screenshot to file | file_path (directory), name (optional filename) |
|
|
1257
|
-
| assert_exists | Verify element exists | predicate, timeout_ms |
|
|
1258
|
-
| assert_not_exists | Verify element gone | predicate |
|
|
1259
|
-
| assert_count | Verify element count | predicate, count |
|
|
1260
|
-
| assert_screen_changed | Verify screen changed | (compared to last observe) |
|
|
1261
|
-
| checkpoint | Mark a test checkpoint | name |
|
|
1262
|
-
| if_exists | Conditional | predicate, then, else |
|
|
1263
|
-
| select_web_context | Select browser/WebView | url_contains, title_contains (optional filters) |
|
|
1264
|
-
| execute_js | Run JavaScript in web context | script |
|
|
1265
|
-
| set_location | Simulate GPS location (Android 12+ for real devices) | lat, lon |
|
|
1266
|
-
| reset_location | Reset to real GPS (Android 12+ for real devices) | (no fields) |
|
|
1267
|
-
| metrics_start | Start performance monitoring | types, bundle_id, label, thresholds, capture_logs |
|
|
1268
|
-
| metrics_stop | Stop monitoring, get summary | format ("summary" or "detailed") |
|
|
1269
|
-
|
|
1270
|
-
## Predicates
|
|
1271
|
-
|
|
1272
|
-
Match elements by:
|
|
1273
|
-
- \`text\`: Exact text match
|
|
1274
|
-
- \`text_contains\`: Contains substring (case-insensitive)
|
|
1275
|
-
- \`text_starts_with\`: Starts with prefix
|
|
1276
|
-
- \`text_regex\`: Regex pattern
|
|
1277
|
-
- \`type\`: Element type (button, input, switch, text, image, cell, scrollview)
|
|
1278
|
-
- \`label\`: Accessibility label (exact)
|
|
1279
|
-
- \`label_contains\`: Accessibility label (partial)
|
|
1280
|
-
- \`bounds_hint\`: Screen region (top_half, bottom_half, left_half, right_half, center)
|
|
1281
|
-
- \`near\`: Near another element: {"text": "Label"} or {"text": "Label", "direction": "below"}
|
|
1282
|
-
- \`index\`: Select Nth match (0-based)
|
|
1283
|
-
- \`parent_of\`: Find parent containing child: {"parent_of": {"text": "child"}}
|
|
1284
|
-
- \`enabled\`/\`visible\`/\`selected\`: Boolean state filters
|
|
1285
|
-
|
|
1286
|
-
## Examples
|
|
1287
|
-
|
|
1288
|
-
### Tap Element
|
|
1289
|
-
\`\`\`json
|
|
1290
|
-
{"action": "tap", "predicate": {"text_contains": "Settings"}}
|
|
1291
|
-
\`\`\`
|
|
1292
|
-
|
|
1293
|
-
### Type Text
|
|
1294
|
-
\`\`\`json
|
|
1295
|
-
{"action": "type", "text": "Hello", "predicate": {"type": "input"}}
|
|
1296
|
-
\`\`\`
|
|
1297
|
-
|
|
1298
|
-
Note: \`predicate\` is required if keyboard is not already open. Use \`dismiss_keyboard: true\` to close keyboard after typing.
|
|
1299
|
-
|
|
1300
|
-
### Double Tap
|
|
1301
|
-
\`\`\`json
|
|
1302
|
-
{"action": "double_tap", "predicate": {"text": "Image"}}
|
|
1303
|
-
\`\`\`
|
|
1304
|
-
|
|
1305
|
-
### Long Press
|
|
1306
|
-
\`\`\`json
|
|
1307
|
-
{"action": "long_press", "predicate": {"text": "Message"}, "duration_ms": 1500}
|
|
1308
|
-
\`\`\`
|
|
1309
|
-
|
|
1310
|
-
### Toggle Switch
|
|
1311
|
-
\`\`\`json
|
|
1312
|
-
{"action": "toggle", "predicate": {"type": "switch", "text_contains": "WiFi"}, "state": "on"}
|
|
1313
|
-
\`\`\`
|
|
1314
|
-
|
|
1315
|
-
### Drag Element
|
|
1316
|
-
\`\`\`json
|
|
1317
|
-
{"action": "drag", "from": {"predicate": {"text": "Item"}}, "to_element": {"predicate": {"text": "Trash"}}}
|
|
1318
|
-
{"action": "drag", "from_coords": {"x": 100, "y": 200}, "to_coords": {"x": 300, "y": 400}, "duration_ms": 500}
|
|
1319
|
-
\`\`\`
|
|
1320
|
-
|
|
1321
|
-
### Assert Count
|
|
1322
|
-
\`\`\`json
|
|
1323
|
-
{"action": "assert_count", "predicate": {"type": "cell"}, "count": 5}
|
|
1324
|
-
\`\`\`
|
|
1325
|
-
|
|
1326
|
-
### Scroll Until Found
|
|
1327
|
-
\`\`\`json
|
|
1328
|
-
{"action": "scroll", "direction": "down", "to_element": {"predicate": {"text": "Privacy"}}, "max_scrolls": 10}
|
|
1329
|
-
\`\`\`
|
|
1330
|
-
|
|
1331
|
-
### Conditional (Dismiss Popup)
|
|
1332
|
-
\`\`\`json
|
|
1333
|
-
{
|
|
1334
|
-
"action": "if_exists",
|
|
1335
|
-
"predicate": {"text_contains": "Allow"},
|
|
1336
|
-
"then": [{"action": "tap", "predicate": {"text": "Allow"}}]
|
|
1337
|
-
}
|
|
1338
|
-
\`\`\`
|
|
1339
|
-
|
|
1340
|
-
## Failure Strategies
|
|
1341
|
-
|
|
1342
|
-
- \`abort\`: Stop on failure (default)
|
|
1343
|
-
- \`skip\`: Skip failed step, continue
|
|
1344
|
-
- \`retry\`: Retry with delay
|
|
1345
|
-
|
|
1346
|
-
## OCR (iOS only)
|
|
1347
|
-
|
|
1348
|
-
Use \`include: ["ocr"]\` in observe to get text recognition when UI tree is empty:
|
|
1349
|
-
|
|
1350
|
-
\`\`\`json
|
|
1351
|
-
{"action": "observe", "context": "native", "include": ["ocr"]}
|
|
1352
|
-
\`\`\`
|
|
1353
|
-
|
|
1354
|
-
Returns text with coordinates for tapping (already adjusted for tapping).
|
|
1355
|
-
|
|
1356
|
-
## Performance Metrics
|
|
1357
|
-
|
|
1358
|
-
Collect CPU, memory, FPS, network, and battery metrics during test flows with optional logging capture.
|
|
1359
|
-
|
|
1360
|
-
### Start Metrics Collection
|
|
1361
|
-
\`\`\`json
|
|
1362
|
-
{
|
|
1363
|
-
"action": "metrics_start",
|
|
1364
|
-
"types": ["system_cpu", "system_memory", "fps"],
|
|
1365
|
-
"bundle_id": "com.example.app",
|
|
1366
|
-
"label": "login_flow",
|
|
1367
|
-
"capture_logs": true,
|
|
1368
|
-
"thresholds": {
|
|
1369
|
-
"cpu_high": 80,
|
|
1370
|
-
"fps_low": 45,
|
|
1371
|
-
"memory_growth_mb_min": 50
|
|
1372
|
-
}
|
|
1373
|
-
}
|
|
1374
|
-
\`\`\`
|
|
1375
|
-
|
|
1376
|
-
**Fields:**
|
|
1377
|
-
- \`types\`: Metrics to collect - system_cpu, system_memory, fps, network, battery, process
|
|
1378
|
-
- \`bundle_id\`: Filter to specific app (optional)
|
|
1379
|
-
- \`label\`: Human-readable session label (optional)
|
|
1380
|
-
- \`thresholds\`: Custom thresholds for anomaly detection (optional)
|
|
1381
|
-
- \`capture_logs\`: Capture device logs during session (default: false)
|
|
1382
|
-
|
|
1383
|
-
### Stop and Get Summary
|
|
1384
|
-
\`\`\`json
|
|
1385
|
-
{"action": "metrics_stop", "format": "summary"}
|
|
1386
|
-
\`\`\`
|
|
1387
|
-
|
|
1388
|
-
**Response:**
|
|
1389
|
-
\`\`\`json
|
|
1390
|
-
{
|
|
1391
|
-
"metrics_summary": {
|
|
1392
|
-
"session": {
|
|
1393
|
-
"label": "login_flow",
|
|
1394
|
-
"duration_seconds": 45.2,
|
|
1395
|
-
"sample_count": 45,
|
|
1396
|
-
"session_id": "abc123",
|
|
1397
|
-
"data_file": "/tmp/mobai/metrics/abc123.jsonl",
|
|
1398
|
-
"logs_file": "/tmp/mobai/logs/abc123.jsonl",
|
|
1399
|
-
"logs_available": true
|
|
1400
|
-
},
|
|
1401
|
-
"overall_health": "warning",
|
|
1402
|
-
"health_score": 72,
|
|
1403
|
-
"system_cpu": {"avg": 34.5, "max": 89.2, "p95": 78.1, "status": "ok"},
|
|
1404
|
-
"system_memory": {"avg_percent": 45.2, "growth_mb": 28.5, "trend": "increasing", "status": "warning"},
|
|
1405
|
-
"fps": {"avg": 58.2, "min": 24.0, "jank_percent": 8.5, "status": "warning"},
|
|
1406
|
-
"anomalies": {
|
|
1407
|
-
"cpu_spikes": [
|
|
1408
|
-
{"at_s": 0.5, "peak": 288, "duration_ms": 18147, "source": "system"}
|
|
1409
|
-
],
|
|
1410
|
-
"fps_drops": [
|
|
1411
|
-
{"start_s": 1.2, "end_s": 16.8, "min_fps": 39.5, "avg_fps": 42.3, "samples": 1}
|
|
1412
|
-
],
|
|
1413
|
-
},
|
|
1414
|
-
"recommendations": [
|
|
1415
|
-
"FPS dropped to 24 at +15s - investigate screen transition"
|
|
1416
|
-
]
|
|
1417
|
-
}
|
|
1418
|
-
}
|
|
1419
|
-
\`\`\`
|
|
1420
|
-
|
|
1421
|
-
### Example: Performance Test Flow
|
|
1422
|
-
\`\`\`json
|
|
1423
|
-
{
|
|
1424
|
-
"version": "0.2",
|
|
1425
|
-
"steps": [
|
|
1426
|
-
{"action": "metrics_start", "types": ["system_cpu", "system_memory", "fps"], "label": "app_launch"},
|
|
1427
|
-
{"action": "open_app", "bundle_id": "com.example.app"},
|
|
1428
|
-
{"action": "wait_for", "predicate": {"text": "Welcome"}, "timeout_ms": 10000},
|
|
1429
|
-
{"action": "tap", "predicate": {"text": "Login"}},
|
|
1430
|
-
{"action": "delay", "duration_ms": 5000},
|
|
1431
|
-
{"action": "metrics_stop", "format": "summary"}
|
|
1432
|
-
]
|
|
1433
|
-
}
|
|
1434
|
-
\`\`\`
|
|
1435
|
-
`;
|
|
1436
|
-
const NATIVE_RUNNER_GUIDE = `# Native App Automation Guide
|
|
1437
|
-
|
|
1438
|
-
Use this for automating native mobile apps (Settings, Mail, Instagram, etc.).
|
|
1439
|
-
|
|
1440
|
-
## Script Writing Guidelines
|
|
1441
|
-
|
|
1442
|
-
The DSL's purpose is to **minimize LLM calls** by encoding assumptions into comprehensive scripts. Write scripts that handle common scenarios without needing to re-observe.
|
|
1443
|
-
|
|
1444
|
-
### Example: Handle Cookie Banner
|
|
1445
|
-
\`\`\`json
|
|
1446
|
-
{
|
|
1447
|
-
"action": "if_exists",
|
|
1448
|
-
"predicate": {"text_contains": "Accept Cookies"},
|
|
1449
|
-
"then": [{"action": "tap", "predicate": {"text_contains": "Accept"}}]
|
|
1450
|
-
}
|
|
1451
|
-
\`\`\`
|
|
1452
|
-
|
|
1453
|
-
### Common Knowledge (use without observing)
|
|
1454
|
-
- Safari has an address bar at the top
|
|
1455
|
-
- Settings app has Wi-Fi, Bluetooth, General sections
|
|
1456
|
-
- Alert dialogs have "OK", "Cancel", "Allow", "Don't Allow" buttons
|
|
1457
|
-
- iOS keyboard has "Done", "Return", "Search" keys
|
|
1458
|
-
|
|
1459
|
-
### Script Writing Rules
|
|
1460
|
-
- **Use open_app** - Always start scripts with open_app to ensure correct app
|
|
1461
|
-
- **UI tree provided upfront** - You receive the initial UI tree, use it to plan the script
|
|
1462
|
-
- **Use if_exists for popups** - Handle cookie banners, permission dialogs, notifications
|
|
1463
|
-
- **observe only for assert_screen_changed** - Use observe to establish baseline, then assert_screen_changed to verify navigation
|
|
1464
|
-
|
|
1465
|
-
## IMPORTANT: Browser Native UI
|
|
1466
|
-
|
|
1467
|
-
When automating browsers (Safari, Chrome), use **Native Runner** for the browser's own UI:
|
|
1468
|
-
- Address bar / URL bar
|
|
1469
|
-
- Tab bar and tab management
|
|
1470
|
-
- Navigation buttons (back, forward, refresh)
|
|
1471
|
-
- Bookmarks bar
|
|
1472
|
-
- Browser menus and settings
|
|
1473
|
-
|
|
1474
|
-
These are native OS elements, NOT web content. Only use Web Runner for the actual webpage content inside the browser.
|
|
1475
|
-
|
|
1476
|
-
## Workflow
|
|
1477
|
-
|
|
1478
|
-
1. **Observe UI** - Get the accessibility tree
|
|
1479
|
-
2. **Match Elements** - Use predicates to find elements
|
|
1480
|
-
3. **Execute Actions** - Tap, type, swipe, press_key, etc.
|
|
1481
|
-
4. **Verify Results** - Check UI state changed
|
|
1482
|
-
|
|
1483
|
-
## Type Action
|
|
1484
|
-
|
|
1485
|
-
The \`type\` action requires either:
|
|
1486
|
-
1. Keyboard already open (from previous tap on input), OR
|
|
1487
|
-
2. A predicate to identify and tap the input field
|
|
1488
|
-
|
|
1489
|
-
**dismiss_keyboard** default is \`false\` (keyboard stays open after typing).
|
|
1490
|
-
|
|
1491
|
-
### Pattern 1: Tap then Type
|
|
1492
|
-
\`\`\`json
|
|
1493
|
-
[
|
|
1494
|
-
{"action": "tap", "predicate": {"type": "input"}},
|
|
1495
|
-
{"action": "type", "text": "username"},
|
|
1496
|
-
{"action": "press_key", "key": "tab"}
|
|
1497
|
-
]
|
|
1498
|
-
\`\`\`
|
|
1499
|
-
|
|
1500
|
-
### Pattern 2: Type with Predicate
|
|
1501
|
-
\`\`\`json
|
|
1502
|
-
{"action": "type", "text": "username", "predicate": {"type": "input", "label": "Username"}}
|
|
1503
|
-
\`\`\`
|
|
1504
|
-
|
|
1505
|
-
### Dismissing Keyboard
|
|
1506
|
-
- Use \`press_key: return\` to submit and close the keyboard
|
|
1507
|
-
- If submit is not desired, look for a "Close", "Cancel", "Done" or "Back" button in the UI tree and tap it
|
|
1508
|
-
- On Android, \`press_key: back\` also dismisses the keyboard
|
|
1509
|
-
|
|
1510
|
-
## Common Patterns
|
|
1511
|
-
|
|
1512
|
-
### Open App and Navigate
|
|
1513
|
-
\`\`\`json
|
|
1514
|
-
{
|
|
1515
|
-
"version": "0.2",
|
|
1516
|
-
"steps": [
|
|
1517
|
-
{"action": "open_app", "bundle_id": "com.apple.Preferences"},
|
|
1518
|
-
{"action": "delay", "duration_ms": 1000},
|
|
1519
|
-
{"action": "observe", "context": "native", "include": ["ui_tree"]},
|
|
1520
|
-
{"action": "tap", "predicate": {"text_contains": "General"}}
|
|
1521
|
-
]
|
|
1522
|
-
}
|
|
1523
|
-
\`\`\`
|
|
1524
|
-
|
|
1525
|
-
### Fill Form
|
|
1526
|
-
\`\`\`json
|
|
1527
|
-
{
|
|
1528
|
-
"version": "0.2",
|
|
1529
|
-
"steps": [
|
|
1530
|
-
{"action": "tap", "predicate": {"type": "input"}},
|
|
1531
|
-
{"action": "type", "text": "username"},
|
|
1532
|
-
{"action": "press_key", "key": "tab"},
|
|
1533
|
-
{"action": "type", "text": "password"},
|
|
1534
|
-
{"action": "press_key", "key": "return"}
|
|
1535
|
-
]
|
|
1536
|
-
}
|
|
1537
|
-
\`\`\`
|
|
1538
|
-
|
|
1539
|
-
### Scroll to Find Element
|
|
1540
|
-
\`\`\`json
|
|
1541
|
-
{
|
|
1542
|
-
"version": "0.2",
|
|
1543
|
-
"steps": [
|
|
1544
|
-
{"action": "scroll", "direction": "down", "to_element": {"predicate": {"text": "Privacy"}}, "max_scrolls": 10},
|
|
1545
|
-
{"action": "tap", "predicate": {"text": "Privacy"}}
|
|
1546
|
-
]
|
|
1547
|
-
}
|
|
1548
|
-
\`\`\`
|
|
1549
|
-
|
|
1550
|
-
### Handle Dialogs
|
|
1551
|
-
\`\`\`json
|
|
1552
|
-
{
|
|
1553
|
-
"version": "0.2",
|
|
1554
|
-
"steps": [
|
|
1555
|
-
{
|
|
1556
|
-
"action": "if_exists",
|
|
1557
|
-
"predicate": {"text_contains": "Allow"},
|
|
1558
|
-
"then": [{"action": "tap", "predicate": {"text": "Allow"}}]
|
|
1559
|
-
}
|
|
1560
|
-
]
|
|
1561
|
-
}
|
|
1562
|
-
\`\`\`
|
|
1563
|
-
|
|
1564
|
-
## Quick Reference
|
|
1565
|
-
|
|
1566
|
-
| Action | Description | Key Fields |
|
|
1567
|
-
|--------|-------------|------------|
|
|
1568
|
-
| tap | Tap element | predicate or coords |
|
|
1569
|
-
| double_tap | Double-tap element | predicate or coords |
|
|
1570
|
-
| long_press | Long-press element | predicate or coords, duration_ms |
|
|
1571
|
-
| type | Type text | text, predicate (if keyboard not open), clear_first, dismiss_keyboard |
|
|
1572
|
-
| press_key | Press keyboard key | key (return, tab, delete, etc.) |
|
|
1573
|
-
| toggle | Set switch state | predicate, state ("on"/"off") |
|
|
1574
|
-
| swipe | Swipe gesture | direction, distance, duration_ms, or from_coords/to_coords |
|
|
1575
|
-
| scroll | Scroll container | direction, to_element, max_scrolls |
|
|
1576
|
-
| drag | Drag element | from/to_element (predicates), or from_coords/to_coords |
|
|
1577
|
-
|
|
1578
|
-
## Tips
|
|
1579
|
-
|
|
1580
|
-
- **Always observe first** - Get UI tree before interacting
|
|
1581
|
-
- **Use predicates** - More robust than hardcoded indices
|
|
1582
|
-
- **Add delays after navigation** - Apps need time to render
|
|
1583
|
-
- **Use retry strategy** - Transient failures are common
|
|
1584
|
-
- **Use press_key for form navigation** - Tab between fields, Return to submit
|
|
1585
|
-
- **Use OCR for system dialogs (iOS)** - When UI tree is empty, use \`include: ["ocr"]\`
|
|
1586
|
-
`;
|
|
1587
|
-
const WEB_RUNNER_GUIDE = `# Web Automation Guide
|
|
1588
|
-
|
|
1589
|
-
**Try native-runner first for simple taps/types.** Only use Web Runner when you need DOM manipulation, CSS selectors, or JavaScript execution.
|
|
1590
|
-
|
|
1591
|
-
## iOS Simulator Limitation
|
|
1592
|
-
|
|
1593
|
-
**IMPORTANT: Web context is NOT supported on iOS simulators.** Web automation features (select_web_context, web DOM access, CSS selectors, JavaScript execution) only work on:
|
|
1594
|
-
- **Physical iOS devices** (iPhone, iPad)
|
|
1595
|
-
- **Android emulators and physical devices**
|
|
1596
|
-
|
|
1597
|
-
## When to Use Web Runner
|
|
1598
|
-
|
|
1599
|
-
**USE Web Runner for:**
|
|
1600
|
-
- Native runner returns NO_MATCH for web elements
|
|
1601
|
-
- CSS selector-based element targeting
|
|
1602
|
-
- JavaScript execution in page context
|
|
1603
|
-
- DOM manipulation and inspection
|
|
1604
|
-
- Complex form interactions requiring DOM access
|
|
1605
|
-
|
|
1606
|
-
**DO NOT use Web Runner for:**
|
|
1607
|
-
- Browser address bar / URL bar → use Native Runner
|
|
1608
|
-
- Browser tab bar → use Native Runner
|
|
1609
|
-
- Browser navigation buttons (back, forward, refresh) → use Native Runner
|
|
1610
|
-
- Browser menus and settings → use Native Runner
|
|
1611
|
-
- Any UI outside the webpage or webview content area → use Native Runner
|
|
1612
|
-
|
|
1613
|
-
The browser's own UI (address bar, tabs, navigation) are **native OS elements**, not web content.
|
|
1614
|
-
|
|
1615
|
-
## Platform Support
|
|
1616
|
-
|
|
1617
|
-
| Platform | Browser | Protocol |
|
|
1618
|
-
|----------|---------|----------|
|
|
1619
|
-
| iOS | Safari, WebViews | WebInspector |
|
|
1620
|
-
| Android | Chrome, WebViews | Chrome DevTools Protocol |
|
|
1621
|
-
|
|
1622
|
-
## Workflow
|
|
1623
|
-
|
|
1624
|
-
1. **Select web context** - Connect to browser
|
|
1625
|
-
2. **Navigate** - Go to URL
|
|
1626
|
-
3. **Get DOM** - Inspect page structure
|
|
1627
|
-
4. **Interact** - Click, type, press_key using CSS selectors
|
|
1628
|
-
|
|
1629
|
-
## select_web_context Options
|
|
1630
|
-
|
|
1631
|
-
\`\`\`json
|
|
1632
|
-
{"action": "select_web_context"}
|
|
1633
|
-
{"action": "select_web_context", "url_contains": "example.com"}
|
|
1634
|
-
{"action": "select_web_context", "title_contains": "Login"}
|
|
1635
|
-
\`\`\`
|
|
1636
|
-
|
|
1637
|
-
Use \`url_contains\` or \`title_contains\` to select a specific tab/WebView when multiple are available.
|
|
1638
|
-
|
|
1639
|
-
## press_key (Web Context)
|
|
1640
|
-
|
|
1641
|
-
Press keyboard keys in web context. Supported keys: \`enter\`, \`tab\`, \`delete\`, \`escape\`
|
|
1642
|
-
|
|
1643
|
-
\`\`\`json
|
|
1644
|
-
{"action": "press_key", "context": "web", "key": "enter"}
|
|
1645
|
-
{"action": "press_key", "context": "web", "key": "tab"}
|
|
1646
|
-
\`\`\`
|
|
1647
|
-
|
|
1648
|
-
## Common Patterns
|
|
1649
|
-
|
|
1650
|
-
### Navigate and Fill Form
|
|
1651
|
-
\`\`\`json
|
|
1652
|
-
{
|
|
1653
|
-
"version": "0.2",
|
|
1654
|
-
"steps": [
|
|
1655
|
-
{"action": "select_web_context"},
|
|
1656
|
-
{"action": "navigate", "url": "https://example.com/login"},
|
|
1657
|
-
{"action": "wait_for", "context": "web", "predicate": {"css_selector": "form"}, "timeout_ms": 5000},
|
|
1658
|
-
{"action": "type", "context": "web", "predicate": {"css_selector": "input[name='email']"}, "text": "user@example.com"},
|
|
1659
|
-
{"action": "type", "context": "web", "predicate": {"css_selector": "input[type='password']"}, "text": "password"},
|
|
1660
|
-
{"action": "tap", "context": "web", "predicate": {"css_selector": "button[type='submit']"}}
|
|
1661
|
-
]
|
|
1662
|
-
}
|
|
1663
|
-
\`\`\`
|
|
1664
|
-
|
|
1665
|
-
### Click Element
|
|
1666
|
-
\`\`\`json
|
|
1667
|
-
{"action": "tap", "context": "web", "predicate": {"css_selector": "button.submit"}}
|
|
1668
|
-
\`\`\`
|
|
1669
|
-
|
|
1670
|
-
### Execute JavaScript
|
|
1671
|
-
\`\`\`json
|
|
1672
|
-
{"action": "execute_js", "script": "return document.querySelector('h1').textContent"}
|
|
1673
|
-
\`\`\`
|
|
1674
|
-
|
|
1675
|
-
## CSS Selectors
|
|
1676
|
-
|
|
1677
|
-
| Selector | Description |
|
|
1678
|
-
|----------|-------------|
|
|
1679
|
-
| #id | Element by ID |
|
|
1680
|
-
| .class | Elements by class |
|
|
1681
|
-
| button.submit | Button with class |
|
|
1682
|
-
| input[type='email'] | Input by attribute |
|
|
1683
|
-
| input[name='username'] | Input by name |
|
|
1684
|
-
| a[href*='login'] | Link containing text in href |
|
|
1685
|
-
|
|
1686
|
-
## Tips
|
|
1687
|
-
|
|
1688
|
-
- **Select context first** - Use select_web_context before web operations
|
|
1689
|
-
- **Use specific selectors** - Prefer id > name > class
|
|
1690
|
-
- **Re-fetch DOM after navigation** - Page content changes
|
|
1691
|
-
- **Use JavaScript for complex logic** - When CSS selectors aren't enough
|
|
1692
|
-
`;
|
|
1693
|
-
// Start the server
|
|
532
|
+
// ---------------------------------------------------------------------------
|
|
533
|
+
// Start
|
|
534
|
+
// ---------------------------------------------------------------------------
|
|
1694
535
|
async function main() {
|
|
1695
536
|
const transport = new StdioServerTransport();
|
|
1696
537
|
await server.connect(transport);
|