mobai-mcp 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.js +212 -37
- package/dist/resources.js +179 -3
- package/package.json +1 -1
- package/server.json +2 -2
package/README.md
CHANGED
|
@@ -107,7 +107,7 @@ Tests are `.mob` files on disk inside project directories. You read, write, and
|
|
|
107
107
|
|---|---|
|
|
108
108
|
| `test_get_active` | Get the active test project directory and its `.mob` cases |
|
|
109
109
|
| `test_list_projects` | List all known test project directories with their `.mob` cases |
|
|
110
|
-
| `test_run` | Run a `.mob` test case on a device (`project_dir` + `case_path` + `device_id`) |
|
|
110
|
+
| `test_run` | Run a `.mob` test case on a device (`project_dir` + `case_path` + `device_id`, optional `params` for `${name}` substitution) |
|
|
111
111
|
|
|
112
112
|
## Resources
|
|
113
113
|
|
package/dist/index.js
CHANGED
|
@@ -7,10 +7,11 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
9
9
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
10
|
-
import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
|
|
10
|
+
import { CallToolRequestSchema, ListToolsRequestSchema, ListResourcesRequestSchema, ReadResourceRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
|
|
11
11
|
import * as fs from "fs";
|
|
12
12
|
import * as os from "os";
|
|
13
13
|
import * as path from "path";
|
|
14
|
+
import { RESOURCES, getResourceContent } from "./resources.js";
|
|
14
15
|
const API_BASE_URL = "http://127.0.0.1:8686/api/v1";
|
|
15
16
|
const DEFAULT_TIMEOUT_MS = 300000; // 5 minutes (matches Go httpClient timeout)
|
|
16
17
|
const SCREENSHOT_DIR = path.join(os.tmpdir(), "mobai", "screenshots");
|
|
@@ -22,15 +23,6 @@ function ensureScreenshotDir() {
|
|
|
22
23
|
fs.mkdirSync(SCREENSHOT_DIR, { recursive: true });
|
|
23
24
|
}
|
|
24
25
|
}
|
|
25
|
-
function saveBase64ToTemp(base64Data, prefix) {
|
|
26
|
-
if (!base64Data || base64Data.length <= 200)
|
|
27
|
-
return null;
|
|
28
|
-
ensureScreenshotDir();
|
|
29
|
-
const filename = `${prefix}_${Date.now()}.png`;
|
|
30
|
-
const filePath = path.join(SCREENSHOT_DIR, filename);
|
|
31
|
-
fs.writeFileSync(filePath, Buffer.from(base64Data, "base64"));
|
|
32
|
-
return filePath;
|
|
33
|
-
}
|
|
34
26
|
function screenshotToFile(body) {
|
|
35
27
|
if (body?.path) {
|
|
36
28
|
return `Screenshot saved to ${body.path}`;
|
|
@@ -47,28 +39,6 @@ function screenshotToFile(body) {
|
|
|
47
39
|
}
|
|
48
40
|
return JSON.stringify(body, null, 2);
|
|
49
41
|
}
|
|
50
|
-
function extractDSLScreenshots(body) {
|
|
51
|
-
if (!body?.step_results)
|
|
52
|
-
return body;
|
|
53
|
-
for (const step of body.step_results) {
|
|
54
|
-
const native = step.result?.observations?.native;
|
|
55
|
-
if (native?.screenshot && typeof native.screenshot === "string" && native.screenshot.length > 200) {
|
|
56
|
-
const filePath = saveBase64ToTemp(native.screenshot, "observe");
|
|
57
|
-
if (filePath) {
|
|
58
|
-
native.screenshot = filePath;
|
|
59
|
-
native.screenshot_saved = true;
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
if (step.debug?.screenshot && typeof step.debug.screenshot === "string" && step.debug.screenshot.length > 200) {
|
|
63
|
-
const filePath = saveBase64ToTemp(step.debug.screenshot, "debug");
|
|
64
|
-
if (filePath) {
|
|
65
|
-
step.debug.screenshot = filePath;
|
|
66
|
-
step.debug.screenshot_saved = true;
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
return body;
|
|
71
|
-
}
|
|
72
42
|
// ---------------------------------------------------------------------------
|
|
73
43
|
// HTTP helpers
|
|
74
44
|
// ---------------------------------------------------------------------------
|
|
@@ -127,6 +97,7 @@ const server = new Server({ name: "mobai", version: "1.0.0" }, {
|
|
|
127
97
|
instructions: `MobAI controls Android and iOS devices. Before starting any device task, read the relevant MCP resources:
|
|
128
98
|
- mobai://reference/device-automation — how to control devices (read before ANY device interaction)
|
|
129
99
|
- mobai://reference/testing — .mob script syntax (read ONLY when user asks to create or fix test scripts)
|
|
100
|
+
- mobai://reference/debugging — how to attach lldb, set breakpoints, inspect state (read before ANY debug_* tool)
|
|
130
101
|
Check available skills in current work directory and load any relevant to the user's request.`,
|
|
131
102
|
});
|
|
132
103
|
// ---------------------------------------------------------------------------
|
|
@@ -272,17 +243,106 @@ Input: JSON string with "version": "0.2" and "steps" array. Example:
|
|
|
272
243
|
},
|
|
273
244
|
{
|
|
274
245
|
name: "test_run",
|
|
275
|
-
description: "Run a .mob test case on a device. The case_path is relative to the project directory.",
|
|
246
|
+
description: "Run a .mob test case on a device. The case_path is relative to the project directory. Pass params to supply values for ${name} substitution in the script.",
|
|
276
247
|
inputSchema: {
|
|
277
248
|
type: "object",
|
|
278
249
|
properties: {
|
|
279
250
|
project_dir: { type: "string", description: "Absolute path to the project directory" },
|
|
280
251
|
case_path: { type: "string", description: "Relative path to the .mob file within the project, e.g. auth/login.mob" },
|
|
281
252
|
device_id: { type: "string", description: "Device ID to run the test on" },
|
|
253
|
+
params: { type: "object", additionalProperties: { type: "string" }, description: "Optional key-value parameters for ${name} substitution in the script" },
|
|
282
254
|
},
|
|
283
255
|
required: ["project_dir", "case_path", "device_id"],
|
|
284
256
|
},
|
|
285
257
|
},
|
|
258
|
+
// Live app debugging via lldb-dap. iOS only. Read mobai://reference/debugging
|
|
259
|
+
// before using any of these.
|
|
260
|
+
{
|
|
261
|
+
name: "debug_attach",
|
|
262
|
+
description: "Start a debug session for an iOS app. Provide either bundle_id (launches and attaches) or pid (attaches to a running process). Optional breakpoints[] are armed before the target resumes. Read mobai://reference/debugging first.",
|
|
263
|
+
inputSchema: {
|
|
264
|
+
type: "object",
|
|
265
|
+
properties: {
|
|
266
|
+
device_id: { type: "string", description: "Device ID" },
|
|
267
|
+
bundle_id: { type: "string", description: "App bundle ID to launch and attach. Either this or pid is required." },
|
|
268
|
+
pid: { type: "number", description: "Attach to an already-running PID. Either this or bundle_id is required." },
|
|
269
|
+
breakpoints: {
|
|
270
|
+
type: "array",
|
|
271
|
+
items: { type: "string" },
|
|
272
|
+
description: `Initial breakpoint specs. "File.swift:42" (preferred), "Module.Type.method" (no parameter signature), "-[Class method:]", or runtime symbol.`,
|
|
273
|
+
},
|
|
274
|
+
stop_on_entry: { type: "boolean", description: "Simulator only — pause at first instruction." },
|
|
275
|
+
},
|
|
276
|
+
required: ["device_id"],
|
|
277
|
+
},
|
|
278
|
+
},
|
|
279
|
+
{
|
|
280
|
+
name: "debug_state",
|
|
281
|
+
description: "Query the current debug session. Returns {state, breakpoints} by default. Set include_stack=true to also fetch the stack of the stopped thread; include_vars=true to also fetch frame[0] locals; include_threads=true to enumerate all threads.",
|
|
282
|
+
inputSchema: {
|
|
283
|
+
type: "object",
|
|
284
|
+
properties: {
|
|
285
|
+
device_id: { type: "string", description: "Device ID" },
|
|
286
|
+
include_stack: { type: "boolean", description: "Include stack of stopped thread." },
|
|
287
|
+
include_vars: { type: "boolean", description: "Include frame[0] locals." },
|
|
288
|
+
include_threads: { type: "boolean", description: "Include all threads." },
|
|
289
|
+
},
|
|
290
|
+
required: ["device_id"],
|
|
291
|
+
},
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
name: "debug_breakpoint",
|
|
295
|
+
description: "Add or remove a breakpoint in the active debug session. For action=add provide spec; for action=remove provide id.",
|
|
296
|
+
inputSchema: {
|
|
297
|
+
type: "object",
|
|
298
|
+
properties: {
|
|
299
|
+
device_id: { type: "string", description: "Device ID" },
|
|
300
|
+
action: { type: "string", enum: ["add", "remove"], description: `"add" or "remove"` },
|
|
301
|
+
spec: { type: "string", description: `Breakpoint spec for action=add. "File.swift:42", "Module.Type.method", "-[Class method:]", or runtime symbol.` },
|
|
302
|
+
id: { type: "number", description: "Breakpoint id for action=remove." },
|
|
303
|
+
},
|
|
304
|
+
required: ["device_id", "action"],
|
|
305
|
+
},
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
name: "debug_eval",
|
|
309
|
+
description: `Evaluate a Swift/ObjC expression at the current pause. Session must be paused. Examples: "p defaultPrivate", "po self.viewModel.user.email", "frame variable".`,
|
|
310
|
+
inputSchema: {
|
|
311
|
+
type: "object",
|
|
312
|
+
properties: {
|
|
313
|
+
device_id: { type: "string", description: "Device ID" },
|
|
314
|
+
expression: { type: "string", description: "Expression to evaluate" },
|
|
315
|
+
frame_id: { type: "number", description: "Optional frame id to evaluate in" },
|
|
316
|
+
},
|
|
317
|
+
required: ["device_id", "expression"],
|
|
318
|
+
},
|
|
319
|
+
},
|
|
320
|
+
{
|
|
321
|
+
name: "debug_step",
|
|
322
|
+
description: `Advance the target.\n "in" — step into next call (blocks ~ms, returns {state, breakpoints, stack, frame0_locals})\n "over" — step over next call (same shape)\n "out" — run until current frame returns (same shape)\n "continue" — resume until next breakpoint (fire-and-forget; returns just {state, breakpoints} — poll debug_state for next stop)`,
|
|
323
|
+
inputSchema: {
|
|
324
|
+
type: "object",
|
|
325
|
+
properties: {
|
|
326
|
+
device_id: { type: "string", description: "Device ID" },
|
|
327
|
+
direction: { type: "string", enum: ["in", "over", "out", "continue"], description: `"in" | "over" | "out" | "continue"` },
|
|
328
|
+
include_stack: { type: "boolean", description: `Include the new stack. Default true. Ignored for direction="continue".` },
|
|
329
|
+
include_vars: { type: "boolean", description: `Include the new frame[0] locals. Default true. Ignored for direction="continue".` },
|
|
330
|
+
},
|
|
331
|
+
required: ["device_id", "direction"],
|
|
332
|
+
},
|
|
333
|
+
},
|
|
334
|
+
{
|
|
335
|
+
name: "debug_detach",
|
|
336
|
+
description: "End the debug session. Pass kill=true to terminate the debuggee; otherwise it keeps running.",
|
|
337
|
+
inputSchema: {
|
|
338
|
+
type: "object",
|
|
339
|
+
properties: {
|
|
340
|
+
device_id: { type: "string", description: "Device ID" },
|
|
341
|
+
kill: { type: "boolean", description: "Terminate debuggee on detach." },
|
|
342
|
+
},
|
|
343
|
+
required: ["device_id"],
|
|
344
|
+
},
|
|
345
|
+
},
|
|
286
346
|
];
|
|
287
347
|
// ---------------------------------------------------------------------------
|
|
288
348
|
// List tools
|
|
@@ -291,6 +351,22 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
|
|
|
291
351
|
return { tools: TOOLS };
|
|
292
352
|
});
|
|
293
353
|
// ---------------------------------------------------------------------------
|
|
354
|
+
// Resources (reference docs)
|
|
355
|
+
// ---------------------------------------------------------------------------
|
|
356
|
+
server.setRequestHandler(ListResourcesRequestSchema, async () => {
|
|
357
|
+
return { resources: RESOURCES };
|
|
358
|
+
});
|
|
359
|
+
server.setRequestHandler(ReadResourceRequestSchema, async (request) => {
|
|
360
|
+
const { uri } = request.params;
|
|
361
|
+
const text = getResourceContent(uri);
|
|
362
|
+
if (text == null) {
|
|
363
|
+
throw new Error(`Unknown resource: ${uri}`);
|
|
364
|
+
}
|
|
365
|
+
return {
|
|
366
|
+
contents: [{ uri, mimeType: "text/plain", text }],
|
|
367
|
+
};
|
|
368
|
+
});
|
|
369
|
+
// ---------------------------------------------------------------------------
|
|
294
370
|
// Tool call handler
|
|
295
371
|
// ---------------------------------------------------------------------------
|
|
296
372
|
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
@@ -347,19 +423,118 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
347
423
|
throw new Error("invalid DSL JSON: " + commandsStr);
|
|
348
424
|
}
|
|
349
425
|
const body = await doPost(`/devices/${args?.device_id}/dsl/execute`, script);
|
|
350
|
-
return textResult(
|
|
426
|
+
return textResult(body);
|
|
351
427
|
}
|
|
352
428
|
// Test management
|
|
353
429
|
case "test_get_active":
|
|
354
430
|
return textResult(await doGet("/tests/active"));
|
|
355
431
|
case "test_list_projects":
|
|
356
432
|
return textResult(await doGet("/tests/projects"));
|
|
357
|
-
case "test_run":
|
|
358
|
-
|
|
433
|
+
case "test_run": {
|
|
434
|
+
const body = {
|
|
359
435
|
project_dir: args?.project_dir,
|
|
360
436
|
case_path: args?.case_path,
|
|
361
437
|
device_id: args?.device_id,
|
|
362
|
-
}
|
|
438
|
+
};
|
|
439
|
+
const rawParams = args?.params;
|
|
440
|
+
if (rawParams && typeof rawParams === "object") {
|
|
441
|
+
const params = {};
|
|
442
|
+
for (const [k, v] of Object.entries(rawParams)) {
|
|
443
|
+
params[k] = String(v);
|
|
444
|
+
}
|
|
445
|
+
body.params = params;
|
|
446
|
+
}
|
|
447
|
+
return textResult(await doPost("/tests/cases/run", body));
|
|
448
|
+
}
|
|
449
|
+
// Debug session
|
|
450
|
+
case "debug_attach": {
|
|
451
|
+
const dev = args?.device_id;
|
|
452
|
+
const bundleID = args?.bundle_id;
|
|
453
|
+
const pid = args?.pid;
|
|
454
|
+
if (!bundleID && !pid)
|
|
455
|
+
throw new Error("either bundle_id or pid is required");
|
|
456
|
+
const body = {};
|
|
457
|
+
if (Array.isArray(args?.breakpoints))
|
|
458
|
+
body.breakpoints = args.breakpoints;
|
|
459
|
+
if (args?.stop_on_entry)
|
|
460
|
+
body.stopOnEntry = true;
|
|
461
|
+
let path;
|
|
462
|
+
if (pid && pid > 0) {
|
|
463
|
+
body.pid = pid;
|
|
464
|
+
path = `/devices/${dev}/debug-session/attach-running`;
|
|
465
|
+
}
|
|
466
|
+
else {
|
|
467
|
+
body.bundleId = bundleID;
|
|
468
|
+
path = `/devices/${dev}/debug-session/attach`;
|
|
469
|
+
}
|
|
470
|
+
return textResult(await doPost(path, body));
|
|
471
|
+
}
|
|
472
|
+
case "debug_detach": {
|
|
473
|
+
const dev = args?.device_id;
|
|
474
|
+
const body = args?.kill ? { kill: true } : {};
|
|
475
|
+
return textResult(await doRequest("DELETE", `/devices/${dev}/debug-session`, body));
|
|
476
|
+
}
|
|
477
|
+
case "debug_state": {
|
|
478
|
+
const dev = args?.device_id;
|
|
479
|
+
const includeStack = args?.include_stack === true;
|
|
480
|
+
const includeVars = args?.include_vars === true;
|
|
481
|
+
const includeThreads = args?.include_threads === true;
|
|
482
|
+
const base = `/devices/${dev}/debug-session`;
|
|
483
|
+
const snap = await doGet(base);
|
|
484
|
+
if (snap?.state === "paused") {
|
|
485
|
+
if (includeThreads) {
|
|
486
|
+
const t = await doGet(`${base}/threads`);
|
|
487
|
+
snap.threads = t?.threads;
|
|
488
|
+
}
|
|
489
|
+
if (includeStack || includeVars) {
|
|
490
|
+
const stack = await doGet(`${base}/stack`);
|
|
491
|
+
if (includeStack)
|
|
492
|
+
snap.stack = stack?.frames;
|
|
493
|
+
if (includeVars && Array.isArray(stack?.frames) && stack.frames.length > 0) {
|
|
494
|
+
const frameID = stack.frames[0].id;
|
|
495
|
+
const vars = await doGet(`${base}/frames/${frameID}/variables`);
|
|
496
|
+
snap.frame0_locals = vars?.scopes;
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
return textResult(snap);
|
|
501
|
+
}
|
|
502
|
+
case "debug_breakpoint": {
|
|
503
|
+
const dev = args?.device_id;
|
|
504
|
+
const action = args?.action;
|
|
505
|
+
if (action === "add") {
|
|
506
|
+
const spec = args?.spec;
|
|
507
|
+
if (!spec)
|
|
508
|
+
throw new Error("spec is required for action=add");
|
|
509
|
+
return textResult(await doPost(`/devices/${dev}/debug-session/breakpoints`, { spec }));
|
|
510
|
+
}
|
|
511
|
+
else if (action === "remove") {
|
|
512
|
+
const id = args?.id;
|
|
513
|
+
if (!id || id <= 0)
|
|
514
|
+
throw new Error("id is required for action=remove");
|
|
515
|
+
return textResult(await doDelete(`/devices/${dev}/debug-session/breakpoints/${id}`));
|
|
516
|
+
}
|
|
517
|
+
throw new Error(`action must be "add" or "remove"`);
|
|
518
|
+
}
|
|
519
|
+
case "debug_eval": {
|
|
520
|
+
const dev = args?.device_id;
|
|
521
|
+
const body = { expression: args?.expression };
|
|
522
|
+
if (args?.frame_id)
|
|
523
|
+
body.frameId = args.frame_id;
|
|
524
|
+
return textResult(await doPost(`/devices/${dev}/debug-session/eval`, body));
|
|
525
|
+
}
|
|
526
|
+
case "debug_step": {
|
|
527
|
+
const dev = args?.device_id;
|
|
528
|
+
const direction = args?.direction;
|
|
529
|
+
if (!direction)
|
|
530
|
+
throw new Error(`direction is required ("in" | "over" | "out")`);
|
|
531
|
+
const body = { direction };
|
|
532
|
+
if (typeof args?.include_stack === "boolean")
|
|
533
|
+
body.includeStack = args.include_stack;
|
|
534
|
+
if (typeof args?.include_vars === "boolean")
|
|
535
|
+
body.includeVars = args.include_vars;
|
|
536
|
+
return textResult(await doPost(`/devices/${dev}/debug-session/step`, body));
|
|
537
|
+
}
|
|
363
538
|
default:
|
|
364
539
|
return { content: [{ type: "text", text: `Unknown tool: ${name}` }], isError: true };
|
|
365
540
|
}
|
package/dist/resources.js
CHANGED
|
@@ -17,6 +17,12 @@ export const RESOURCES = [
|
|
|
17
17
|
description: "How to preview a MobAI device's control UI inside Claude Code's preview panel",
|
|
18
18
|
mimeType: "text/plain",
|
|
19
19
|
},
|
|
20
|
+
{
|
|
21
|
+
uri: "mobai://reference/debugging",
|
|
22
|
+
name: "App Debugging Reference",
|
|
23
|
+
description: "How to attach lldb, set breakpoints, inspect stack/variables, evaluate Swift/ObjC expressions — read before any debug_* tool",
|
|
24
|
+
mimeType: "text/plain",
|
|
25
|
+
},
|
|
20
26
|
];
|
|
21
27
|
export function getResourceContent(uri) {
|
|
22
28
|
switch (uri) {
|
|
@@ -26,6 +32,8 @@ export function getResourceContent(uri) {
|
|
|
26
32
|
return TESTING_REF;
|
|
27
33
|
case "mobai://claude-code-preview":
|
|
28
34
|
return CLAUDE_CODE_PREVIEW;
|
|
35
|
+
case "mobai://reference/debugging":
|
|
36
|
+
return DEBUGGING_REF;
|
|
29
37
|
default:
|
|
30
38
|
return null;
|
|
31
39
|
}
|
|
@@ -69,6 +77,7 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
|
|
|
69
77
|
<script-format>
|
|
70
78
|
{"version": "0.2", "steps": [...actions...], "on_fail": {"strategy": "retry", "max_retries": 2}}
|
|
71
79
|
Every script must include "version": "0.2" and a "steps" array.
|
|
80
|
+
Optional "params": {"name": "default_value"} declares parameters. Callers supply values via the API; \${name} is substituted in step string fields at runtime.
|
|
72
81
|
</script-format>
|
|
73
82
|
|
|
74
83
|
<important>
|
|
@@ -96,6 +105,14 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
|
|
|
96
105
|
|
|
97
106
|
<workflow>Observe screen → plan → act via execute_dsl → verify (end script with wait_for stable + observe) → repeat until done.</workflow>
|
|
98
107
|
|
|
108
|
+
<siri-shortcuts>
|
|
109
|
+
iOS only. Before navigating through multiple screens to reach a feature, check if Siri can take you there directly. Many apps register SiriKit intents and App Shortcuts — a single siri action can replace 5-10 tap/scroll/wait steps.
|
|
110
|
+
Use observe with include: installed_apps to check what an app exposes. Common shortcuts: play media, send messages, open specific screens, make payments, start workouts, get directions.
|
|
111
|
+
Examples: "Open my cart in Amazon", "Play my playlist on Spotify", "Show my reservations in Booking", "Search YouTube for cats".
|
|
112
|
+
If Siri asks a follow-up question, dismiss and re-invoke with a more specific prompt that includes the missing detail.
|
|
113
|
+
Always prefer siri over manual UI navigation when the app supports it — it is faster, more reliable, and survives UI redesigns.
|
|
114
|
+
</siri-shortcuts>
|
|
115
|
+
|
|
99
116
|
<per-app-skills>
|
|
100
117
|
Before working with a known app, check ~/.claude/skills/ for a skill matching its bundle id or name (e.g. com-instagram-android, uber) and load it — it may already encode selectors, flows, and quirks learned on a prior run.
|
|
101
118
|
When you discover app-specific gotchas that would cost future sessions time — unstable selectors that only work with a specific predicate, hidden taps, flows that need an extra wait_for, React Native / Flutter screens that need OCR, dialogs that hijack input — create or update a skill at ~/.claude/skills/<app-slug>/SKILL.md capturing the finding. Keep each skill short: the specific quirk, the selector/flow that works, and one sentence on why the obvious approach fails. Do not write generic mobile-automation advice there — that belongs in this reference.
|
|
@@ -219,9 +236,11 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
|
|
|
219
236
|
<field name="from_coords" type="Coordinates" required="one-of"/>
|
|
220
237
|
<field name="to_element" type="TargetElement" required="one-of"/>
|
|
221
238
|
<field name="to_coords" type="Coordinates" required="one-of"/>
|
|
222
|
-
<field name="duration_ms" type="int"
|
|
223
|
-
<field name="press_duration_ms" type="int">
|
|
239
|
+
<field name="duration_ms" type="int">Drag motion duration (default 500)</field>
|
|
240
|
+
<field name="press_duration_ms" type="int">Hold before moving (for moving app icons, picking up list items)</field>
|
|
241
|
+
<field name="hold_duration_ms" type="int">Hold at destination before release (useful for iOS drop zones that need a dwell)</field>
|
|
224
242
|
<example>{"action": "drag", "from": {"predicate": {"text": "Item"}}, "to_element": {"predicate": {"text": "Trash"}}}</example>
|
|
243
|
+
<example>{"action": "drag", "from": {"predicate": {"text": "App"}}, "to_element": {"predicate": {"text": "Folder"}}, "press_duration_ms": 500, "hold_duration_ms": 200}</example>
|
|
225
244
|
</action>
|
|
226
245
|
|
|
227
246
|
<action name="press_key">
|
|
@@ -299,6 +318,16 @@ const DEVICE_AUTOMATION_REF = `<device-automation-reference>
|
|
|
299
318
|
<action name="reset_location">
|
|
300
319
|
<example>{"action": "reset_location"}</example>
|
|
301
320
|
</action>
|
|
321
|
+
|
|
322
|
+
<action name="siri">
|
|
323
|
+
iOS only. Sends a voice command to Siri via XCUISiriService. Auto-approves consent dialogs, captures Siri's response text, then dismisses the Siri UI.
|
|
324
|
+
Use for triggering SiriKit intents and App Shortcuts registered by apps (media playback, messaging, banking shortcuts, etc.).
|
|
325
|
+
The captured response is stored in "siri_response" and returned in the step result. If Siri asks a follow-up question, reformulate the prompt with more detail and call siri again.
|
|
326
|
+
<field name="prompt" required="yes">Voice command text</field>
|
|
327
|
+
<example>{"action": "siri", "prompt": "Search YouTube for cat videos"}</example>
|
|
328
|
+
<example>{"action": "siri", "prompt": "Send an email to john@example.com via Gmail"}</example>
|
|
329
|
+
<note>Check the app's siri field in the installed apps list (observe with include: installed_apps) to see which intents and activities it supports before calling siri.</note>
|
|
330
|
+
</action>
|
|
302
331
|
</native-actions>
|
|
303
332
|
|
|
304
333
|
<web-actions>
|
|
@@ -496,7 +525,7 @@ const TESTING_REF = `<testing-reference>
|
|
|
496
525
|
toggle type:switch near "Wi-Fi" on — modifier-only
|
|
497
526
|
drag "Item" to "Trash" — drag element
|
|
498
527
|
drag 100,200 to 300,400 duration:500 — coordinate drag
|
|
499
|
-
drag "App" to "Folder" press_duration:500 — press-
|
|
528
|
+
drag "App" to "Folder" press_duration:500 hold_duration:200 — press-hold-move-hold-release
|
|
500
529
|
wait_for "Element" timeout:5000 — wait for element
|
|
501
530
|
wait_for type:button bounds:bottom_half timeout:3000 — modifier-only
|
|
502
531
|
delay 1000 — wait N ms
|
|
@@ -510,6 +539,7 @@ const TESTING_REF = `<testing-reference>
|
|
|
510
539
|
paste_text "Field" — paste clipboard into element
|
|
511
540
|
set_location 40.7128,-74.0060 — simulate GPS location (lat,lon)
|
|
512
541
|
reset_location — stop location simulation
|
|
542
|
+
siri "Search YouTube for cats" — invoke Siri with voice command (iOS only)
|
|
513
543
|
observe — observe screen
|
|
514
544
|
screenshot "path.png" — take screenshot
|
|
515
545
|
</actions>
|
|
@@ -533,8 +563,30 @@ const TESTING_REF = `<testing-reference>
|
|
|
533
563
|
# Device: iPhone 15 — device filter
|
|
534
564
|
# Timeout: 30000 — global timeout (ms)
|
|
535
565
|
# On-Fail: abort — abort or continue
|
|
566
|
+
# Param: username — declare a parameter (no default)
|
|
567
|
+
# Param: timeout = 5000 — declare with default value
|
|
536
568
|
</metadata>
|
|
537
569
|
|
|
570
|
+
<variables>
|
|
571
|
+
\${name} substitution: use \${param_name} anywhere in a step line to reference a parameter or extracted value.
|
|
572
|
+
Parameters declared via # Param: are available as \${name}. Extracted values (see extract below) are also available.
|
|
573
|
+
Example:
|
|
574
|
+
# Param: email
|
|
575
|
+
# Param: password = secret123
|
|
576
|
+
type "Email" → "\${email}"
|
|
577
|
+
type "Password" → "\${password}"
|
|
578
|
+
</variables>
|
|
579
|
+
|
|
580
|
+
<extract>
|
|
581
|
+
extract key from "Element" — extract text from matched element into \${key}
|
|
582
|
+
extract key from #AccessibilityID — extract text by accessibility ID
|
|
583
|
+
extract key from ~"partial" regex:"(\\d+)" — extract with regex capture group
|
|
584
|
+
extract key screenshot — save screenshot to disk, store path in \${key}
|
|
585
|
+
extract key = "literal value" — store a literal string in \${key}
|
|
586
|
+
Extracted values are available as \${key} in subsequent steps and returned in the API response as "extracted" map.
|
|
587
|
+
The optional regex: modifier applies a regex to the matched text; if it has a capture group, group 1 is stored.
|
|
588
|
+
</extract>
|
|
589
|
+
|
|
538
590
|
<platform-blocks>
|
|
539
591
|
# ios / # android — open platform block
|
|
540
592
|
# end — close block
|
|
@@ -549,7 +601,131 @@ const TESTING_REF = `<testing-reference>
|
|
|
549
601
|
tap "Other"
|
|
550
602
|
}
|
|
551
603
|
</conditionals>
|
|
604
|
+
|
|
605
|
+
<run-includes>
|
|
606
|
+
run "./path/to/other.mob" — inline another .mob file at compile time
|
|
607
|
+
run "./auth/login.mob" email="x@y" password="hunter2" — pass args; values overlay the target file's # Param: defaults
|
|
608
|
+
run "/abs/path/to/file.mob" — absolute path is allowed
|
|
609
|
+
run "~/shared/login.mob" — ~ expands to the user home directory
|
|
610
|
+
Path is relative to the calling file's directory unless absolute. Args use key=value (no colon, no quotes around the key). Values may contain \${name} references that resolve from the caller's scope at execute time. The target file's extracts flow back into the caller's scope (flat namespace).
|
|
611
|
+
</run-includes>
|
|
552
612
|
</mob-script-syntax>
|
|
553
613
|
|
|
614
|
+
<apis>
|
|
615
|
+
Mobile apps can be turned into callable APIs by saving parameterized .mob scripts to the APIs directory.
|
|
616
|
+
|
|
617
|
+
<directory>{MOBAI_DATA_DIR}/apis/ — global directory for API scripts. Each .mob file is a named API. Subdirectories are supported and become slash-separated names (e.g. apis/youtube/search.mob is callable as "youtube/search"). Resolves to ~/Library/Application Support/mobai/data/apis on macOS, %AppData%/mobai/data/apis on Windows, ~/.config/mobai/data/apis on Linux.</directory>
|
|
618
|
+
|
|
619
|
+
<workflow-create-api>
|
|
620
|
+
When the user asks to create an API from a mobile app flow:
|
|
621
|
+
1. Observe the app and understand the flow
|
|
622
|
+
2. Write a .mob script with # Param: declarations for inputs and extract actions for outputs
|
|
623
|
+
3. Save it to {MOBAI_DATA_DIR}/apis/{name}.mob — flat (gmail-send.mob) or nested (gmail/send.mob)
|
|
624
|
+
4. Test it with test_run using project_dir: {MOBAI_DATA_DIR}/apis/ and case_path: {name}.mob
|
|
625
|
+
5. List available APIs: GET /api/v1/apis
|
|
626
|
+
Call an API: POST /api/v1/apis/run/{name} with {"device_id": "...", "params": {...}}
|
|
627
|
+
The {name} segment is the path inside apis/ minus the .mob extension.
|
|
628
|
+
API runs do not persist results to .mobai/runs/ — only the extracted values come back in the response.
|
|
629
|
+
</workflow-create-api>
|
|
630
|
+
|
|
631
|
+
<example-api>
|
|
632
|
+
# Search YouTube
|
|
633
|
+
# Param: query
|
|
634
|
+
siri "Search YouTube for \${query}"
|
|
635
|
+
wait_for ~"\${query}" timeout:5000
|
|
636
|
+
extract result from ~"\${query}"
|
|
637
|
+
|
|
638
|
+
POST /api/v1/apis/run/youtube-search {"device_id":"X","params":{"query":"cats"}}
|
|
639
|
+
→ {"result": "cats"}
|
|
640
|
+
</example-api>
|
|
641
|
+
</apis>
|
|
642
|
+
|
|
554
643
|
</testing-reference>
|
|
555
644
|
`;
|
|
645
|
+
const DEBUGGING_REF = `<debugging-reference>
|
|
646
|
+
|
|
647
|
+
<scope>
|
|
648
|
+
Live debugging of an iOS app running on a connected device or booted simulator. Attach lldb, set breakpoints, inspect stack and variables, evaluate Swift/ObjC expressions, continue. Six MCP tools cover the full workflow.
|
|
649
|
+
|
|
650
|
+
Requires: a debug-signed build of the app (debug provisioning profile with get-task-allow). App Store / TestFlight builds cannot be attached. iOS 17+ for physical devices. macOS host with Xcode installed.
|
|
651
|
+
</scope>
|
|
652
|
+
|
|
653
|
+
<workflow>
|
|
654
|
+
Bps fire asynchronously when the user (or your execute_dsl) drives the UI. The agent observes via debug_state, not by waiting on debug_continue.
|
|
655
|
+
|
|
656
|
+
1. debug_attach {device_id, bundle_id, breakpoints: ["File.swift:42"]}
|
|
657
|
+
2. (trigger the action — usually via execute_dsl)
|
|
658
|
+
3. debug_state {device_id, include_stack: true, include_vars: true} // poll until state == "paused"
|
|
659
|
+
4. debug_eval {device_id, expression: "po self.viewModel.user"}
|
|
660
|
+
5. debug_step {device_id, direction: "continue"} // resume; fire-and-forget
|
|
661
|
+
6. (loop 2-5 as needed)
|
|
662
|
+
7. debug_detach {device_id}
|
|
663
|
+
|
|
664
|
+
direction: "continue" is fire-and-forget. For deterministic line-stepping use "in" / "over" / "out" — those block (~ms) and return fresh stack + locals.
|
|
665
|
+
</workflow>
|
|
666
|
+
|
|
667
|
+
<tools>
|
|
668
|
+
debug_attach — start a debug session.
|
|
669
|
+
device_id (required), bundle_id OR pid (one required), breakpoints (optional [string]),
|
|
670
|
+
stop_on_entry (optional bool, simulator only).
|
|
671
|
+
|
|
672
|
+
debug_state — query the current session state.
|
|
673
|
+
device_id (required), include_stack (bool, default false), include_vars (bool, default false), include_threads (bool, default false).
|
|
674
|
+
Default returns just {state, breakpoints}. Stack, frame[0] locals, and the thread list are opt-in (each costs a round-trip; ~few seconds on physical hardware).
|
|
675
|
+
|
|
676
|
+
debug_breakpoint — add or remove a breakpoint.
|
|
677
|
+
device_id, action: "add" | "remove", spec (for add), id (for remove).
|
|
678
|
+
|
|
679
|
+
debug_eval — evaluate a Swift/ObjC expression at the current pause.
|
|
680
|
+
device_id, expression, frame_id (optional). Session must be paused.
|
|
681
|
+
|
|
682
|
+
debug_step — advance the target.
|
|
683
|
+
device_id, direction one of:
|
|
684
|
+
"in" / "over" / "out" — block ~ms until next stop; return {state, breakpoints, stack, frame0_locals}.
|
|
685
|
+
"continue" — fire-and-forget; return {state, breakpoints}; poll debug_state for next stop.
|
|
686
|
+
include_stack / include_vars (default true; ignored for "continue").
|
|
687
|
+
|
|
688
|
+
debug_detach — end the session. kill (default false) terminates the debuggee.
|
|
689
|
+
</tools>
|
|
690
|
+
|
|
691
|
+
<breakpoint-specs>
|
|
692
|
+
Three accepted forms. Prefer file:line for application code.
|
|
693
|
+
|
|
694
|
+
"File.swift:42" file:line (basename or absolute path)
|
|
695
|
+
"Module.Type.method" Swift demangled prefix (NO parameter signature, NO return type)
|
|
696
|
+
"-[ClassName method:]" ObjC method
|
|
697
|
+
"swift_willThrow" bare runtime symbol
|
|
698
|
+
|
|
699
|
+
Caveats:
|
|
700
|
+
- Release/optimized builds without DWARF return verified=false.
|
|
701
|
+
- swift_willThrow / objc_exception_throw fire on EVERY internal Swift/ObjC throw — Apple frameworks throw constantly under the hood. Use only when actually hunting an uncaught error.
|
|
702
|
+
</breakpoint-specs>
|
|
703
|
+
|
|
704
|
+
<eval-expressions>
|
|
705
|
+
debug_eval runs lldb expression --. Accepts ObjC++ syntax by default; Swift syntax when frame is in a Swift compile unit.
|
|
706
|
+
|
|
707
|
+
p expr evaluate, default-format
|
|
708
|
+
po expr call objects description
|
|
709
|
+
frame variable list all locals (no eval — fast)
|
|
710
|
+
bt full backtrace
|
|
711
|
+
image lookup -n NAME resolve a symbol name to module + addresses
|
|
712
|
+
</eval-expressions>
|
|
713
|
+
|
|
714
|
+
<state-machine>
|
|
715
|
+
paused — debug_eval works; debug_continue resumes.
|
|
716
|
+
running — debug_eval returns 409; debug_breakpoint still works for next hit.
|
|
717
|
+
dead — exited or crashed. Detach and reattach.
|
|
718
|
+
|
|
719
|
+
While the foreground app on a device is paused at a bp, UI input via execute_dsl tap/swipe blocks at WDA until you debug_continue.
|
|
720
|
+
</state-machine>
|
|
721
|
+
|
|
722
|
+
<common-failures>
|
|
723
|
+
"lldb-dap not found" — install Xcode 15+.
|
|
724
|
+
"device debugging requires iOS 17+" — physical-device path needs the on-device tunnel.
|
|
725
|
+
"bundle is not installed on device" — install_app first with a debug-signed build.
|
|
726
|
+
verified=false — symbol mangling mismatch or no debug info. Try image lookup -n NAME via debug_eval.
|
|
727
|
+
"___lldb_unnamed_symbol_*" in stacks — dSYM not loaded. For device builds, run debug_eval "target symbols add /path/to/MyApp.app.dSYM".
|
|
728
|
+
</common-failures>
|
|
729
|
+
|
|
730
|
+
</debugging-reference>
|
|
731
|
+
`;
|
package/package.json
CHANGED
package/server.json
CHANGED
|
@@ -6,12 +6,12 @@
|
|
|
6
6
|
"url": "https://github.com/MobAI-App/mobai-mcp",
|
|
7
7
|
"source": "github"
|
|
8
8
|
},
|
|
9
|
-
"version": "2.
|
|
9
|
+
"version": "2.3.0",
|
|
10
10
|
"packages": [
|
|
11
11
|
{
|
|
12
12
|
"registryType": "npm",
|
|
13
13
|
"identifier": "mobai-mcp",
|
|
14
|
-
"version": "2.
|
|
14
|
+
"version": "2.3.0",
|
|
15
15
|
"transport": {
|
|
16
16
|
"type": "stdio"
|
|
17
17
|
}
|