@sensaiorg/adapter-ios 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,870 @@
1
+ "use strict";
2
+ /**
3
+ * iOS MCP Tools — registered when an iOS Simulator is connected.
4
+ *
5
+ * Uses xcrun simctl for all operations.
6
+ */
7
+ Object.defineProperty(exports, "__esModule", { value: true });
8
+ exports.registerIosTools = registerIosTools;
9
+ const zod_1 = require("zod");
10
+ /** Sleep helper */
11
+ function sleep(ms) {
12
+ return new Promise((resolve) => setTimeout(resolve, ms));
13
+ }
14
+ /**
15
+ * Parse the idb `ui describe-all` output into structured elements.
16
+ *
17
+ * idb output varies but typically contains lines like:
18
+ * Type: Button, Label: "Save", Frame: {{10, 20}, {100, 44}}, Enabled: true
19
+ * or a flat accessibility dump. We parse what we can and include raw on failure.
20
+ */
21
+ function parseIdbAccessibilityOutput(raw) {
22
+ const elements = [];
23
+ const lines = raw.split("\n").filter(l => l.trim().length > 0);
24
+ for (const line of lines) {
25
+ try {
26
+ // Try to extract structured fields from each line
27
+ const typeMatch = line.match(/(?:Type|AXType|class)\s*[:=]\s*["']?([^"',}\n]+)/i);
28
+ const labelMatch = line.match(/(?:Label|AXLabel|title|text)\s*[:=]\s*["']?([^"'\n}]+?)["']?\s*(?:,|$|\})/i);
29
+ const valueMatch = line.match(/(?:Value|AXValue)\s*[:=]\s*["']?([^"'\n}]+?)["']?\s*(?:,|$|\})/i);
30
+ const frameMatch = line.match(/(?:Frame|frame|AXFrame)\s*[:=]\s*\{\{?\s*([\d.]+)\s*,\s*([\d.]+)\s*\}\s*,\s*\{?\s*([\d.]+)\s*,\s*([\d.]+)/i);
31
+ const enabledMatch = line.match(/(?:Enabled|enabled|AXEnabled)\s*[:=]\s*(true|false|1|0|yes|no)/i);
32
+ // Only include if we extracted at least something useful
33
+ const hasData = typeMatch || labelMatch || valueMatch || frameMatch;
34
+ if (!hasData) {
35
+ // Try JSON parsing as fallback (some idb versions output JSON)
36
+ if (line.trim().startsWith("{")) {
37
+ try {
38
+ const obj = JSON.parse(line.trim());
39
+ elements.push({
40
+ type: obj.type || obj.AXType || obj.class || null,
41
+ label: obj.label || obj.AXLabel || obj.title || obj.text || null,
42
+ value: obj.value || obj.AXValue || null,
43
+ frame: obj.frame ? { x: obj.frame.x ?? 0, y: obj.frame.y ?? 0, w: obj.frame.width ?? obj.frame.w ?? 0, h: obj.frame.height ?? obj.frame.h ?? 0 } : null,
44
+ enabled: obj.enabled !== false,
45
+ });
46
+ continue;
47
+ }
48
+ catch {
49
+ // not JSON either
50
+ }
51
+ }
52
+ // Include raw line if it looks like it has content
53
+ if (line.trim().length > 2) {
54
+ elements.push({
55
+ type: null,
56
+ label: line.trim(),
57
+ value: null,
58
+ frame: null,
59
+ enabled: true,
60
+ rawLine: line.trim(),
61
+ });
62
+ }
63
+ continue;
64
+ }
65
+ elements.push({
66
+ type: typeMatch?.[1]?.trim() ?? null,
67
+ label: labelMatch?.[1]?.trim() ?? null,
68
+ value: valueMatch?.[1]?.trim() ?? null,
69
+ frame: frameMatch ? {
70
+ x: parseFloat(frameMatch[1]),
71
+ y: parseFloat(frameMatch[2]),
72
+ w: parseFloat(frameMatch[3]),
73
+ h: parseFloat(frameMatch[4]),
74
+ } : null,
75
+ enabled: enabledMatch ? ["true", "1", "yes"].includes(enabledMatch[1].toLowerCase()) : true,
76
+ });
77
+ }
78
+ catch {
79
+ // If line parsing fails entirely, include raw
80
+ elements.push({
81
+ type: null,
82
+ label: line.trim(),
83
+ value: null,
84
+ frame: null,
85
+ enabled: true,
86
+ rawLine: line.trim(),
87
+ });
88
+ }
89
+ }
90
+ return elements;
91
+ }
92
+ function registerIosTools(server, simctl, bundleId, prefix) {
93
+ // ── diagnose_screen ──────────────────────────────────────────────
94
+ server.tool(`${prefix}diagnose_screen`, "Get a comprehensive diagnosis of the current iOS screen: screenshot (base64), recent logs, and device info. START HERE for any iOS debugging session.", {}, async () => {
95
+ try {
96
+ const [deviceInfo, screenshotBuf, logs] = await Promise.allSettled([
97
+ simctl.getDeviceInfo(),
98
+ simctl.screenshot(),
99
+ simctl.getLogs(`processImagePath contains "${bundleId}"`, "30s")
100
+ .catch(() => "(log retrieval failed)"),
101
+ ]);
102
+ const device = deviceInfo.status === "fulfilled" ? deviceInfo.value : null;
103
+ const screenshot = screenshotBuf.status === "fulfilled"
104
+ ? screenshotBuf.value.toString("base64")
105
+ : null;
106
+ const recentLogs = logs.status === "fulfilled"
107
+ ? logs.value.split("\n").slice(-50).join("\n")
108
+ : "(unavailable)";
109
+ const result = {
110
+ platform: "ios",
111
+ device: device ? { name: device.name, runtime: device.runtime, udid: device.udid } : "unknown",
112
+ bundleId,
113
+ hasScreenshot: !!screenshot,
114
+ recentLogs,
115
+ };
116
+ const content = [
117
+ { type: "text", text: JSON.stringify(result) },
118
+ ];
119
+ if (screenshot) {
120
+ content.push({ type: "image", data: screenshot, mimeType: "image/png" });
121
+ }
122
+ return { content };
123
+ }
124
+ catch (err) {
125
+ return {
126
+ content: [{ type: "text", text: `Error diagnosing iOS screen: ${err instanceof Error ? err.message : String(err)}` }],
127
+ isError: true,
128
+ };
129
+ }
130
+ });
131
+ // ── take_screenshot ──────────────────────────────────────────────
132
+ server.tool(`${prefix}take_screenshot`, "Capture the current iOS Simulator screen as a PNG image.", {}, async () => {
133
+ try {
134
+ const buf = await simctl.screenshot();
135
+ return {
136
+ content: [{
137
+ type: "image",
138
+ data: buf.toString("base64"),
139
+ mimeType: "image/png",
140
+ }],
141
+ };
142
+ }
143
+ catch (err) {
144
+ return {
145
+ content: [{ type: "text", text: `Screenshot failed: ${err instanceof Error ? err.message : String(err)}` }],
146
+ isError: true,
147
+ };
148
+ }
149
+ });
150
+ // ── tap ──────────────────────────────────────────────────────────
151
+ server.tool(`${prefix}tap`, "Tap at specific coordinates on the iOS Simulator screen.", {
152
+ x: zod_1.z.number().describe("X coordinate"),
153
+ y: zod_1.z.number().describe("Y coordinate"),
154
+ }, async ({ x, y }) => {
155
+ try {
156
+ await simctl.tap(x, y);
157
+ return {
158
+ content: [{ type: "text", text: JSON.stringify({ ok: true }) }],
159
+ };
160
+ }
161
+ catch (err) {
162
+ return {
163
+ content: [{ type: "text", text: `Tap failed: ${err instanceof Error ? err.message : String(err)}` }],
164
+ isError: true,
165
+ };
166
+ }
167
+ });
168
+ // ── type_text ────────────────────────────────────────────────────
169
+ server.tool(`${prefix}type_text`, "Type text into the currently focused field on the iOS Simulator.", {
170
+ text: zod_1.z.string().describe("Text to type"),
171
+ }, async ({ text }) => {
172
+ try {
173
+ await simctl.typeText(text);
174
+ return {
175
+ content: [{ type: "text", text: JSON.stringify({ ok: true }) }],
176
+ };
177
+ }
178
+ catch (err) {
179
+ return {
180
+ content: [{ type: "text", text: `Type failed: ${err instanceof Error ? err.message : String(err)}` }],
181
+ isError: true,
182
+ };
183
+ }
184
+ });
185
+ // ── get_logs ─────────────────────────────────────────────────────
186
+ server.tool(`${prefix}get_logs`, "Get recent iOS Simulator logs filtered by predicate. Defaults to app logs.", {
187
+ predicate: zod_1.z.string().optional().describe("Log predicate filter (default: app logs)"),
188
+ duration: zod_1.z.string().optional().describe("Time window, e.g. '30s', '5m' (default: '30s')"),
189
+ grep: zod_1.z.string().optional().describe("Additional grep filter on output"),
190
+ maxLines: zod_1.z.number().optional().describe("Maximum lines to return (default: 100)"),
191
+ }, async ({ predicate, duration, grep, maxLines }) => {
192
+ try {
193
+ const pred = predicate ?? `processImagePath contains "${bundleId}"`;
194
+ const dur = duration ?? "30s";
195
+ let output = await simctl.getLogs(pred, dur);
196
+ if (grep) {
197
+ const regex = new RegExp(grep, "i");
198
+ output = output.split("\n").filter(line => regex.test(line)).join("\n");
199
+ }
200
+ const lines = output.split("\n");
201
+ const limit = maxLines ?? 100;
202
+ const trimmed = lines.slice(-limit).join("\n");
203
+ return {
204
+ content: [{
205
+ type: "text",
206
+ text: JSON.stringify({
207
+ totalLines: lines.length,
208
+ returnedLines: Math.min(lines.length, limit),
209
+ predicate: pred,
210
+ duration: dur,
211
+ logs: trimmed,
212
+ }),
213
+ }],
214
+ };
215
+ }
216
+ catch (err) {
217
+ return {
218
+ content: [{ type: "text", text: `Log retrieval failed: ${err instanceof Error ? err.message : String(err)}` }],
219
+ isError: true,
220
+ };
221
+ }
222
+ });
223
+ // ── launch_app ───────────────────────────────────────────────────
224
+ server.tool(`${prefix}launch_app`, "Launch an app on the iOS Simulator by bundle ID.", {
225
+ bundle: zod_1.z.string().optional().describe(`Bundle ID (default: ${bundleId})`),
226
+ }, async ({ bundle }) => {
227
+ try {
228
+ const bid = bundle ?? bundleId;
229
+ await simctl.launchApp(bid);
230
+ return {
231
+ content: [{ type: "text", text: JSON.stringify({ launched: bid, success: true }) }],
232
+ };
233
+ }
234
+ catch (err) {
235
+ return {
236
+ content: [{ type: "text", text: `Launch failed: ${err instanceof Error ? err.message : String(err)}` }],
237
+ isError: true,
238
+ };
239
+ }
240
+ });
241
+ // ── terminate_app ────────────────────────────────────────────────
242
+ server.tool(`${prefix}terminate_app`, "Terminate a running app on the iOS Simulator.", {
243
+ bundle: zod_1.z.string().optional().describe(`Bundle ID (default: ${bundleId})`),
244
+ }, async ({ bundle }) => {
245
+ try {
246
+ const bid = bundle ?? bundleId;
247
+ await simctl.terminateApp(bid);
248
+ return {
249
+ content: [{ type: "text", text: JSON.stringify({ terminated: bid, success: true }) }],
250
+ };
251
+ }
252
+ catch (err) {
253
+ return {
254
+ content: [{ type: "text", text: `Terminate failed: ${err instanceof Error ? err.message : String(err)}` }],
255
+ isError: true,
256
+ };
257
+ }
258
+ });
259
+ // ── swipe ───────────────────────────────────────────────────────
260
+ server.tool(`${prefix}swipe`, "Swipe from one point to another on the iOS Simulator screen.", {
261
+ startX: zod_1.z.number().describe("Start X coordinate"),
262
+ startY: zod_1.z.number().describe("Start Y coordinate"),
263
+ endX: zod_1.z.number().describe("End X coordinate"),
264
+ endY: zod_1.z.number().describe("End Y coordinate"),
265
+ duration: zod_1.z.number().optional().describe("Swipe duration in seconds (default: 0.3)"),
266
+ }, async ({ startX, startY, endX, endY, duration }) => {
267
+ try {
268
+ await simctl.swipe(startX, startY, endX, endY, duration ?? 0.3);
269
+ return {
270
+ content: [{ type: "text", text: JSON.stringify({ ok: true }) }],
271
+ };
272
+ }
273
+ catch (err) {
274
+ return {
275
+ content: [{ type: "text", text: `Swipe failed: ${err instanceof Error ? err.message : String(err)}` }],
276
+ isError: true,
277
+ };
278
+ }
279
+ });
280
+ // ── describe_all ───────────────────────────────────────────────
281
+ server.tool(`${prefix}describe_all`, "Get the full accessibility tree of the current iOS screen. Returns structured element info including labels, roles, frames, and states.", {}, async () => {
282
+ try {
283
+ const output = await simctl.describeAll();
284
+ return {
285
+ content: [{ type: "text", text: output }],
286
+ };
287
+ }
288
+ catch (err) {
289
+ return {
290
+ content: [{ type: "text", text: `Describe all failed: ${err instanceof Error ? err.message : String(err)}` }],
291
+ isError: true,
292
+ };
293
+ }
294
+ });
295
+ // ── describe_point ─────────────────────────────────────────────
296
+ server.tool(`${prefix}describe_point`, "Get accessibility info for the element at specific coordinates on the iOS screen.", {
297
+ x: zod_1.z.number().describe("X coordinate"),
298
+ y: zod_1.z.number().describe("Y coordinate"),
299
+ }, async ({ x, y }) => {
300
+ try {
301
+ const output = await simctl.describePoint(x, y);
302
+ return {
303
+ content: [{ type: "text", text: output }],
304
+ };
305
+ }
306
+ catch (err) {
307
+ return {
308
+ content: [{ type: "text", text: `Describe point failed: ${err instanceof Error ? err.message : String(err)}` }],
309
+ isError: true,
310
+ };
311
+ }
312
+ });
313
+ // ── open_deep_link ──────────────────────────────────────────────
314
+ server.tool(`${prefix}open_deep_link`, "Open the app via a deep link URL scheme on iOS Simulator. Navigates directly to a specific screen without tapping through the UI.", {
315
+ url: zod_1.z.string().describe("Deep link URL (e.g., myapp://screen/detail?id=123 or https://example.com/path)"),
316
+ }, async ({ url }) => {
317
+ try {
318
+ await simctl.exec("openurl", [url]);
319
+ return {
320
+ content: [{ type: "text", text: JSON.stringify({ success: true, url }) }],
321
+ };
322
+ }
323
+ catch (err) {
324
+ return {
325
+ content: [{ type: "text", text: `open_deep_link failed: ${err instanceof Error ? err.message : String(err)}` }],
326
+ isError: true,
327
+ };
328
+ }
329
+ });
330
+ // ── get_device_info ──────────────────────────────────────────────
331
+ server.tool(`${prefix}get_device_info`, "Get detailed info about the connected iOS Simulator (name, runtime, UDID, state).", {}, async () => {
332
+ try {
333
+ const info = await simctl.getDeviceInfo();
334
+ return {
335
+ content: [{ type: "text", text: JSON.stringify(info) }],
336
+ };
337
+ }
338
+ catch (err) {
339
+ return {
340
+ content: [{ type: "text", text: `Device info failed: ${err instanceof Error ? err.message : String(err)}` }],
341
+ isError: true,
342
+ };
343
+ }
344
+ });
345
+ // ── get_ui_tree ────────────────────────────────────────────────
346
+ server.tool(`${prefix}get_ui_tree`, "Parse the iOS accessibility tree into structured JSON. Returns elements with type, label, value, frame (x,y,w,h), and enabled state.", {}, async () => {
347
+ try {
348
+ const raw = await simctl.describeAll();
349
+ const elements = parseIdbAccessibilityOutput(raw);
350
+ return {
351
+ content: [{
352
+ type: "text",
353
+ text: JSON.stringify({
354
+ totalElements: elements.length,
355
+ clickableElements: elements.filter(e => e.type === "Button" || e.type === "Link").length,
356
+ textElements: elements.filter(e => e.label || e.value).length,
357
+ elements,
358
+ }),
359
+ }],
360
+ };
361
+ }
362
+ catch (err) {
363
+ return {
364
+ content: [{ type: "text", text: `get_ui_tree failed: ${err instanceof Error ? err.message : String(err)}` }],
365
+ isError: true,
366
+ };
367
+ }
368
+ });
369
+ // ── get_screen_text ────────────────────────────────────────────
370
+ server.tool(`${prefix}get_screen_text`, "Extract all visible text from the iOS screen's accessibility tree. Returns a flat array of text strings in reading order.", {}, async () => {
371
+ try {
372
+ const raw = await simctl.describeAll();
373
+ const elements = parseIdbAccessibilityOutput(raw);
374
+ const texts = elements
375
+ .map(e => e.label || e.value || "")
376
+ .filter(t => t.length > 0);
377
+ return {
378
+ content: [{ type: "text", text: JSON.stringify({ screenText: texts }) }],
379
+ };
380
+ }
381
+ catch (err) {
382
+ return {
383
+ content: [{ type: "text", text: `get_screen_text failed: ${err instanceof Error ? err.message : String(err)}` }],
384
+ isError: true,
385
+ };
386
+ }
387
+ });
388
+ // ── get_element_details ────────────────────────────────────────
389
+ server.tool(`${prefix}get_element_details`, "Find and inspect iOS UI elements matching a text query or at specific coordinates. Returns full details including frame, type, and state.", {
390
+ query: zod_1.z.string().optional().describe("Text to search for (case-insensitive substring match)"),
391
+ x: zod_1.z.number().optional().describe("X coordinate to find element at"),
392
+ y: zod_1.z.number().optional().describe("Y coordinate to find element at"),
393
+ }, async ({ query, x, y }) => {
394
+ try {
395
+ // If coordinates given, use describe_point for precise result
396
+ if (x !== undefined && y !== undefined) {
397
+ const pointInfo = await simctl.describePoint(x, y);
398
+ return {
399
+ content: [{ type: "text", text: JSON.stringify({ found: 1, raw: pointInfo }) }],
400
+ };
401
+ }
402
+ if (!query) {
403
+ return {
404
+ content: [{ type: "text", text: JSON.stringify({ error: "Provide either 'query' or x/y coordinates" }) }],
405
+ isError: true,
406
+ };
407
+ }
408
+ const raw = await simctl.describeAll();
409
+ const elements = parseIdbAccessibilityOutput(raw);
410
+ const lowerQuery = query.toLowerCase();
411
+ const matches = elements.filter(e => (e.label && e.label.toLowerCase().includes(lowerQuery)) ||
412
+ (e.value && e.value.toLowerCase().includes(lowerQuery)) ||
413
+ (e.type && e.type.toLowerCase().includes(lowerQuery)));
414
+ if (matches.length === 0) {
415
+ return {
416
+ content: [{
417
+ type: "text",
418
+ text: JSON.stringify({ found: 0, message: `No elements found matching "${query}"`, hint: "Try ios_get_screen_text to see all visible text." }),
419
+ }],
420
+ };
421
+ }
422
+ const details = matches.map(e => ({
423
+ ...e,
424
+ centerX: e.frame ? Math.round(e.frame.x + e.frame.w / 2) : null,
425
+ centerY: e.frame ? Math.round(e.frame.y + e.frame.h / 2) : null,
426
+ }));
427
+ return {
428
+ content: [{ type: "text", text: JSON.stringify({ found: matches.length, elements: details }) }],
429
+ };
430
+ }
431
+ catch (err) {
432
+ return {
433
+ content: [{ type: "text", text: `get_element_details failed: ${err instanceof Error ? err.message : String(err)}` }],
434
+ isError: true,
435
+ };
436
+ }
437
+ });
438
+ // ── wait_for_text ──────────────────────────────────────────────
439
+ server.tool(`${prefix}wait_for_text`, "Poll the iOS screen until specific text appears. Essential after navigation, screen transitions, or async operations.", {
440
+ text: zod_1.z.string().describe("Text to wait for (case-insensitive substring match)"),
441
+ timeoutMs: zod_1.z.number().optional().describe("Maximum wait time in ms (default: 10000)"),
442
+ pollIntervalMs: zod_1.z.number().optional().describe("Polling interval in ms (default: 1000)"),
443
+ }, async ({ text, timeoutMs, pollIntervalMs }) => {
444
+ const timeout = timeoutMs ?? 10_000;
445
+ const interval = pollIntervalMs ?? 1_000;
446
+ const startTime = Date.now();
447
+ const searchLower = text.toLowerCase();
448
+ let attempts = 0;
449
+ while (Date.now() - startTime < timeout) {
450
+ attempts++;
451
+ try {
452
+ const raw = await simctl.describeAll();
453
+ const elements = parseIdbAccessibilityOutput(raw);
454
+ const allText = elements
455
+ .map(e => `${e.label ?? ""} ${e.value ?? ""}`.toLowerCase())
456
+ .join(" ");
457
+ if (allText.includes(searchLower)) {
458
+ return {
459
+ content: [{ type: "text", text: JSON.stringify({ success: true, found: true, text, elapsedMs: Date.now() - startTime, attempts }) }],
460
+ };
461
+ }
462
+ }
463
+ catch {
464
+ // retry
465
+ }
466
+ if (Date.now() - startTime + interval < timeout) {
467
+ await sleep(interval);
468
+ }
469
+ }
470
+ return {
471
+ content: [{
472
+ type: "text",
473
+ text: JSON.stringify({ success: false, found: false, text, elapsedMs: Date.now() - startTime, attempts, hint: "Text did not appear within timeout." }),
474
+ }],
475
+ };
476
+ });
477
+ // ── scroll_to_text ─────────────────────────────────────────────
478
+ server.tool(`${prefix}scroll_to_text`, "Swipe/scroll repeatedly until specific text appears on the iOS screen. Useful for finding elements in long scrollable lists.", {
479
+ text: zod_1.z.string().describe("Text to scroll to (case-insensitive substring match)"),
480
+ direction: zod_1.z.enum(["up", "down", "left", "right"]).optional().describe("Scroll direction (default: down)"),
481
+ maxScrolls: zod_1.z.number().optional().describe("Maximum scroll attempts (default: 5)"),
482
+ }, async ({ text, direction, maxScrolls }) => {
483
+ const dir = direction ?? "down";
484
+ const max = maxScrolls ?? 5;
485
+ const searchLower = text.toLowerCase();
486
+ // Default screen dimensions for iPhone (logical points)
487
+ const screenW = 390;
488
+ const screenH = 844;
489
+ const centerX = Math.round(screenW / 2);
490
+ for (let i = 0; i < max; i++) {
491
+ try {
492
+ const raw = await simctl.describeAll();
493
+ const elements = parseIdbAccessibilityOutput(raw);
494
+ const match = elements.find(e => (e.label && e.label.toLowerCase().includes(searchLower)) ||
495
+ (e.value && e.value.toLowerCase().includes(searchLower)));
496
+ if (match) {
497
+ const center = match.frame
498
+ ? { x: Math.round(match.frame.x + match.frame.w / 2), y: Math.round(match.frame.y + match.frame.h / 2) }
499
+ : null;
500
+ return {
501
+ content: [{
502
+ type: "text",
503
+ text: JSON.stringify({ success: true, found: true, text: match.label || match.value, center, scrollsNeeded: i }),
504
+ }],
505
+ };
506
+ }
507
+ }
508
+ catch {
509
+ // continue scrolling
510
+ }
511
+ // Perform swipe
512
+ let startX = centerX, startY = Math.round(screenH * 0.7);
513
+ let endX = centerX, endY = Math.round(screenH * 0.3);
514
+ if (dir === "up") {
515
+ startY = Math.round(screenH * 0.3);
516
+ endY = Math.round(screenH * 0.7);
517
+ }
518
+ if (dir === "left") {
519
+ startX = Math.round(screenW * 0.7);
520
+ endX = Math.round(screenW * 0.3);
521
+ startY = Math.round(screenH / 2);
522
+ endY = startY;
523
+ }
524
+ if (dir === "right") {
525
+ startX = Math.round(screenW * 0.3);
526
+ endX = Math.round(screenW * 0.7);
527
+ startY = Math.round(screenH / 2);
528
+ endY = startY;
529
+ }
530
+ await simctl.swipe(startX, startY, endX, endY, 0.3);
531
+ await sleep(500);
532
+ }
533
+ return {
534
+ content: [{
535
+ type: "text",
536
+ text: JSON.stringify({ success: false, found: false, text, scrollAttempts: max, hint: `Text "${text}" not found after ${max} scroll attempts.` }),
537
+ }],
538
+ };
539
+ });
540
+ // ── fill_form ──────────────────────────────────────────────────
541
+ server.tool(`${prefix}fill_form`, "Batch fill form fields on iOS. For each field: finds element by label in the accessibility tree, taps its coordinates, clears existing text, types new value.", {
542
+ fields: zod_1.z.array(zod_1.z.object({
543
+ label: zod_1.z.string().describe("Label/text to find the field by (case-insensitive)"),
544
+ value: zod_1.z.string().describe("Text to enter in the field"),
545
+ clearFirst: zod_1.z.boolean().optional().describe("Clear existing text before typing (default: true)"),
546
+ })).describe("Array of field labels and values to fill"),
547
+ }, async ({ fields }) => {
548
+ const results = [];
549
+ for (const field of fields) {
550
+ const clearFirst = field.clearFirst ?? true;
551
+ try {
552
+ const raw = await simctl.describeAll();
553
+ const elements = parseIdbAccessibilityOutput(raw);
554
+ const lowerLabel = field.label.toLowerCase();
555
+ // Find the element by label
556
+ const match = elements.find(e => (e.label && e.label.toLowerCase().includes(lowerLabel)) ||
557
+ (e.value && e.value.toLowerCase().includes(lowerLabel)));
558
+ if (!match || !match.frame) {
559
+ results.push({ field: field.label, success: false, error: "Element not found or no frame" });
560
+ continue;
561
+ }
562
+ const tapX = Math.round(match.frame.x + match.frame.w / 2);
563
+ const tapY = Math.round(match.frame.y + match.frame.h / 2);
564
+ // Tap the field
565
+ await simctl.tap(tapX, tapY);
566
+ await sleep(300);
567
+ // Clear existing text if needed (select all + delete)
568
+ if (clearFirst) {
569
+ // Triple-tap to select all, then delete
570
+ await simctl.tap(tapX, tapY);
571
+ await sleep(50);
572
+ await simctl.tap(tapX, tapY);
573
+ await sleep(50);
574
+ await simctl.tap(tapX, tapY);
575
+ await sleep(200);
576
+ await simctl.typeText(""); // This may not clear; use idb key sequence instead
577
+ try {
578
+ await simctl.keyPress(42); // DELETE key via idb
579
+ }
580
+ catch {
581
+ // If keyPress fails, try typing empty — field may already be selected
582
+ }
583
+ }
584
+ // Type the value
585
+ await simctl.typeText(field.value);
586
+ await sleep(200);
587
+ results.push({ field: field.label, success: true });
588
+ }
589
+ catch (err) {
590
+ results.push({ field: field.label, success: false, error: err instanceof Error ? err.message : String(err) });
591
+ }
592
+ }
593
+ const allSuccess = results.every(r => r.success);
594
+ return {
595
+ content: [{
596
+ type: "text",
597
+ text: JSON.stringify({ success: allSuccess, filledCount: results.filter(r => r.success).length, totalFields: fields.length, results }),
598
+ }],
599
+ };
600
+ });
601
+ // ── tap_and_wait ───────────────────────────────────────────────
602
+ server.tool(`${prefix}tap_and_wait`, "Tap an element (by label or coordinates) on iOS, then poll until expected text appears. Combines tap + wait_for_text in one call.", {
603
+ label: zod_1.z.string().optional().describe("Label/text of element to tap (case-insensitive)"),
604
+ x: zod_1.z.number().optional().describe("X coordinate to tap (used if label not found or not provided)"),
605
+ y: zod_1.z.number().optional().describe("Y coordinate to tap (used if label not found or not provided)"),
606
+ waitForText: zod_1.z.string().describe("Text to wait for after tap"),
607
+ timeoutMs: zod_1.z.number().optional().describe("Max wait time in ms (default: 10000)"),
608
+ }, async ({ label, x, y, waitForText, timeoutMs }) => {
609
+ try {
610
+ let tapX = x;
611
+ let tapY = y;
612
+ let matchedBy = tapX !== undefined ? `coordinates (${tapX}, ${tapY})` : "";
613
+ // Find element by label if provided
614
+ if (label) {
615
+ const raw = await simctl.describeAll();
616
+ const elements = parseIdbAccessibilityOutput(raw);
617
+ const lowerLabel = label.toLowerCase();
618
+ const match = elements.find(e => (e.label && e.label.toLowerCase().includes(lowerLabel)) ||
619
+ (e.value && e.value.toLowerCase().includes(lowerLabel)));
620
+ if (match?.frame) {
621
+ tapX = Math.round(match.frame.x + match.frame.w / 2);
622
+ tapY = Math.round(match.frame.y + match.frame.h / 2);
623
+ matchedBy = `label="${match.label || match.value}"`;
624
+ }
625
+ }
626
+ if (tapX === undefined || tapY === undefined) {
627
+ return {
628
+ content: [{ type: "text", text: JSON.stringify({ success: false, error: "Element not found and no coordinates provided" }) }],
629
+ };
630
+ }
631
+ // Tap
632
+ await simctl.tap(tapX, tapY);
633
+ // Wait for text
634
+ const timeout = timeoutMs ?? 10_000;
635
+ const interval = 500;
636
+ const startTime = Date.now();
637
+ const searchLower = waitForText.toLowerCase();
638
+ while (Date.now() - startTime < timeout) {
639
+ await sleep(interval);
640
+ try {
641
+ const raw = await simctl.describeAll();
642
+ const elements = parseIdbAccessibilityOutput(raw);
643
+ const allText = elements.map(e => `${e.label ?? ""} ${e.value ?? ""}`.toLowerCase()).join(" ");
644
+ if (allText.includes(searchLower)) {
645
+ return {
646
+ content: [{
647
+ type: "text",
648
+ text: JSON.stringify({ success: true, tappedBy: matchedBy, tapped: { x: tapX, y: tapY }, waitedFor: waitForText, elapsedMs: Date.now() - startTime }),
649
+ }],
650
+ };
651
+ }
652
+ }
653
+ catch {
654
+ // retry
655
+ }
656
+ }
657
+ return {
658
+ content: [{
659
+ type: "text",
660
+ text: JSON.stringify({ success: false, tappedBy: matchedBy, tapped: { x: tapX, y: tapY }, waitedFor: waitForText, timedOut: true, elapsedMs: Date.now() - startTime }),
661
+ }],
662
+ };
663
+ }
664
+ catch (err) {
665
+ return {
666
+ content: [{ type: "text", text: `tap_and_wait failed: ${err instanceof Error ? err.message : String(err)}` }],
667
+ isError: true,
668
+ };
669
+ }
670
+ });
671
+ // ── assert_screen ──────────────────────────────────────────────
672
+ server.tool(`${prefix}assert_screen`, "Quickly verify the iOS screen contains (or doesn't contain) expected text. Returns pass/fail without the full UI tree, saving tokens.", {
673
+ contains: zod_1.z.array(zod_1.z.string()).optional().describe("Text strings that MUST be on screen"),
674
+ notContains: zod_1.z.array(zod_1.z.string()).optional().describe("Text strings that must NOT be on screen"),
675
+ }, async ({ contains, notContains }) => {
676
+ try {
677
+ const raw = await simctl.describeAll();
678
+ const elements = parseIdbAccessibilityOutput(raw);
679
+ const allText = elements
680
+ .map(e => `${e.label ?? ""} ${e.value ?? ""}`.toLowerCase());
681
+ const missing = [];
682
+ const unexpected = [];
683
+ for (const expected of contains ?? []) {
684
+ if (!allText.some(t => t.includes(expected.toLowerCase()))) {
685
+ missing.push(expected);
686
+ }
687
+ }
688
+ for (const banned of notContains ?? []) {
689
+ if (allText.some(t => t.includes(banned.toLowerCase()))) {
690
+ unexpected.push(banned);
691
+ }
692
+ }
693
+ const pass = missing.length === 0 && unexpected.length === 0;
694
+ return {
695
+ content: [{
696
+ type: "text",
697
+ text: JSON.stringify({
698
+ pass,
699
+ ...(missing.length > 0 ? { missing } : {}),
700
+ ...(unexpected.length > 0 ? { unexpected } : {}),
701
+ }),
702
+ }],
703
+ };
704
+ }
705
+ catch (err) {
706
+ return {
707
+ content: [{ type: "text", text: `assert_screen failed: ${err instanceof Error ? err.message : String(err)}` }],
708
+ isError: true,
709
+ };
710
+ }
711
+ });
712
+ // ── get_crash_info ─────────────────────────────────────────────
713
+ server.tool(`${prefix}get_crash_info`, "Get recent crash and exception logs from the iOS Simulator. Searches the last N minutes for crash/exception/fatal/SIGABRT messages.", {
714
+ minutes: zod_1.z.number().optional().describe("How many minutes back to search (default: 5)"),
715
+ maxLines: zod_1.z.number().optional().describe("Maximum lines to return (default: 50)"),
716
+ }, async ({ minutes, maxLines }) => {
717
+ try {
718
+ const output = await simctl.getCrashLogs(minutes ?? 5);
719
+ const lines = output.split("\n").filter(l => l.trim().length > 0);
720
+ const limit = maxLines ?? 50;
721
+ const trimmed = lines.slice(-limit);
722
+ return {
723
+ content: [{
724
+ type: "text",
725
+ text: JSON.stringify({ totalLines: lines.length, returnedLines: trimmed.length, logs: trimmed.join("\n") }),
726
+ }],
727
+ };
728
+ }
729
+ catch (err) {
730
+ return {
731
+ content: [{ type: "text", text: `get_crash_info failed: ${err instanceof Error ? err.message : String(err)}` }],
732
+ isError: true,
733
+ };
734
+ }
735
+ });
736
+ // ── hot_reload ─────────────────────────────────────────────────
737
+ server.tool(`${prefix}hot_reload`, "Trigger a React Native reload via the Metro bundler on iOS. 'hot' performs a fast refresh, 'full' performs a complete bundle reload.", {
738
+ type: zod_1.z.enum(["hot", "full"]).optional().describe("Reload type: 'hot' for fast refresh (default), 'full' for complete reload"),
739
+ }, async ({ type }) => {
740
+ const reloadType = type ?? "hot";
741
+ try {
742
+ const endpoint = reloadType === "full"
743
+ ? "http://localhost:8081/reload"
744
+ : "http://localhost:8081/message";
745
+ if (reloadType === "full") {
746
+ await fetch(endpoint);
747
+ }
748
+ else {
749
+ await fetch(endpoint, {
750
+ method: "POST",
751
+ headers: { "Content-Type": "application/json" },
752
+ body: JSON.stringify({ method: "reload" }),
753
+ });
754
+ }
755
+ return {
756
+ content: [{
757
+ type: "text",
758
+ text: JSON.stringify({ success: true, type: reloadType, message: `${reloadType === "full" ? "Full" : "Hot"} reload triggered.` }),
759
+ }],
760
+ };
761
+ }
762
+ catch (err) {
763
+ return {
764
+ content: [{
765
+ type: "text",
766
+ text: JSON.stringify({
767
+ success: false,
768
+ error: err instanceof Error ? err.message : String(err),
769
+ hints: ["Ensure Metro bundler is running (npx react-native start)", "Ensure the app is in dev mode"],
770
+ }),
771
+ }],
772
+ isError: true,
773
+ };
774
+ }
775
+ });
776
+ // ── start_recording ───────────────────────────────────────────
777
+ // eslint-disable-next-line @typescript-eslint/consistent-type-imports
778
+ let iosRecordingProcess = null;
779
+ let iosRecordingPath = null;
780
+ server.tool(`${prefix}start_recording`, "Start recording the iOS Simulator screen. Recording runs in the background. Use stop_recording to finish and retrieve the video.", {
781
+ maxDurationSec: zod_1.z.number().optional().describe("Maximum recording duration in seconds (default: 60)"),
782
+ }, async ({ maxDurationSec }) => {
783
+ if (iosRecordingProcess) {
784
+ return {
785
+ content: [{ type: "text", text: JSON.stringify({ error: "Recording already in progress. Call ios_stop_recording first." }) }],
786
+ isError: true,
787
+ };
788
+ }
789
+ try {
790
+ const { spawn } = await import("node:child_process");
791
+ const duration = maxDurationSec ?? 60;
792
+ iosRecordingPath = `/tmp/sensai-ios-recording-${Date.now()}.mp4`;
793
+ // simctl recordVideo runs until SIGINT
794
+ const proc = spawn("xcrun", [
795
+ "simctl", "io", "booted", "recordVideo",
796
+ "--force", iosRecordingPath,
797
+ ], { stdio: "ignore" });
798
+ iosRecordingProcess = proc;
799
+ // Auto-stop after max duration
800
+ setTimeout(() => {
801
+ if (iosRecordingProcess === proc) {
802
+ proc.kill("SIGINT");
803
+ iosRecordingProcess = null;
804
+ }
805
+ }, duration * 1000);
806
+ proc.on("exit", () => {
807
+ if (iosRecordingProcess === proc) {
808
+ iosRecordingProcess = null;
809
+ }
810
+ });
811
+ return {
812
+ content: [{
813
+ type: "text",
814
+ text: JSON.stringify({ ok: true, maxDurationSec: duration, path: iosRecordingPath }),
815
+ }],
816
+ };
817
+ }
818
+ catch (err) {
819
+ iosRecordingProcess = null;
820
+ return {
821
+ content: [{ type: "text", text: `start_recording failed: ${err instanceof Error ? err.message : String(err)}` }],
822
+ isError: true,
823
+ };
824
+ }
825
+ });
826
+ // ── stop_recording ────────────────────────────────────────────
827
+ server.tool(`${prefix}stop_recording`, "Stop the active iOS screen recording and retrieve the MP4 video file.", {
828
+ savePath: zod_1.z.string().optional().describe("Local path to save the MP4 (optional). If not provided, returns base64."),
829
+ }, async ({ savePath }) => {
830
+ if (!iosRecordingProcess) {
831
+ return {
832
+ content: [{ type: "text", text: JSON.stringify({ error: "No active recording. Call ios_start_recording first." }) }],
833
+ isError: true,
834
+ };
835
+ }
836
+ try {
837
+ // Send SIGINT to finalize the recording
838
+ iosRecordingProcess.kill("SIGINT");
839
+ iosRecordingProcess = null;
840
+ // Wait for file to be finalized
841
+ await sleep(2000);
842
+ const recordPath = iosRecordingPath;
843
+ const { readFile: rf, unlink: ul, copyFile: cp } = await import("node:fs/promises");
844
+ if (savePath) {
845
+ await cp(recordPath, savePath);
846
+ await ul(recordPath).catch(() => { });
847
+ return {
848
+ content: [{ type: "text", text: JSON.stringify({ ok: true, savedTo: savePath }) }],
849
+ };
850
+ }
851
+ const data = await rf(recordPath);
852
+ await ul(recordPath).catch(() => { });
853
+ iosRecordingPath = null;
854
+ return {
855
+ content: [{
856
+ type: "text",
857
+ text: JSON.stringify({ ok: true, format: "mp4", sizeBytes: data.length, base64: data.toString("base64") }),
858
+ }],
859
+ };
860
+ }
861
+ catch (err) {
862
+ iosRecordingProcess = null;
863
+ return {
864
+ content: [{ type: "text", text: `stop_recording failed: ${err instanceof Error ? err.message : String(err)}` }],
865
+ isError: true,
866
+ };
867
+ }
868
+ });
869
+ }
870
+ //# sourceMappingURL=ios-tools.js.map