@midscene/ios 1.9.8 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1485 +0,0 @@
1
- import { BaseMCPServer, createMCPServerLauncher } from "@midscene/shared/mcp";
2
- import { Agent } from "@midscene/core/agent";
3
- import { MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE } from "@midscene/shared/env";
4
- import { getDebug } from "@midscene/shared/logger";
5
- import { mergeAndNormalizeAppNameMapping, normalizeForComparison } from "@midscene/shared/utils";
6
- import node_assert from "node:assert";
7
- import { z } from "@midscene/core";
8
- import { createDefaultMobileActions, defineAction } from "@midscene/core/device";
9
- import { sleep } from "@midscene/core/utils";
10
- import { DEFAULT_WDA_PORT } from "@midscene/shared/constants";
11
- import { createImgBase64ByFormat } from "@midscene/shared/img";
12
- import { WDAManager, WebDriverClient } from "@midscene/webdriver";
13
- import { agentBehaviorInitArgShape, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs } from "@midscene/shared/mcp/agent-behavior-init-args";
14
- import { BaseMidsceneTools } from "@midscene/shared/mcp/base-tools";
15
- const defaultAppNameMapping = {
16
- 微信: 'com.tencent.xin',
17
- 企业微信: 'com.tencent.ww',
18
- 微信读书: 'com.tencent.weread',
19
- 微信听书: 'com.tencent.wehear',
20
- QQ: 'com.tencent.mqq',
21
- QQ音乐: 'com.tencent.QQMusic',
22
- QQ阅读: 'com.tencent.qqreaderiphone',
23
- QQ邮箱: 'com.tencent.qqmail',
24
- QQ浏览器: 'com.tencent.mttlite',
25
- TIM: 'com.tencent.tim',
26
- 微视: 'com.tencent.microvision',
27
- 腾讯新闻: 'com.tencent.info',
28
- 腾讯视频: 'com.tencent.live4iphone',
29
- 腾讯动漫: 'com.tencent.ied.app.comic',
30
- 腾讯微云: 'com.tencent.weiyun',
31
- 腾讯体育: 'com.tencent.sportskbs',
32
- 腾讯文档: 'com.tencent.txdocs',
33
- 腾讯翻译君: 'com.tencent.qqtranslator',
34
- 腾讯课堂: 'com.tencent.edu',
35
- 腾讯地图: 'com.tencent.sosomap',
36
- 小鹅拼拼: 'com.tencent.dwdcoco',
37
- 全民k歌: 'com.tencent.QQKSong',
38
- 支付宝: 'com.alipay.iphoneclient',
39
- 钉钉: 'com.laiwang.DingTalk',
40
- 闲鱼: 'com.taobao.fleamarket',
41
- 淘宝: 'com.taobao.taobao4iphone',
42
- 斗鱼: 'tv.douyu.live',
43
- 天猫: 'com.taobao.tmall',
44
- 口碑: 'com.taobao.kbmeishi',
45
- 饿了么: 'me.ele.ios.eleme',
46
- 高德地图: 'com.autonavi.amap',
47
- UC浏览器: 'com.ucweb.iphone.lowversion',
48
- 一淘: 'com.taobao.etaocoupon',
49
- 飞猪: 'com.taobao.travel',
50
- 虾米音乐: 'com.xiami.spark',
51
- 淘票票: 'com.taobao.movie.MoviePhoneClient',
52
- 优酷: 'com.youku.YouKu',
53
- 菜鸟裹裹: 'com.cainiao.cnwireless',
54
- 土豆视频: 'com.tudou.tudouiphone',
55
- 抖音: 'com.ss.iphone.ugc.Aweme',
56
- 抖音极速版: 'com.ss.iphone.ugc.aweme.lite',
57
- 抖音火山版: 'com.ss.iphone.ugc.Live',
58
- 懂车帝: 'com.ss.ios.auto',
59
- Tiktok: 'com.zhiliaoapp.musically',
60
- 飞书: 'com.bytedance.ee.lark',
61
- 今日头条: 'com.ss.iphone.article.News',
62
- 西瓜视频: 'com.ss.iphone.article.Video',
63
- 皮皮虾: 'com.bd.iphone.super',
64
- 美团: 'com.meituan.imeituan',
65
- 美团外卖: 'com.meituan.itakeaway',
66
- 大众点评: 'com.dianping.dpscope',
67
- 美团优选: 'com.meituan.iyouxuan',
68
- 美团优选团长: 'com.meituan.igrocery.gh',
69
- 美团骑手: 'com.meituan.banma.homebrew',
70
- 美团开店宝: 'com.meituan.imerchantbiz',
71
- 美团拍店: 'com.meituan.pai',
72
- 美团众包: 'com.meituan.banma.crowdsource',
73
- 美团买菜: 'com.baobaoaichi.imaicai',
74
- 京东: 'com.360buy.jdmobile',
75
- 京东读书: 'com.jd.reader',
76
- 网易新闻: 'com.netease.news',
77
- 网易云音乐: 'com.netease.cloudmusic',
78
- 网易邮箱大师: 'com.netease.macmail',
79
- 网易严选: 'com.netease.yanxuan',
80
- 网易公开课: 'com.netease.videoHD',
81
- 网易有道词典: 'youdaoPro',
82
- 有道云笔记: 'com.youdao.note.YoudaoNoteMac',
83
- 百度: 'com.baidu.BaiduMobile',
84
- 百度网盘: 'com.baidu.netdisk',
85
- 百度贴吧: 'com.baidu.tieba',
86
- 百度地图: 'com.baidu.map',
87
- 百度阅读: 'com.baidu.yuedu',
88
- 百度翻译: 'com.baidu.translate',
89
- 百度文库: 'com.baidu.Wenku',
90
- 百度视频: 'com.baidu.videoiphone',
91
- 百度输入法: 'com.baidu.inputMethod',
92
- 快手: 'com.jiangjia.gif',
93
- 快手极速版: 'com.kuaishou.nebula',
94
- 哔哩哔哩: 'tv.danmaku.bilianime',
95
- 芒果TV: 'com.hunantv.imgotv',
96
- 苏宁易购: 'SuningEMall',
97
- 微博: 'com.sina.weibo',
98
- 微博极速版: 'com.sina.weibolite',
99
- 微博国际: 'com.weibo.international',
100
- 墨客: 'com.moke.moke.iphone',
101
- 豆瓣: 'com.douban.frodo',
102
- 知乎: 'com.zhihu.ios',
103
- 小红书: 'com.xingin.discover',
104
- 喜马拉雅: 'com.gemd.iting',
105
- 得到: 'com.luojilab.LuoJiFM-IOS',
106
- 得物: 'com.siwuai.duapp',
107
- 起点读书: 'm.qidian.QDReaderAppStore',
108
- 番茄小说: 'com.dragon.read',
109
- 书旗小说: 'com.shuqicenter.reader',
110
- 拼多多: 'com.xunmeng.pinduoduo',
111
- 多点: 'com.dmall.dmall',
112
- 便利蜂: 'com.bianlifeng.customer.ios',
113
- 亿通行: 'com.ruubypay.yitongxing',
114
- 云闪付: 'com.unionpay.chsp',
115
- 大都会Metro: 'com.DDH.SHSubway',
116
- 爱奇艺视频: 'com.qiyi.iphone',
117
- 搜狐视频: 'com.sohu.iPhoneVideo',
118
- 搜狐新闻: 'com.sohu.newspaper',
119
- 搜狗浏览器: 'com.sogou.SogouExplorerMobile',
120
- 虎牙: 'com.yy.kiwi',
121
- 比心: 'com.yitan.bixin',
122
- 转转: 'com.wuba.zhuanzhuan',
123
- YY: 'yyvoice',
124
- 绿洲: 'com.sina.oasis',
125
- 陌陌: 'com.wemomo.momoappdemo1',
126
- 什么值得买: 'com.smzdm.client.ios',
127
- 美团秀秀: 'com.meitu.mtxx',
128
- 唯品会: 'com.vipshop.iphone',
129
- 唱吧: 'com.changba.ktv',
130
- 酷狗音乐: 'com.kugou.kugou1002',
131
- CSDN: 'net.csdn.CsdnPlus',
132
- 多抓鱼: 'com.duozhuyu.dejavu',
133
- 自如: 'com.ziroom.ZiroomProject',
134
- 携程: 'ctrip.com',
135
- 去哪儿旅行: 'com.qunar.iphoneclient8',
136
- Xmind: 'net.xmind.brownieapp',
137
- 印象笔记: 'com.yinxiang.iPhone',
138
- 欧陆词典: 'eusoft.eudic.pro',
139
- 115: 'com.115.personal',
140
- 名片全能王: 'com.intsig.camcard.lite',
141
- 中国银行: 'com.boc.BOCMBCI',
142
- '58同城': 'com.taofang.iphone',
143
- 'Google Chrome': 'com.google.chrome.ios',
144
- Gmail: 'com.google.Gmail',
145
- Facebook: 'com.facebook.Facebook',
146
- Firefox: 'org.mozilla.ios.Firefox',
147
- Messenger: 'com.facebook.Messenger',
148
- Instagram: 'com.burbn.instagram',
149
- Starbucks: 'com.starbucks.mystarbucks',
150
- 'Luckin Coffee': 'com.bjlc.luckycoffee',
151
- Line: 'jp.naver.line',
152
- Linkedin: 'com.linkedin.LinkedIn',
153
- Dcard: 'com.dcard.app.Dcard',
154
- Youtube: 'com.google.ios.youtube',
155
- Spotify: 'com.spotify.client',
156
- Netflix: 'com.netflix.Netflix',
157
- Twitter: 'com.atebits.Tweetie2',
158
- WhatsApp: 'net.whatsapp.WhatsApp',
159
- Safari: 'com.apple.mobilesafari',
160
- 'App Store': 'com.apple.AppStore',
161
- 设置: 'com.apple.Preferences',
162
- 相机: 'com.apple.camera',
163
- 照片: 'com.apple.mobileslideshow',
164
- 时钟: 'com.apple.mobiletimer',
165
- 闹钟: 'com.apple.mobiletimer',
166
- 备忘录: 'com.apple.mobilenotes',
167
- 提醒事项: 'com.apple.reminders',
168
- 快捷指令: 'com.apple.shortcuts',
169
- 天气: 'com.apple.weather',
170
- 日历: 'com.apple.mobilecal',
171
- 地图: 'com.apple.Maps',
172
- 电话: 'com.apple.mobilephone',
173
- 通讯录: 'com.apple.MobileAddressBook',
174
- 信息: 'com.apple.MobileSMS',
175
- FaceTime: 'com.apple.facetime',
176
- 计算器: 'com.apple.calculator',
177
- 家庭: 'com.apple.Home',
178
- 健康: 'com.apple.Health',
179
- 钱包: 'com.apple.Passbook',
180
- 股市: 'com.apple.stocks',
181
- 图书: 'com.apple.iBooks',
182
- 新闻: 'com.apple.news',
183
- 视频: 'com.apple.tv',
184
- 文件: 'com.apple.DocumentsApp',
185
- 邮件: 'com.apple.mobilemail',
186
- 查找: 'com.apple.findmy',
187
- 翻译: 'com.apple.Translate',
188
- 音乐: 'com.apple.Music',
189
- 播客: 'com.apple.podcasts',
190
- 库乐队: 'com.apple.mobilegarageband',
191
- 语音备忘录: 'com.apple.VoiceMemos',
192
- iMovie: 'com.apple.iMovie',
193
- Watch: 'com.apple.Bridge',
194
- 'Apple Store': 'com.apple.store.Jolly',
195
- TestFlight: 'com.apple.TestFlight',
196
- Keynote: 'com.apple.Keynote',
197
- 'Keynote 讲演': 'com.apple.Keynote'
198
- };
199
- const debugIOS = getDebug('webdriver:ios');
200
- const WDA_MJPEG_SCREENSHOT_QUALITY = 50;
201
- const WDA_MJPEG_FRAMERATE = 30;
202
- const WDA_MJPEG_SCALING_FACTOR = 50;
203
- class IOSWebDriverClient extends WebDriverClient {
204
- async launchApp(bundleId) {
205
- this.ensureSession();
206
- try {
207
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/apps/launch`, {
208
- bundleId
209
- });
210
- debugIOS(`Launched app: ${bundleId}`);
211
- } catch (error) {
212
- debugIOS(`Failed to launch app ${bundleId}: ${error}`);
213
- throw error;
214
- }
215
- }
216
- async activateApp(bundleId) {
217
- this.ensureSession();
218
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/apps/activate`, {
219
- bundleId
220
- });
221
- }
222
- async terminateApp(bundleId) {
223
- this.ensureSession();
224
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/apps/terminate`, {
225
- bundleId
226
- });
227
- }
228
- async openUrl(url) {
229
- this.ensureSession();
230
- try {
231
- await this.makeRequest('POST', `/session/${this.sessionId}/url`, {
232
- url
233
- });
234
- } catch (error) {
235
- debugIOS(`Direct URL opening failed, trying Safari fallback: ${error}`);
236
- await this.launchApp('com.apple.mobilesafari');
237
- await new Promise((resolve)=>setTimeout(resolve, 2000));
238
- await this.makeRequest('POST', `/session/${this.sessionId}/url`, {
239
- url
240
- });
241
- }
242
- }
243
- async pressHomeButton() {
244
- this.ensureSession();
245
- try {
246
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/pressButton`, {
247
- name: 'home'
248
- });
249
- debugIOS('Home button pressed using hardware key');
250
- } catch (error) {
251
- debugIOS(`Failed to press home button: ${error}`);
252
- throw new Error(`Failed to press home button: ${error}`);
253
- }
254
- }
255
- async appSwitcher() {
256
- this.ensureSession();
257
- try {
258
- const windowSize = await this.getWindowSize();
259
- debugIOS('Triggering app switcher with slow swipe up gesture');
260
- const centerX = windowSize.width / 2;
261
- const startY = windowSize.height - 5;
262
- const endY = 0.5 * windowSize.height;
263
- await this.swipe(centerX, startY, centerX, endY, 1500);
264
- await new Promise((resolve)=>setTimeout(resolve, 800));
265
- } catch (error) {
266
- debugIOS(`App switcher failed: ${error}`);
267
- throw new Error(`Failed to trigger app switcher: ${error}`);
268
- }
269
- }
270
- async pressKey(key) {
271
- this.ensureSession();
272
- debugIOS(`Attempting to press key: ${key}`);
273
- if ('Enter' === key || 'Return' === key || 'return' === key) {
274
- debugIOS('Handling Enter/Return key for iOS');
275
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/keys`, {
276
- value: [
277
- '\n'
278
- ]
279
- });
280
- debugIOS('Sent newline character for Enter key');
281
- await new Promise((resolve)=>setTimeout(resolve, 100));
282
- return;
283
- }
284
- if ('Backspace' === key || 'Delete' === key) try {
285
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/keys`, {
286
- value: [
287
- '\b'
288
- ]
289
- });
290
- debugIOS('Sent backspace character');
291
- return;
292
- } catch (error) {
293
- debugIOS(`Backspace failed: ${error}`);
294
- }
295
- if ('Space' === key) try {
296
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/keys`, {
297
- value: [
298
- ' '
299
- ]
300
- });
301
- debugIOS('Sent space character');
302
- return;
303
- } catch (error) {
304
- debugIOS(`Space key failed: ${error}`);
305
- }
306
- const normalizedKey = this.normalizeKeyName(key);
307
- const iosKeyMap = {
308
- Tab: '\t',
309
- ArrowUp: '\uE013',
310
- ArrowDown: '\uE015',
311
- ArrowLeft: '\uE012',
312
- ArrowRight: '\uE014',
313
- Home: '\uE011',
314
- End: '\uE010'
315
- };
316
- if (iosKeyMap[normalizedKey]) try {
317
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/keys`, {
318
- value: [
319
- iosKeyMap[normalizedKey]
320
- ]
321
- });
322
- debugIOS(`Sent WebDriver key code for: ${key}`);
323
- return;
324
- } catch (error) {
325
- debugIOS(`WebDriver key failed for "${key}": ${error}`);
326
- }
327
- if (1 === key.length) try {
328
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/keys`, {
329
- value: [
330
- key
331
- ]
332
- });
333
- debugIOS(`Sent single character: "${key}"`);
334
- return;
335
- } catch (error) {
336
- debugIOS(`Failed to send character "${key}": ${error}`);
337
- }
338
- debugIOS(`Warning: Key "${key}" is not supported on iOS platform`);
339
- throw new Error(`Key "${key}" is not supported on iOS platform`);
340
- }
341
- async getActiveElement() {
342
- this.ensureSession();
343
- debugIOS('Getting active element');
344
- try {
345
- const response = await this.makeRequest('GET', `/session/${this.sessionId}/element/active`);
346
- const elementId = response.value?.ELEMENT || response.value?.['element-6066-11e4-a52e-4f735466cecf'] || response.ELEMENT || response['element-6066-11e4-a52e-4f735466cecf'];
347
- if (elementId) {
348
- debugIOS(`Got active element ID: ${elementId}`);
349
- return elementId;
350
- }
351
- debugIOS('No active element found');
352
- return null;
353
- } catch (error) {
354
- debugIOS(`Failed to get active element: ${error}`);
355
- return null;
356
- }
357
- }
358
- async clearElement(elementId) {
359
- this.ensureSession();
360
- debugIOS(`Clearing element: ${elementId}`);
361
- try {
362
- await this.makeRequest('POST', `/session/${this.sessionId}/element/${elementId}/clear`);
363
- debugIOS('Element cleared successfully');
364
- } catch (error) {
365
- debugIOS(`Failed to clear element: ${error}`);
366
- throw new Error(`Failed to clear element: ${error}`);
367
- }
368
- }
369
- async clearActiveElement() {
370
- try {
371
- const elementId = await this.getActiveElement();
372
- if (!elementId) {
373
- debugIOS('No active element to clear');
374
- return false;
375
- }
376
- await this.clearElement(elementId);
377
- return true;
378
- } catch (error) {
379
- debugIOS(`Failed to clear active element: ${error}`);
380
- return false;
381
- }
382
- }
383
- normalizeKeyName(key) {
384
- return key.charAt(0).toUpperCase() + key.slice(1).toLowerCase();
385
- }
386
- async dismissKeyboard(keyNames) {
387
- this.ensureSession();
388
- try {
389
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/keyboard/dismiss`, {
390
- keyNames: keyNames || [
391
- 'done'
392
- ]
393
- });
394
- debugIOS('Dismissed keyboard using WDA API');
395
- return true;
396
- } catch (error) {
397
- debugIOS(`Failed to dismiss keyboard: ${error}`);
398
- return false;
399
- }
400
- }
401
- async typeText(text) {
402
- this.ensureSession();
403
- try {
404
- const cleanText = text.trim();
405
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/keys`, {
406
- value: cleanText.split('')
407
- });
408
- debugIOS(`Typed text: "${text}"`);
409
- } catch (error) {
410
- debugIOS(`Failed to type text "${text}": ${error}`);
411
- throw new Error(`Failed to type text: ${error}`);
412
- }
413
- }
414
- async tap(x, y) {
415
- this.ensureSession();
416
- try {
417
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/tap`, {
418
- x,
419
- y
420
- });
421
- debugIOS(`Tapped at coordinates (${x}, ${y})`);
422
- } catch (error) {
423
- debugIOS(`New tap endpoint failed, trying legacy endpoint: ${error}`);
424
- try {
425
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/tap/0`, {
426
- x,
427
- y
428
- });
429
- debugIOS(`Tapped at coordinates (${x}, ${y}) using legacy endpoint`);
430
- } catch (fallbackError) {
431
- debugIOS(`Failed to tap at (${x}, ${y}): ${fallbackError}`);
432
- throw new Error(`Failed to tap at coordinates: ${fallbackError}`);
433
- }
434
- }
435
- }
436
- async swipe(fromX, fromY, toX, toY, duration = 500) {
437
- this.ensureSession();
438
- const actions = {
439
- actions: [
440
- {
441
- type: 'pointer',
442
- id: 'finger1',
443
- parameters: {
444
- pointerType: 'touch'
445
- },
446
- actions: [
447
- {
448
- type: 'pointerMove',
449
- duration: 0,
450
- x: fromX,
451
- y: fromY
452
- },
453
- {
454
- type: 'pointerDown',
455
- button: 0
456
- },
457
- {
458
- type: 'pause',
459
- duration: 100
460
- },
461
- {
462
- type: 'pointerMove',
463
- duration,
464
- x: toX,
465
- y: toY
466
- },
467
- {
468
- type: 'pointerUp',
469
- button: 0
470
- }
471
- ]
472
- }
473
- ]
474
- };
475
- await this.makeRequest('POST', `/session/${this.sessionId}/actions`, actions);
476
- debugIOS(`Swiped using W3C Actions from (${fromX}, ${fromY}) to (${toX}, ${toY}) in ${duration}ms`);
477
- }
478
- async pinch(centerX, centerY, startDistance, endDistance, duration = 500) {
479
- this.ensureSession();
480
- const halfStart = startDistance / 2;
481
- const halfEnd = endDistance / 2;
482
- const actions = {
483
- actions: [
484
- {
485
- type: 'pointer',
486
- id: 'finger1',
487
- parameters: {
488
- pointerType: 'touch'
489
- },
490
- actions: [
491
- {
492
- type: 'pointerMove',
493
- duration: 0,
494
- x: centerX,
495
- y: Math.round(centerY - halfStart)
496
- },
497
- {
498
- type: 'pointerDown',
499
- button: 0
500
- },
501
- {
502
- type: 'pause',
503
- duration: 100
504
- },
505
- {
506
- type: 'pointerMove',
507
- duration,
508
- x: centerX,
509
- y: Math.round(centerY - halfEnd)
510
- },
511
- {
512
- type: 'pointerUp',
513
- button: 0
514
- }
515
- ]
516
- },
517
- {
518
- type: 'pointer',
519
- id: 'finger2',
520
- parameters: {
521
- pointerType: 'touch'
522
- },
523
- actions: [
524
- {
525
- type: 'pointerMove',
526
- duration: 0,
527
- x: centerX,
528
- y: Math.round(centerY + halfStart)
529
- },
530
- {
531
- type: 'pointerDown',
532
- button: 0
533
- },
534
- {
535
- type: 'pause',
536
- duration: 100
537
- },
538
- {
539
- type: 'pointerMove',
540
- duration,
541
- x: centerX,
542
- y: Math.round(centerY + halfEnd)
543
- },
544
- {
545
- type: 'pointerUp',
546
- button: 0
547
- }
548
- ]
549
- }
550
- ]
551
- };
552
- await this.makeRequest('POST', `/session/${this.sessionId}/actions`, actions);
553
- debugIOS(`Pinched at (${centerX}, ${centerY}) from distance ${startDistance} to ${endDistance} in ${duration}ms`);
554
- }
555
- async longPress(x, y, duration = 1000) {
556
- this.ensureSession();
557
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/touchAndHold`, {
558
- x,
559
- y,
560
- duration: duration / 1000
561
- });
562
- debugIOS(`Long pressed at coordinates (${x}, ${y}) for ${duration}ms`);
563
- }
564
- async doubleTap(x, y) {
565
- this.ensureSession();
566
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/doubleTap`, {
567
- x,
568
- y
569
- });
570
- debugIOS(`Double tapped at coordinates (${x}, ${y})`);
571
- }
572
- async tripleTap(x, y) {
573
- this.ensureSession();
574
- await this.makeRequest('POST', `/session/${this.sessionId}/wda/tapWithNumberOfTaps`, {
575
- x,
576
- y,
577
- numberOfTaps: 3,
578
- numberOfTouches: 1
579
- });
580
- debugIOS(`Triple tapped at coordinates (${x}, ${y})`);
581
- }
582
- async getScreenScale() {
583
- this.ensureSession();
584
- try {
585
- const screenResponse = await this.makeRequest('GET', `/session/${this.sessionId}/wda/screen`);
586
- if (screenResponse?.value?.scale) {
587
- debugIOS(`Got screen scale from WDA screen endpoint: ${screenResponse.value.scale}`);
588
- return screenResponse.value.scale;
589
- }
590
- } catch (error) {
591
- debugIOS(`Failed to get screen scale from /wda/screen: ${error}`);
592
- }
593
- try {
594
- debugIOS('Calculating screen scale from screenshot and window size');
595
- const [screenshotBase64, windowSize] = await Promise.all([
596
- this.takeScreenshot(),
597
- this.getWindowSize()
598
- ]);
599
- const { imageInfoOfBase64 } = await import("@midscene/shared/img");
600
- const { width: screenshotWidth, height: screenshotHeight } = await imageInfoOfBase64(screenshotBase64);
601
- const scale = Math.max(screenshotWidth, screenshotHeight) / Math.max(windowSize.width, windowSize.height);
602
- const roundedScale = Math.round(scale);
603
- debugIOS(`Calculated screen scale: ${roundedScale} (screenshot: ${screenshotWidth}x${screenshotHeight}, window: ${windowSize.width}x${windowSize.height})`);
604
- return roundedScale;
605
- } catch (error) {
606
- debugIOS(`Failed to calculate screen scale: ${error}`);
607
- }
608
- debugIOS('No screen scale found');
609
- return null;
610
- }
611
- async createSession(capabilities) {
612
- const defaultCapabilities = {
613
- platformName: 'iOS',
614
- automationName: 'XCUITest',
615
- shouldUseSingletonTestManager: false,
616
- shouldUseTestManagerForVisibilityDetection: false,
617
- ...capabilities
618
- };
619
- const session = await super.createSession(defaultCapabilities);
620
- await this.setupIOSSession();
621
- return session;
622
- }
623
- async setupIOSSession() {
624
- if (!this.sessionId) return;
625
- try {
626
- await this.makeRequest('POST', `/session/${this.sessionId}/appium/settings`, {
627
- snapshotMaxDepth: 50,
628
- elementResponseAttributes: 'type,label,name,value,rect,enabled,visible',
629
- mjpegServerScreenshotQuality: WDA_MJPEG_SCREENSHOT_QUALITY,
630
- mjpegServerFramerate: WDA_MJPEG_FRAMERATE,
631
- mjpegScalingFactor: WDA_MJPEG_SCALING_FACTOR
632
- });
633
- debugIOS('iOS session configuration applied (including MJPEG settings)');
634
- } catch (error) {
635
- debugIOS(`Failed to apply iOS session configuration: ${error}`);
636
- }
637
- }
638
- async setupExistingSession() {
639
- this.ensureSession();
640
- await this.setupIOSSession();
641
- }
642
- async executeRequest(method, endpoint, data) {
643
- return this.makeRequest(method, this.buildSessionEndpoint(endpoint), data);
644
- }
645
- }
646
- function _define_property(obj, key, value) {
647
- if (key in obj) Object.defineProperty(obj, key, {
648
- value: value,
649
- enumerable: true,
650
- configurable: true,
651
- writable: true
652
- });
653
- else obj[key] = value;
654
- return obj;
655
- }
656
- const debugDevice = getDebug('ios:device');
657
- const WDA_HTTP_METHODS = [
658
- 'GET',
659
- 'POST',
660
- 'DELETE',
661
- 'PUT'
662
- ];
663
- const DEFAULT_WDA_MJPEG_PORT = 9100;
664
- class IOSDevice {
665
- async tapPoint(point) {
666
- debugDevice(`tap at coordinates (${point.x}, ${point.y})`);
667
- await this.wdaBackend.tap(Math.round(point.x), Math.round(point.y));
668
- }
669
- async doubleTapPoint(point) {
670
- await this.wdaBackend.doubleTap(Math.round(point.x), Math.round(point.y));
671
- }
672
- async longPressPoint(point, duration = 1000) {
673
- await this.wdaBackend.longPress(Math.round(point.x), Math.round(point.y), duration);
674
- }
675
- async swipePoint(start, end, duration = 500) {
676
- await this.wdaBackend.swipe(Math.round(start.x), Math.round(start.y), Math.round(end.x), Math.round(end.y), duration);
677
- }
678
- async clearInputAt(point) {
679
- if (point) {
680
- await this.tapPoint(point);
681
- await sleep(100);
682
- }
683
- debugDevice('Attempting to clear input with WebDriver Clear API');
684
- const cleared = await this.wdaBackend.clearActiveElement();
685
- cleared ? debugDevice('Successfully cleared input with WebDriver Clear API') : debugDevice('WebDriver Clear API returned false (no active element or clear failed)');
686
- }
687
- actionSpace() {
688
- const mobileActionContext = {
689
- input: this.inputPrimitives,
690
- size: ()=>this.size(),
691
- sleep: async (timeMs)=>{
692
- await sleep(timeMs);
693
- },
694
- getDefaultAutoDismissKeyboard: ()=>this.options?.autoDismissKeyboard
695
- };
696
- const defaultActions = [
697
- ...createDefaultMobileActions(mobileActionContext)
698
- ];
699
- const platformSpecificActions = Object.values(createPlatformActions(this));
700
- const customActions = this.customActions || [];
701
- return [
702
- ...defaultActions,
703
- ...platformSpecificActions,
704
- ...customActions
705
- ];
706
- }
707
- async performActionScroll(param) {
708
- const element = param.locate;
709
- const startingPoint = element ? {
710
- left: element.center[0],
711
- top: element.center[1]
712
- } : void 0;
713
- const scrollToEventName = param?.scrollType;
714
- if ('scrollToTop' === scrollToEventName) await this.scrollUntilTop(startingPoint);
715
- else if ('scrollToBottom' === scrollToEventName) await this.scrollUntilBottom(startingPoint);
716
- else if ('scrollToRight' === scrollToEventName) await this.scrollUntilRight(startingPoint);
717
- else if ('scrollToLeft' === scrollToEventName) await this.scrollUntilLeft(startingPoint);
718
- else if ('singleAction' !== scrollToEventName && scrollToEventName) throw new Error(`Unknown scroll event type: ${scrollToEventName}, param: ${JSON.stringify(param)}`);
719
- else {
720
- if (param?.direction !== 'down' && param && param.direction) if ('up' === param.direction) await this.scrollUp(param.distance || void 0, startingPoint);
721
- else if ('left' === param.direction) await this.scrollLeft(param.distance || void 0, startingPoint);
722
- else if ('right' === param.direction) await this.scrollRight(param.distance || void 0, startingPoint);
723
- else throw new Error(`Unknown scroll direction: ${param.direction}`);
724
- else await this.scrollDown(param?.distance || void 0, startingPoint);
725
- await sleep(500);
726
- }
727
- }
728
- describe() {
729
- return this.description || `Device ID: ${this.deviceId}`;
730
- }
731
- async getConnectedDeviceInfo() {
732
- return await this.wdaBackend.getDeviceInfo();
733
- }
734
- async connect() {
735
- node_assert(!this.destroyed, `IOSDevice ${this.deviceId} has been destroyed and cannot execute commands`);
736
- debugDevice(`Connecting to iOS device: ${this.deviceId}`);
737
- try {
738
- await this.wdaManager.start();
739
- if (this.options?.sessionId) {
740
- debugDevice(`Using existing WDA session: ${this.options.sessionId}`);
741
- await this.wdaBackend.setupExistingSession();
742
- } else await this.wdaBackend.createSession();
743
- const deviceInfo = await this.wdaBackend.getDeviceInfo();
744
- if (deviceInfo?.udid) {
745
- this.deviceId = deviceInfo.udid;
746
- debugDevice(`Updated device ID to real UDID: ${this.deviceId}`);
747
- }
748
- const size = await this.getScreenSize();
749
- this.description = `
750
- UDID: ${this.deviceId}${deviceInfo ? `
751
- Name: ${deviceInfo.name}
752
- Model: ${deviceInfo.model}` : ''}
753
- Type: WebDriverAgent
754
- ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
755
- `;
756
- debugDevice('iOS device connected successfully', this.description);
757
- } catch (e) {
758
- debugDevice(`Failed to connect to iOS device: ${e}`);
759
- throw new Error(`Unable to connect to iOS device ${this.deviceId}: ${e}`);
760
- }
761
- }
762
- setAppNameMapping(mapping) {
763
- this.appNameMapping = mapping;
764
- }
765
- resolveBundleId(appName) {
766
- const normalizedAppName = normalizeForComparison(appName);
767
- return this.appNameMapping[normalizedAppName];
768
- }
769
- async launch(uri) {
770
- this.uri = uri;
771
- try {
772
- debugDevice(`Launching app: ${uri}`);
773
- if (uri.startsWith('http://') || uri.startsWith('https://') || uri.includes('://')) await this.openUrl(uri);
774
- else {
775
- const resolvedUri = this.resolveBundleId(uri) ?? uri;
776
- await this.wdaBackend.launchApp(resolvedUri);
777
- }
778
- debugDevice(`Successfully launched: ${uri}`);
779
- } catch (error) {
780
- debugDevice(`Error launching ${uri}: ${error}`);
781
- throw new Error(`Failed to launch ${uri}: ${error.message}`);
782
- }
783
- return this;
784
- }
785
- async terminate(bundleId) {
786
- const resolved = this.resolveBundleId(bundleId) ?? bundleId;
787
- try {
788
- debugDevice(`Terminating app: ${resolved}`);
789
- await this.wdaBackend.terminateApp(resolved);
790
- debugDevice(`Successfully terminated: ${resolved}`);
791
- } catch (error) {
792
- debugDevice(`Error terminating ${resolved}: ${error}`);
793
- throw new Error(`Failed to terminate ${resolved}: ${error.message}`);
794
- }
795
- }
796
- async getElementsInfo() {
797
- return [];
798
- }
799
- async getElementsNodeTree() {
800
- return {
801
- node: null,
802
- children: []
803
- };
804
- }
805
- async initializeDevicePixelRatio() {
806
- if (this.devicePixelRatioInitialized) return;
807
- const apiScale = await this.wdaBackend.getScreenScale();
808
- node_assert(apiScale && apiScale > 0, 'Failed to get device pixel ratio from WebDriverAgent API');
809
- debugDevice(`Got screen scale from WebDriverAgent API: ${apiScale}`);
810
- this.devicePixelRatio = apiScale;
811
- this.devicePixelRatioInitialized = true;
812
- }
813
- async getScreenSize() {
814
- await this.initializeDevicePixelRatio();
815
- const windowSize = await this.wdaBackend.getWindowSize();
816
- return {
817
- width: windowSize.width,
818
- height: windowSize.height,
819
- scale: this.devicePixelRatio
820
- };
821
- }
822
- async size() {
823
- const screenSize = await this.getScreenSize();
824
- return {
825
- width: screenSize.width,
826
- height: screenSize.height
827
- };
828
- }
829
- async screenshotBase64() {
830
- debugDevice('Taking screenshot via WDA');
831
- try {
832
- const base64Data = await this.wdaBackend.takeScreenshot();
833
- const result = createImgBase64ByFormat('png', base64Data);
834
- debugDevice('Screenshot taken successfully');
835
- return result;
836
- } catch (error) {
837
- debugDevice(`Screenshot failed: ${error}`);
838
- throw new Error(`Failed to take screenshot: ${error}`);
839
- }
840
- }
841
- async clearInput(element) {
842
- await this.clearInputAt(element ? {
843
- x: element.center[0],
844
- y: element.center[1]
845
- } : void 0);
846
- }
847
- async url() {
848
- return '';
849
- }
850
- async tap(x, y) {
851
- await this.tapPoint({
852
- x,
853
- y
854
- });
855
- }
856
- async swipe(fromX, fromY, toX, toY, duration = 500) {
857
- await this.swipeCoordinates(fromX, fromY, toX, toY, duration);
858
- }
859
- async swipeCoordinates(fromX, fromY, toX, toY, duration = 500) {
860
- await this.swipePoint({
861
- x: fromX,
862
- y: fromY
863
- }, {
864
- x: toX,
865
- y: toY
866
- }, duration);
867
- }
868
- async typeText(text, options) {
869
- if (!text) return;
870
- const shouldAutoDismissKeyboard = options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;
871
- debugDevice(`Typing text: "${text}"`);
872
- try {
873
- await sleep(200);
874
- await this.wdaBackend.typeText(text);
875
- await sleep(300);
876
- } catch (error) {
877
- debugDevice(`Failed to type text with WDA: ${error}`);
878
- throw error;
879
- }
880
- if (shouldAutoDismissKeyboard) await this.hideKeyboard();
881
- }
882
- async pressKey(key) {
883
- await this.wdaBackend.pressKey(key);
884
- }
885
- async scrollUp(distance, startPoint) {
886
- const { width, height } = await this.size();
887
- const start = startPoint ? {
888
- x: Math.round(startPoint.left),
889
- y: Math.round(startPoint.top)
890
- } : {
891
- x: Math.round(width / 2),
892
- y: Math.round(height / 2)
893
- };
894
- const scrollDistance = Math.round(distance || height / 3);
895
- await this.swipeCoordinates(start.x, start.y, start.x, start.y + scrollDistance);
896
- }
897
- async scrollDown(distance, startPoint) {
898
- const { width, height } = await this.size();
899
- const start = startPoint ? {
900
- x: Math.round(startPoint.left),
901
- y: Math.round(startPoint.top)
902
- } : {
903
- x: Math.round(width / 2),
904
- y: Math.round(height / 2)
905
- };
906
- const scrollDistance = Math.round(distance || height / 3);
907
- await this.swipeCoordinates(start.x, start.y, start.x, start.y - scrollDistance);
908
- }
909
- async scrollLeft(distance, startPoint) {
910
- const { width, height } = await this.size();
911
- const start = startPoint ? {
912
- x: Math.round(startPoint.left),
913
- y: Math.round(startPoint.top)
914
- } : {
915
- x: Math.round(width / 2),
916
- y: Math.round(height / 2)
917
- };
918
- const scrollDistance = Math.round(distance || 0.7 * width);
919
- await this.swipeCoordinates(start.x, start.y, start.x + scrollDistance, start.y);
920
- }
921
- async scrollRight(distance, startPoint) {
922
- const { width, height } = await this.size();
923
- const start = startPoint ? {
924
- x: Math.round(startPoint.left),
925
- y: Math.round(startPoint.top)
926
- } : {
927
- x: Math.round(width / 2),
928
- y: Math.round(height / 2)
929
- };
930
- const scrollDistance = Math.round(distance || 0.7 * width);
931
- await this.swipeCoordinates(start.x, start.y, start.x - scrollDistance, start.y);
932
- }
933
- async scrollUntilTop(startPoint) {
934
- debugDevice('Using screenshot-based scroll detection for better reliability');
935
- await this.scrollUntilBoundary('up', startPoint, 1);
936
- }
937
- async scrollUntilBottom(startPoint) {
938
- debugDevice('Using screenshot-based scroll detection for better reliability');
939
- await this.scrollUntilBoundary('down', startPoint, 1);
940
- }
941
- compareScreenshots(screenshot1, screenshot2, tolerancePercent = 2) {
942
- if (screenshot1 === screenshot2) {
943
- debugDevice('Screenshots are identical');
944
- return true;
945
- }
946
- const len1 = screenshot1.length;
947
- const len2 = screenshot2.length;
948
- debugDevice(`Screenshots differ: length1=${len1}, length2=${len2}`);
949
- if (Math.abs(len1 - len2) > 0.1 * Math.min(len1, len2)) {
950
- debugDevice('Screenshots have significant length difference');
951
- return false;
952
- }
953
- if (len1 > 0 && len2 > 0) {
954
- const minLength = Math.min(len1, len2);
955
- const sampleSize = Math.min(2000, minLength);
956
- let diffCount = 0;
957
- for(let i = 0; i < sampleSize; i++)if (screenshot1[i] !== screenshot2[i]) diffCount++;
958
- const diffPercent = diffCount / sampleSize * 100;
959
- debugDevice(`Character differences: ${diffCount}/${sampleSize} (${diffPercent.toFixed(2)}%)`);
960
- const isSimilar = diffPercent <= tolerancePercent;
961
- if (isSimilar) debugDevice(`Screenshots are similar enough (${diffPercent.toFixed(2)}% <= ${tolerancePercent}%)`);
962
- return isSimilar;
963
- }
964
- return false;
965
- }
966
- async scrollUntilBoundary(direction, startPoint, maxUnchangedCount = 1) {
967
- const maxAttempts = 20;
968
- const { width, height } = await this.size();
969
- let start;
970
- if (startPoint) start = {
971
- x: Math.round(startPoint.left),
972
- y: Math.round(startPoint.top)
973
- };
974
- else switch(direction){
975
- case 'up':
976
- start = {
977
- x: Math.round(width / 2),
978
- y: Math.round(0.2 * height)
979
- };
980
- break;
981
- case 'down':
982
- start = {
983
- x: Math.round(width / 2),
984
- y: Math.round(0.8 * height)
985
- };
986
- break;
987
- case 'left':
988
- start = {
989
- x: Math.round(0.8 * width),
990
- y: Math.round(height / 2)
991
- };
992
- break;
993
- case 'right':
994
- start = {
995
- x: Math.round(0.2 * width),
996
- y: Math.round(height / 2)
997
- };
998
- break;
999
- }
1000
- let lastScreenshot = null;
1001
- let unchangedCount = 0;
1002
- debugDevice(`Starting scroll to ${direction} with content detection`);
1003
- for(let i = 0; i < maxAttempts; i++)try {
1004
- debugDevice(`Scroll attempt ${i + 1}/${maxAttempts}`);
1005
- await sleep(500);
1006
- const currentScreenshot = await this.screenshotBase64();
1007
- if (lastScreenshot && this.compareScreenshots(lastScreenshot, currentScreenshot, 10)) {
1008
- unchangedCount++;
1009
- debugDevice(`Screen content unchanged (${unchangedCount}/${maxUnchangedCount})`);
1010
- if (unchangedCount >= maxUnchangedCount) {
1011
- debugDevice(`Reached ${direction}: screen content no longer changes`);
1012
- break;
1013
- }
1014
- } else {
1015
- if (lastScreenshot) debugDevice(`Content changed, resetting counter (was ${unchangedCount})`);
1016
- unchangedCount = 0;
1017
- }
1018
- if (i >= 15 && 0 === unchangedCount) {
1019
- debugDevice(`Too many attempts with dynamic content, stopping scroll to ${direction}`);
1020
- break;
1021
- }
1022
- lastScreenshot = currentScreenshot;
1023
- const scrollDistance = Math.round('left' === direction || 'right' === direction ? 0.6 * width : 0.6 * height);
1024
- debugDevice(`Performing scroll: ${direction}, distance: ${scrollDistance}`);
1025
- switch(direction){
1026
- case 'up':
1027
- await this.swipeCoordinates(start.x, start.y, start.x, start.y + scrollDistance, 300);
1028
- break;
1029
- case 'down':
1030
- await this.swipeCoordinates(start.x, start.y, start.x, start.y - scrollDistance, 300);
1031
- break;
1032
- case 'left':
1033
- await this.swipeCoordinates(start.x, start.y, start.x + scrollDistance, start.y, 300);
1034
- break;
1035
- case 'right':
1036
- await this.swipeCoordinates(start.x, start.y, start.x - scrollDistance, start.y, 300);
1037
- break;
1038
- }
1039
- debugDevice('Waiting for scroll and inertia to complete...');
1040
- await sleep(2000);
1041
- } catch (error) {
1042
- debugDevice(`Error during scroll attempt ${i + 1}: ${error}`);
1043
- await sleep(300);
1044
- }
1045
- debugDevice(`Scroll to ${direction} completed after ${maxAttempts} attempts`);
1046
- }
1047
- async scrollUntilLeft(startPoint) {
1048
- await this.scrollUntilBoundary('left', startPoint, 1);
1049
- }
1050
- async scrollUntilRight(startPoint) {
1051
- await this.scrollUntilBoundary('right', startPoint, 3);
1052
- }
1053
- async home() {
1054
- await this.wdaBackend.pressHomeButton();
1055
- }
1056
- async appSwitcher() {
1057
- try {
1058
- debugDevice('Triggering app switcher with slow swipe up gesture');
1059
- const { width, height } = await this.size();
1060
- const centerX = Math.round(width / 2);
1061
- const startY = Math.round(height - 5);
1062
- const endY = Math.round(0.5 * height);
1063
- await this.wdaBackend.swipe(centerX, startY, centerX, endY, 1500);
1064
- await sleep(800);
1065
- } catch (error) {
1066
- debugDevice(`App switcher failed: ${error}`);
1067
- throw new Error(`Failed to trigger app switcher: ${error}`);
1068
- }
1069
- }
1070
- async hideKeyboard(keyNames) {
1071
- try {
1072
- const dismissKeys = keyNames && keyNames.length > 0 ? keyNames : [
1073
- 'return',
1074
- 'done',
1075
- 'go',
1076
- 'search',
1077
- 'next',
1078
- 'send'
1079
- ];
1080
- debugDevice(`Attempting to dismiss keyboard using WDA API with keys: ${dismissKeys.join(', ')}`);
1081
- try {
1082
- await this.wdaBackend.dismissKeyboard(dismissKeys);
1083
- debugDevice('Successfully dismissed keyboard using WDA API');
1084
- await sleep(500);
1085
- return true;
1086
- } catch (wdaError) {
1087
- debugDevice(`WDA dismissKeyboard failed, falling back to swipe gesture: ${wdaError}`);
1088
- }
1089
- const windowSize = await this.wdaBackend.getWindowSize();
1090
- const centerX = Math.round(windowSize.width / 2);
1091
- const startY = Math.round(0.9 * windowSize.height);
1092
- const endY = Math.round(0.5 * windowSize.height);
1093
- await this.swipeCoordinates(centerX, startY, centerX, endY, 300);
1094
- debugDevice('Dismissed keyboard with swipe up gesture from bottom of screen');
1095
- await sleep(500);
1096
- return true;
1097
- } catch (error) {
1098
- debugDevice(`Failed to hide keyboard: ${error}`);
1099
- return false;
1100
- }
1101
- }
1102
- async openUrl(url, options) {
1103
- const opts = {
1104
- useSafariAsBackup: true,
1105
- waitTime: 2000,
1106
- ...options
1107
- };
1108
- try {
1109
- debugDevice(`Opening URL: ${url}`);
1110
- await this.wdaBackend.openUrl(url);
1111
- await sleep(opts.waitTime);
1112
- debugDevice(`Successfully opened URL: ${url}`);
1113
- } catch (error) {
1114
- debugDevice(`Direct URL opening failed: ${error}`);
1115
- if (opts.useSafariAsBackup) {
1116
- debugDevice(`Attempting to open URL via Safari: ${url}`);
1117
- await this.openUrlViaSafari(url);
1118
- } else throw new Error(`Failed to open URL: ${error}`);
1119
- }
1120
- }
1121
- async openUrlViaSafari(url) {
1122
- try {
1123
- debugDevice(`Opening URL via Safari: ${url}`);
1124
- await this.wdaBackend.terminateApp('com.apple.mobilesafari');
1125
- await this.wdaBackend.launchApp('com.apple.mobilesafari');
1126
- await sleep(2000);
1127
- await this.typeText(url);
1128
- await sleep(500);
1129
- await this.pressKey('Return');
1130
- await sleep(1000);
1131
- try {
1132
- await sleep(2000);
1133
- debugDevice(`URL opened via Safari: ${url}`);
1134
- } catch (dialogError) {
1135
- debugDevice(`No confirmation dialog or dialog handling failed: ${dialogError}`);
1136
- }
1137
- } catch (error) {
1138
- debugDevice(`Failed to open URL via Safari: ${error}`);
1139
- throw new Error(`Failed to open URL via Safari: ${error}`);
1140
- }
1141
- }
1142
- async runWdaRequest(method, endpoint, data) {
1143
- return await this.wdaBackend.executeRequest(method, endpoint, data);
1144
- }
1145
- async destroy() {
1146
- if (this.destroyed) return;
1147
- try {
1148
- await this.wdaBackend.deleteSession();
1149
- await this.wdaManager.stop();
1150
- } catch (error) {
1151
- debugDevice(`Error during cleanup: ${error}`);
1152
- }
1153
- this.destroyed = true;
1154
- debugDevice(`iOS device ${this.deviceId} destroyed`);
1155
- }
1156
- constructor(options){
1157
- _define_property(this, "deviceId", void 0);
1158
- _define_property(this, "devicePixelRatio", 1);
1159
- _define_property(this, "devicePixelRatioInitialized", false);
1160
- _define_property(this, "destroyed", false);
1161
- _define_property(this, "description", void 0);
1162
- _define_property(this, "customActions", void 0);
1163
- _define_property(this, "wdaBackend", void 0);
1164
- _define_property(this, "wdaManager", void 0);
1165
- _define_property(this, "mjpegStreamUrl", void 0);
1166
- _define_property(this, "appNameMapping", {});
1167
- _define_property(this, "interfaceType", 'ios');
1168
- _define_property(this, "uri", void 0);
1169
- _define_property(this, "options", void 0);
1170
- _define_property(this, "inputPrimitives", {
1171
- pointer: {
1172
- tap: (point)=>this.tapPoint(point),
1173
- doubleClick: (point)=>this.doubleTapPoint(point),
1174
- longPress: (point, opts)=>this.longPressPoint(point, opts?.duration),
1175
- dragAndDrop: (from, to)=>this.swipePoint(from, to, 1000)
1176
- },
1177
- keyboard: {
1178
- keyboardPress: (keyName)=>this.pressKey(keyName),
1179
- typeText: async (value, opts)=>{
1180
- const target = opts?.target;
1181
- if (target && opts?.replace !== false) await this.clearInput(target);
1182
- else if (target) await this.tapPoint({
1183
- x: target.center[0],
1184
- y: target.center[1]
1185
- });
1186
- if (opts?.focusOnly) return;
1187
- await this.typeText(value, opts);
1188
- },
1189
- clearInput: (target)=>this.clearInput(target),
1190
- cursorMove: async (direction, times = 1)=>{
1191
- const arrowKey = 'left' === direction ? 'ArrowLeft' : 'ArrowRight';
1192
- for(let i = 0; i < times; i++)await this.pressKey(arrowKey);
1193
- }
1194
- },
1195
- touch: {
1196
- swipe: async (start, end, opts)=>{
1197
- const duration = opts?.duration ?? 300;
1198
- const repeat = opts?.repeat ?? 1;
1199
- for(let i = 0; i < repeat; i++)await this.swipePoint(start, end, duration);
1200
- },
1201
- pinch: async (center, opts)=>{
1202
- await this.wdaBackend.pinch(Math.round(center.x), Math.round(center.y), opts.startDistance, opts.endDistance, opts.duration);
1203
- }
1204
- },
1205
- scroll: {
1206
- scroll: (param)=>this.performActionScroll(param)
1207
- }
1208
- });
1209
- this.deviceId = 'pending-connection';
1210
- this.options = options;
1211
- this.customActions = options?.customActions;
1212
- const wdaPort = options?.wdaPort || DEFAULT_WDA_PORT;
1213
- const wdaHost = options?.wdaHost || 'localhost';
1214
- const mjpegPort = options?.wdaMjpegPort ?? DEFAULT_WDA_MJPEG_PORT;
1215
- this.wdaBackend = new IOSWebDriverClient({
1216
- port: wdaPort,
1217
- host: wdaHost,
1218
- ...options?.sessionId ? {
1219
- sessionId: options.sessionId
1220
- } : {}
1221
- });
1222
- this.wdaManager = WDAManager.getInstance(wdaPort, wdaHost);
1223
- this.mjpegStreamUrl = `http://${wdaHost}:${mjpegPort}`;
1224
- }
1225
- }
1226
- const runWdaRequestParamSchema = z.object({
1227
- method: z["enum"](WDA_HTTP_METHODS).describe('HTTP method (GET, POST, DELETE, PUT)'),
1228
- endpoint: z.string().describe('WebDriver API endpoint'),
1229
- data: z.object({}).passthrough().optional().describe('Optional request body data as JSON object')
1230
- });
1231
- const launchParamSchema = z.object({
1232
- uri: z.string().describe('App name, bundle ID, or URL to launch. Prioritize using the exact bundle ID or URL the user has provided. If none provided, use the accurate app name.')
1233
- });
1234
- const terminateParamSchema = z.object({
1235
- uri: z.string().describe('Bundle ID of the app to terminate (close). Use the exact bundle ID, e.g. com.apple.Preferences.')
1236
- });
1237
- const createPlatformActions = (device)=>({
1238
- RunWdaRequest: defineAction({
1239
- name: 'RunWdaRequest',
1240
- description: 'Execute WebDriverAgent API request directly on iOS device',
1241
- interfaceAlias: 'runWdaRequest',
1242
- paramSchema: runWdaRequestParamSchema,
1243
- sample: {
1244
- method: 'GET',
1245
- endpoint: '/status'
1246
- },
1247
- call: async (param)=>await device.runWdaRequest(param.method, param.endpoint, param.data)
1248
- }),
1249
- Launch: defineAction({
1250
- name: 'Launch',
1251
- description: 'Launch an iOS app or URL',
1252
- interfaceAlias: 'launch',
1253
- paramSchema: launchParamSchema,
1254
- sample: {
1255
- uri: 'com.apple.Preferences'
1256
- },
1257
- call: async (param)=>{
1258
- if (!param.uri || '' === param.uri.trim()) throw new Error('Launch requires a non-empty uri parameter');
1259
- await device.launch(param.uri);
1260
- }
1261
- }),
1262
- Terminate: defineAction({
1263
- name: 'Terminate',
1264
- description: 'Terminate (close) an iOS app by its bundle ID',
1265
- interfaceAlias: 'terminate',
1266
- paramSchema: terminateParamSchema,
1267
- sample: {
1268
- uri: 'com.apple.Preferences'
1269
- },
1270
- call: async (param)=>{
1271
- if (!param.uri || '' === param.uri.trim()) throw new Error('Terminate requires a non-empty uri parameter');
1272
- await device.terminate(param.uri);
1273
- }
1274
- }),
1275
- IOSHomeButton: defineAction({
1276
- name: 'IOSHomeButton',
1277
- description: 'Trigger the system "home" operation on iOS devices',
1278
- call: async ()=>{
1279
- await device.home();
1280
- }
1281
- }),
1282
- IOSAppSwitcher: defineAction({
1283
- name: 'IOSAppSwitcher',
1284
- description: 'Trigger the system "app switcher" operation on iOS devices',
1285
- call: async ()=>{
1286
- await device.appSwitcher();
1287
- }
1288
- })
1289
- });
1290
- function agent_define_property(obj, key, value) {
1291
- if (key in obj) Object.defineProperty(obj, key, {
1292
- value: value,
1293
- enumerable: true,
1294
- configurable: true,
1295
- writable: true
1296
- });
1297
- else obj[key] = value;
1298
- return obj;
1299
- }
1300
- const debugAgent = getDebug('ios:agent');
1301
- class IOSAgent extends Agent {
1302
- async launch(uri) {
1303
- const action = this.wrapActionInActionSpace('Launch');
1304
- return action({
1305
- uri
1306
- });
1307
- }
1308
- async terminate(uri) {
1309
- const action = this.wrapActionInActionSpace('Terminate');
1310
- return action({
1311
- uri
1312
- });
1313
- }
1314
- createActionWrapper(name) {
1315
- const action = this.wrapActionInActionSpace(name);
1316
- return (...args)=>action(args[0]);
1317
- }
1318
- constructor(device, opts){
1319
- super(device, opts), agent_define_property(this, "runWdaRequest", void 0), agent_define_property(this, "home", void 0), agent_define_property(this, "appSwitcher", void 0), agent_define_property(this, "appNameMapping", void 0);
1320
- this.appNameMapping = mergeAndNormalizeAppNameMapping(defaultAppNameMapping, opts?.appNameMapping);
1321
- device.setAppNameMapping(this.appNameMapping);
1322
- this.runWdaRequest = this.createActionWrapper('RunWdaRequest');
1323
- this.home = this.createActionWrapper('IOSHomeButton');
1324
- this.appSwitcher = this.createActionWrapper('IOSAppSwitcher');
1325
- }
1326
- }
1327
- async function agentFromWebDriverAgent(opts) {
1328
- debugAgent('Creating iOS agent with WebDriverAgent');
1329
- const overrideModule = opts?.iOSDeviceClassOverride?.trim() || process.env[MIDSCENE_IOS_DEVICE_CLASS_OVERRIDE]?.trim();
1330
- let DeviceClass = IOSDevice;
1331
- if (overrideModule) try {
1332
- const overrideExports = await import(overrideModule);
1333
- const overrideDeviceClass = Object.prototype.hasOwnProperty.call(overrideExports, 'IOSDevice') ? overrideExports.IOSDevice : overrideExports.default;
1334
- if ('function' != typeof overrideDeviceClass) throw new Error(`Module "${overrideModule}" does not export a valid iOS device class (expected "IOSDevice" or default export).`);
1335
- DeviceClass = overrideDeviceClass;
1336
- } catch (error) {
1337
- throw new Error(`Failed to load iOS device class override from "${overrideModule}". Please make sure the package is installed and exports IOSDevice (or default) with Midscene-compatible methods. Original error: ${error instanceof Error ? error.message : String(error)}`);
1338
- }
1339
- const device = new DeviceClass(opts || {});
1340
- await device.connect();
1341
- return new IOSAgent(device, opts);
1342
- }
1343
- function mcp_tools_define_property(obj, key, value) {
1344
- if (key in obj) Object.defineProperty(obj, key, {
1345
- value: value,
1346
- enumerable: true,
1347
- configurable: true,
1348
- writable: true
1349
- });
1350
- else obj[key] = value;
1351
- return obj;
1352
- }
1353
- const debug = getDebug('mcp:ios-tools');
1354
- const iosInitArgShape = {
1355
- deviceId: z.string().optional().describe('iOS device UDID (optional when WDA auto-detect is sufficient)'),
1356
- wdaHost: z.string().optional().describe('WebDriverAgent host, defaults to localhost'),
1357
- wdaPort: z.number().optional().describe('WebDriverAgent port'),
1358
- sessionId: z.string().optional().describe('Existing WebDriverAgent session ID to reuse'),
1359
- useWDA: z.boolean().optional().describe('Whether to reuse an existing WebDriverAgent session'),
1360
- wdaMjpegPort: z.number().optional().describe('WebDriverAgent MJPEG streaming port'),
1361
- ...agentBehaviorInitArgShape
1362
- };
1363
- function getTargetIdentity(initArgs) {
1364
- if (initArgs?.deviceId) return initArgs.sessionId ? `${initArgs.deviceId}-session-${initArgs.sessionId}` : initArgs.deviceId;
1365
- if (initArgs?.wdaHost || initArgs?.wdaPort || initArgs?.sessionId) {
1366
- const wdaHost = initArgs.wdaHost ?? 'localhost';
1367
- const wdaPort = initArgs.wdaPort ?? 'default';
1368
- const sessionSegment = initArgs.sessionId ? `-session-${initArgs.sessionId}` : '';
1369
- return `wda-${wdaHost}-${wdaPort}${sessionSegment}`;
1370
- }
1371
- return 'auto';
1372
- }
1373
- class IOSMidsceneTools extends BaseMidsceneTools {
1374
- getCliReportSessionName() {
1375
- return 'midscene-ios';
1376
- }
1377
- createTemporaryDevice() {
1378
- return new IOSDevice({});
1379
- }
1380
- async ensureAgent(opts) {
1381
- const nextSignature = getAgentInitArgsSignature(opts);
1382
- if (this.agent && shouldRebuildAgentForInitArgs(this.lastOptsSignature, nextSignature)) {
1383
- try {
1384
- await this.agent.destroy?.();
1385
- } catch (error) {
1386
- debug('Failed to destroy agent during cleanup:', error);
1387
- }
1388
- this.agent = void 0;
1389
- }
1390
- if (this.agent) return this.agent;
1391
- debug('Creating iOS agent with WebDriverAgent options:', opts || {});
1392
- const reportOptions = this.readCliReportAgentOptions();
1393
- this.agent = await agentFromWebDriverAgent({
1394
- autoDismissKeyboard: false,
1395
- ...reportOptions ?? {},
1396
- ...opts ?? {}
1397
- });
1398
- this.lastOptsSignature = nextSignature;
1399
- return this.agent;
1400
- }
1401
- preparePlatformTools() {
1402
- return [
1403
- {
1404
- name: 'ios_connect',
1405
- description: 'Connect to iOS device or simulator via WebDriverAgent',
1406
- schema: this.getAgentInitArgSchema(),
1407
- cli: this.getAgentInitArgCliMetadata(),
1408
- handler: async (args)=>{
1409
- const initArgs = this.extractAgentInitParam(args);
1410
- const identity = getTargetIdentity(initArgs);
1411
- const reportSession = this.createNewCliReportSession(identity);
1412
- this.commitCliReportSession(reportSession);
1413
- if (this.agent) {
1414
- try {
1415
- await this.agent.destroy?.();
1416
- } catch (error) {
1417
- debug('Failed to destroy agent during connect:', error);
1418
- }
1419
- this.agent = void 0;
1420
- this.lastOptsSignature = void 0;
1421
- }
1422
- const agent = await this.ensureAgent(initArgs);
1423
- const screenshot = await agent.page.screenshotBase64();
1424
- return {
1425
- content: [
1426
- {
1427
- type: 'text',
1428
- text: `Connected to iOS device${initArgs?.deviceId ? `: ${initArgs.deviceId}` : ''}`
1429
- },
1430
- ...this.buildScreenshotContent(screenshot)
1431
- ],
1432
- isError: false
1433
- };
1434
- }
1435
- },
1436
- {
1437
- name: 'ios_disconnect',
1438
- description: 'Disconnect from current iOS device and release WebDriverAgent resources',
1439
- schema: {},
1440
- handler: this.createDisconnectHandler('iOS device')
1441
- }
1442
- ];
1443
- }
1444
- constructor(...args){
1445
- super(...args), mcp_tools_define_property(this, "initArgSpec", {
1446
- namespace: 'ios',
1447
- shape: iosInitArgShape,
1448
- cli: {
1449
- preferBareKeys: true
1450
- },
1451
- adapt: (extracted)=>extracted
1452
- }), mcp_tools_define_property(this, "lastOptsSignature", void 0);
1453
- }
1454
- }
1455
- class IOSMCPServer extends BaseMCPServer {
1456
- createToolsManager() {
1457
- return new IOSMidsceneTools();
1458
- }
1459
- constructor(toolsManager){
1460
- super({
1461
- name: '@midscene/ios-mcp',
1462
- version: "1.9.8",
1463
- description: 'Control the iOS device using natural language commands'
1464
- }, toolsManager);
1465
- }
1466
- }
1467
- function mcpServerForAgent(agent) {
1468
- return createMCPServerLauncher({
1469
- agent,
1470
- platformName: 'iOS',
1471
- ToolsManagerClass: IOSMidsceneTools,
1472
- MCPServerClass: IOSMCPServer
1473
- });
1474
- }
1475
- async function mcpKitForAgent(agent) {
1476
- const toolsManager = new IOSMidsceneTools();
1477
- const iosAgent = agent instanceof IOSAgent ? agent : agent;
1478
- toolsManager.setAgent(iosAgent);
1479
- await toolsManager.initTools();
1480
- return {
1481
- description: 'Midscene MCP Kit for iOS automation',
1482
- tools: toolsManager.getToolDefinitions()
1483
- };
1484
- }
1485
- export { IOSMCPServer, mcpKitForAgent, mcpServerForAgent };