devicely 2.2.13 → 2.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/bin/devicely.js +1 -1
  2. package/lib/advanced-logger.js +1 -1
  3. package/lib/androidDeviceDetection.js +1 -1
  4. package/lib/appMappings.js +1 -1
  5. package/lib/commanderService.js +1 -1
  6. package/lib/deviceDetection.js +1 -1
  7. package/lib/devices.js +1 -1
  8. package/lib/doctor.js +1 -1
  9. package/lib/encryption.js +1 -1
  10. package/lib/executor.js +1 -1
  11. package/lib/hybridAI.js +1 -1
  12. package/lib/intelligentLocatorService.js +1 -1
  13. package/lib/lightweightAI.js +1 -1
  14. package/lib/localBuiltInAI.js +1 -1
  15. package/lib/localBuiltInAI_backup.js +1 -1
  16. package/lib/localBuiltInAI_simple.js +1 -1
  17. package/lib/locatorStrategy.js +1 -1
  18. package/lib/logger-demo.js +1 -1
  19. package/lib/logger.js +1 -1
  20. package/lib/quick-start-logger.js +1 -1
  21. package/lib/scriptLoader.js +1 -1
  22. package/lib/server.js +1 -1
  23. package/lib/tensorflowAI.js +1 -1
  24. package/lib/tinyAI.js +1 -1
  25. package/lib/universalSessionManager.js +1 -1
  26. package/package.json +1 -1
  27. package/lib/.logging-backup/aiProviders.js.backup +0 -654
  28. package/lib/.logging-backup/appMappings.js.backup +0 -337
  29. package/lib/.logging-backup/commanderService.js.backup +0 -4427
  30. package/lib/.logging-backup/devices.js.backup +0 -54
  31. package/lib/.logging-backup/doctor.js.backup +0 -94
  32. package/lib/.logging-backup/encryption.js.backup +0 -61
  33. package/lib/.logging-backup/executor.js.backup +0 -104
  34. package/lib/.logging-backup/hybridAI.js.backup +0 -154
  35. package/lib/.logging-backup/intelligentLocatorService.js.backup +0 -1541
  36. package/lib/.logging-backup/locatorStrategy.js.backup +0 -342
  37. package/lib/.logging-backup/scriptLoader.js.backup +0 -13
  38. package/lib/.logging-backup/server.js.backup +0 -6298
  39. package/lib/.logging-backup/tensorflowAI.js.backup +0 -714
  40. package/lib/.logging-backup/universalSessionManager.js.backup +0 -370
  41. package/lib/.logging-enhanced-backup/server.js.enhanced-backup +0 -6298
@@ -1,4427 +0,0 @@
1
- const Logger = require('./logger');
2
- const { spawn } = require('child_process');
3
- const LocatorStrategy = require('./locatorStrategy');
4
- const IntelligentLocatorService = require('./intelligentLocatorService'); // NEW: Import intelligent locator service
5
- const universalSessionManager = require('./universalSessionManager'); // REVOLUTIONARY: Universal session management
6
- const axios = require('axios'); // Add axios at module level
7
-
8
- class CommanderService {
9
- constructor(wss, connectedDevices = []) {
10
- this.wss = wss;
11
- this.connectedDevices = connectedDevices; // Reference to global connected devices
12
- this.commanderDevice = null;
13
- this.followerDevices = new Set();
14
- this.isActive = false;
15
- this.captureProcess = null;
16
- this.logger = new Logger('CommanderService');
17
- this.eventQueue = [];
18
- this.sessionId = null;
19
- this.locatorStrategy = new LocatorStrategy();
20
-
21
- // GENIUS: Click Queue System for handling rapid clicks
22
- this.clickQueue = [];
23
- this.processingClick = false;
24
- this.maxQueueSize = 3; // Limit queue to prevent overflow
25
-
26
- // REVOLUTIONARY: Universal Session Management
27
- this.sessionManager = universalSessionManager;
28
-
29
- // 🎯 NEW: Initialize Intelligent Locator Service
30
- this.intelligentLocator = new IntelligentLocatorService(connectedDevices);
31
-
32
- // GENIUS: Inject universal session management into intelligent locator service
33
- this.intelligentLocator.getValidWDASession = this.getValidWDASession.bind(this);
34
-
35
- this.logger.info('🎯 Intelligent Locator Service initialized with Universal Session Management');
36
-
37
- // Make intelligentLocatorService globally accessible for disconnect cleanup
38
- global.intelligentLocatorService = this.intelligentLocator;
39
-
40
- // GENIUS: Use global session manager instead of local wdaSessionId
41
- this.cachedElements = []; // Cache all screen elements
42
- this.lastCacheTime = 0; // Track when cache was last updated
43
- this.isCaching = false; // Prevent duplicate caching operations
44
- this.lastWindowSize = { width: 375, height: 667 };
45
- this.commanderDriver = null; // WebDriver-like interface
46
- this.deviceSizeCache = new Map(); // Cache device screen sizes for coordinate scaling
47
-
48
- // Session operation locking to prevent concurrent access
49
- this.sessionLocks = new Map(); // Map<deviceName, Promise>
50
-
51
- // 🎯 NEW: Intelligent Locator Settings
52
- this.useIntelligentLocators = true; // Enable intelligent locator mode
53
- this.locatorSuccessRate = new Map(); // Track success rate per device
54
- this.coordinateFallbackCount = 0;
55
- this.locatorSuccessCount = 0;
56
- }
57
-
58
- // GENIUS: Get WDA session ID from universal session manager
59
- async getWDASessionId() {
60
- if (!this.commanderDevice) return null;
61
-
62
- try {
63
- // Use universal session manager to get live session
64
- const sessionId = await this.sessionManager.getLiveSessionId(this.commanderDevice.name, this.commanderDevice.port);
65
- return sessionId;
66
- } catch (error) {
67
- this.logger.warn(`⚠️ Could not get WDA session: ${error.message}`);
68
- return null;
69
- }
70
- }
71
-
72
- // Register a WDA session using universal session manager
73
- registerSessionGlobally(device, sessionId, port) {
74
- if (!device || !sessionId) return;
75
-
76
- const udid = device.udid || device.serial;
77
- const platform = device.platform || 'ios';
78
- const deviceName = device.name;
79
-
80
- // Register device port in universal session manager
81
- this.sessionManager.setDevicePort(deviceName, port);
82
- this.logger.debug(`📌 Registered port mapping in Universal Session Manager: ${deviceName} -> ${port}`);
83
-
84
- // Also store in connectedDevices for backward compatibility
85
- const globalDevice = this.connectedDevices.find(d =>
86
- d.name === deviceName || d.udid === udid
87
- );
88
-
89
- if (globalDevice) {
90
- if (platform === 'ios') {
91
- globalDevice.wdaSessionId = sessionId;
92
- } else {
93
- globalDevice.sessionId = sessionId;
94
- }
95
- globalDevice.port = port;
96
- this.logger.debug(`📌 Registered session in connectedDevices: ${deviceName}`);
97
- }
98
- }
99
-
100
- // REVOLUTIONARY: Universal Session Management - Always Live WDA Lookup
101
- async getValidWDASession(deviceName, port) {
102
- try {
103
- // Register the port mapping if provided
104
- if (port) {
105
- this.sessionManager.setDevicePort(deviceName, port);
106
- }
107
-
108
- // Get live session with retry logic
109
- const sessionId = await this.sessionManager.getSessionWithRetry(deviceName, port);
110
-
111
- this.logger.debug(`✅ Live session acquired: ${sessionId.substring(0, 8)}... for ${deviceName}`);
112
- return sessionId;
113
-
114
- } catch (error) {
115
- this.logger.error(`❌ Failed to get live session for ${deviceName}: ${error.message}`);
116
- throw error;
117
- }
118
- }
119
-
120
-
121
-
122
- async withSessionLock(deviceName, operation) {
123
- const lockKey = `${deviceName}_wda_session`;
124
-
125
- // If there's already an operation in progress, wait for it
126
- if (this.sessionLocks.has(lockKey)) {
127
- this.logger.debug(`⏳ Waiting for existing session operation on ${deviceName}...`);
128
- try {
129
- await this.sessionLocks.get(lockKey);
130
- } catch (err) {
131
- // Previous operation failed, continue with ours
132
- }
133
- }
134
-
135
- // Create new lock
136
- const operationPromise = this.executeWithLock(operation, deviceName);
137
- this.sessionLocks.set(lockKey, operationPromise);
138
-
139
- try {
140
- const result = await operationPromise;
141
- this.sessionLocks.delete(lockKey);
142
- return result;
143
- } catch (error) {
144
- this.sessionLocks.delete(lockKey);
145
- throw error;
146
- }
147
- }
148
-
149
- async executeWithLock(operation, deviceName) {
150
- this.logger.debug(`🔒 Acquired session lock for ${deviceName}`);
151
- try {
152
- const result = await operation();
153
- this.logger.debug(`🔓 Released session lock for ${deviceName}`);
154
- return result;
155
- } catch (error) {
156
- this.logger.debug(`🔓 Released session lock for ${deviceName} (error: ${error.message})`);
157
- throw error;
158
- }
159
- }
160
-
161
- // Determine if an element is likely to be interactive vs layout/informational
162
- isElementInteractive(elementType, rect, label) {
163
- // Always include actual interactive elements
164
- if (elementType.includes('Button') || elementType.includes('TextField') ||
165
- elementType.includes('Switch') || elementType.includes('Link')) {
166
- return true;
167
- }
168
-
169
- // Filter out StaticText and Cells that are likely layout elements
170
- if (elementType.includes('StaticText')) {
171
- // Include if it's a reasonable clickable size and has meaningful label
172
- if (rect.width > 40 && rect.height > 20 && label &&
173
- !label.match(/^\d+$/) && // Not just numbers
174
- !label.includes('Label') && // Not generic labels
175
- label.length > 1) {
176
- return true;
177
- }
178
- return false;
179
- }
180
-
181
- if (elementType.includes('Cell')) {
182
- // Only include cells that are reasonably sized (not massive containers)
183
- if (rect.width < 600 && rect.height < 200 && rect.height > 30) {
184
- return true;
185
- }
186
- return false;
187
- }
188
-
189
- if (elementType.includes('Image')) {
190
- // Only include images that might be clickable (reasonable size, has label)
191
- if (rect.width < 200 && rect.height < 200 && rect.width > 20 && rect.height > 20 && label) {
192
- return true;
193
- }
194
- return false;
195
- }
196
-
197
- // Exclude Other elements (containers)
198
- if (elementType.includes('Other')) {
199
- return false;
200
- }
201
-
202
- return false; // Default to exclude
203
- }
204
-
205
- // More permissive filtering for element caching (used for speed)
206
- isElementRelevant(elementType, rect, label, enabled = true) {
207
- // Always include truly interactive elements
208
- if (elementType.includes('Button') || elementType.includes('TextField') ||
209
- elementType.includes('Switch') || elementType.includes('Link')) {
210
- return enabled;
211
- }
212
-
213
- // Include StaticText if it could be clickable
214
- if (elementType.includes('StaticText')) {
215
- // More permissive - include if decent size
216
- if (rect.width > 20 && rect.height > 15 && enabled) {
217
- return true;
218
- }
219
- return false;
220
- }
221
-
222
- // Include reasonable cells
223
- if (elementType.includes('Cell')) {
224
- if (rect.width < 500 && rect.height < 150 && rect.height > 20 && enabled) {
225
- return true;
226
- }
227
- return false;
228
- }
229
-
230
- // Include images that might be clickable
231
- if (elementType.includes('Image')) {
232
- if (rect.width < 150 && rect.height < 150 && rect.width > 15 && rect.height > 15 && enabled) {
233
- return true;
234
- }
235
- return false;
236
- }
237
-
238
- // Include Other elements if they're reasonable sized (might be custom controls)
239
- if (elementType.includes('Other')) {
240
- if (rect.width > 20 && rect.height > 20 && rect.width < 300 && rect.height < 100 && enabled) {
241
- return true;
242
- }
243
- return false;
244
- }
245
-
246
- return false;
247
- }
248
-
249
- // Create a simple driver interface for app control
250
- async createDriverInterface() {
251
- if (!this.commanderDevice) {
252
- return null;
253
- }
254
-
255
- const device = this.commanderDevice;
256
-
257
- if (device.platform === 'ios') {
258
- // iOS WebDriverAgent interface
259
- const wdaSessionId = await this.getWDASessionId();
260
- if (!wdaSessionId) return null;
261
-
262
- const port = device.port || 8100;
263
- const sessionId = wdaSessionId;
264
- const baseUrl = `http://localhost:${port}`;
265
-
266
- return {
267
- execute: async (command, params = {}) => {
268
- const axios = require('axios');
269
-
270
- try {
271
- switch (command) {
272
- case 'mobile: launchApp':
273
- return await axios.post(`${baseUrl}/session/${sessionId}/wda/apps/launch`, {
274
- bundleId: params.bundleId
275
- });
276
-
277
- case 'mobile: terminateApp':
278
- return await axios.post(`${baseUrl}/session/${sessionId}/wda/apps/terminate`, {
279
- bundleId: params.bundleId
280
- });
281
-
282
- case 'mobile: activateApp':
283
- return await axios.post(`${baseUrl}/session/${sessionId}/wda/apps/activate`, {
284
- bundleId: params.bundleId
285
- });
286
-
287
- default:
288
- throw new Error(`Unsupported iOS command: ${command}`);
289
- }
290
- } catch (error) {
291
- this.logger.error(`iOS driver command failed: ${error.message}`);
292
- throw error;
293
- }
294
- },
295
-
296
- getWindowSize: async () => {
297
- return this.lastWindowSize;
298
- }
299
- };
300
- }
301
- else if (device.platform === 'android') {
302
- // Android - use ADB commands via shell
303
- return {
304
- execute: async (command, params = {}) => {
305
- const { spawn } = require('child_process');
306
-
307
- try {
308
- switch (command) {
309
- case 'mobile: launchApp':
310
- return new Promise((resolve, reject) => {
311
- const adb = spawn('adb', ['-s', device.serial || device.udid, 'shell', 'am', 'start', '-n', `${params.bundleId}/.MainActivity`]);
312
-
313
- adb.on('close', (code) => {
314
- if (code === 0) {
315
- resolve({ success: true });
316
- } else {
317
- // Try alternative launch method
318
- const adb2 = spawn('adb', ['-s', device.serial || device.udid, 'shell', 'monkey', '-p', params.bundleId, '-c', 'android.intent.category.LAUNCHER', '1']);
319
- adb2.on('close', (code2) => {
320
- resolve({ success: code2 === 0 });
321
- });
322
- adb2.on('error', () => resolve({ success: false }));
323
- }
324
- });
325
- adb.on('error', (error) => reject(error));
326
- });
327
-
328
- case 'mobile: terminateApp':
329
- return new Promise((resolve, reject) => {
330
- const adb = spawn('adb', ['-s', device.serial || device.udid, 'shell', 'am', 'force-stop', params.bundleId]);
331
-
332
- adb.on('close', (code) => {
333
- resolve({ success: code === 0 });
334
- });
335
- adb.on('error', (error) => reject(error));
336
- });
337
-
338
- case 'mobile: activateApp':
339
- // Same as launch for Android
340
- return this.execute('mobile: launchApp', params);
341
-
342
- default:
343
- throw new Error(`Unsupported Android command: ${command}`);
344
- }
345
- } catch (error) {
346
- this.logger.error(`Android driver command failed: ${error.message}`);
347
- throw error;
348
- }
349
- },
350
-
351
- getWindowSize: async () => {
352
- return this.lastWindowSize;
353
- }
354
- };
355
- }
356
-
357
- return null;
358
- }
359
-
360
- /**
361
- * Get UIAutomator2 session ID from universal session manager
362
- * This ensures session ID is always in sync across all operations
363
- */
364
- async getAndroidSession(device) {
365
- if (!device || device.platform !== 'android') return null;
366
-
367
- try {
368
- // Use universal session manager to get live session for Android
369
- const sessionId = await this.sessionManager.getLiveSessionId(device.name, device.port);
370
- return sessionId;
371
- } catch (error) {
372
- this.logger.warn(`⚠️ Could not get Android session for ${device.name}: ${error.message}`);
373
-
374
- // Fallback: Try different possible keys for Android session lookup
375
- const possibleKeys = [
376
- device.udid,
377
- device.serial,
378
- device.name
379
- ].filter(Boolean);
380
-
381
- for (const key of possibleKeys) {
382
- const session = global.androidSessions?.get(key);
383
- if (session && session.sessionId && session.port) {
384
- // Cache session info on device object for faster access
385
- device.sessionId = session.sessionId;
386
- device.port = session.port;
387
- return session;
388
- }
389
- }
390
-
391
- return null;
392
- }
393
- }
394
-
395
- async setCommander(deviceName, deviceInfo) {
396
- if (this.isActive && this.commanderDevice && this.commanderDevice.name === deviceName) {
397
- // Same device - commander mode is already active, just return success
398
- this.logger.info(`Commander mode already active on ${deviceName}`);
399
- return {
400
- success: true,
401
- commanderDevice: this.commanderDevice,
402
- sessionId: this.sessionId,
403
- message: 'Commander mode already active'
404
- };
405
- }
406
-
407
- if (this.isActive && this.commanderDevice && this.commanderDevice.name !== deviceName) {
408
- // Different device - preserve sessions during commander switch
409
- this.logger.info(`Switching commander from ${this.commanderDevice.name} to ${deviceName}`);
410
-
411
- // CRITICAL: Preserve WDA session of outgoing commander for potential follower role
412
- const outgoingCommander = this.commanderDevice;
413
- if (outgoingCommander) {
414
- this.logger.info(`📝 Preserving WDA session for outgoing commander: ${outgoingCommander.name}`);
415
-
416
- // Add outgoing commander as follower to preserve its WDA session
417
- const followerDevice = {
418
- name: outgoingCommander.name,
419
- udid: outgoingCommander.udid,
420
- platform: outgoingCommander.platform,
421
- port: outgoingCommander.port,
422
- sessionId: this.sessionId, // Preserve the current session
423
- wdaSessionId: this.sessionId,
424
- wasCommander: true // Mark this as a former commander
425
- };
426
-
427
- this.followerDevices.add(followerDevice);
428
- this.logger.info(`📝 Former commander ${outgoingCommander.name} automatically added as follower to preserve WDA session`);
429
-
430
- // Only stop touch monitoring, keep WDA alive and preserve followers
431
- this.stopTouchMonitoring();
432
- this.commanderDevice = null;
433
- this.sessionId = null;
434
- }
435
- }
436
-
437
- this.commanderDevice = {
438
- ...deviceInfo,
439
- name: deviceName // CRITICAL: Ensure deviceName overrides deviceInfo.name
440
- };
441
- this.isActive = true;
442
- this.sessionId = `commander_${Date.now()}`;
443
-
444
- // CRITICAL: Track if this device was recently a follower
445
- this.wasRecentlyFollower = deviceInfo.wasRecentlyFollower || false;
446
-
447
- // Debug device name handling
448
- this.logger.info(`📱 Commander device set: "${deviceName}" (platform: ${deviceInfo.platform})`);
449
- this.logger.info(`📊 Device info name: "${deviceInfo.name}", used name: "${this.commanderDevice.name}"`);
450
-
451
- if (this.wasRecentlyFollower) {
452
- this.logger.info(`🔄 Device ${deviceName} was recently a follower - will reuse existing WDA session`);
453
- }
454
-
455
- // CRITICAL: Ensure we use the passed deviceName, not deviceInfo.name
456
- if (deviceInfo.name && deviceInfo.name !== deviceName) {
457
- this.logger.warn(`⚠️ Name mismatch detected and corrected: deviceInfo.name="${deviceInfo.name}" → using "${deviceName}"`);
458
- }
459
-
460
- // For Android devices, get UIAutomator2 session info using global helper
461
- if (deviceInfo.platform === 'android') {
462
- const session = await this.getAndroidSession(this.commanderDevice);
463
-
464
- if (session) {
465
- this.logger.info(`✅ Android UIAutomator2 session found: port ${session.port}, sessionId ${session.sessionId.substring(0, 8)}...`);
466
- this.commanderDriver = await this.createDriverInterface();
467
- } else {
468
- this.logger.warn(`⚠️ No UIAutomator2 session found, will look up on first screenshot`);
469
- // Still create the driver interface - it will check for session when needed
470
- this.commanderDriver = await this.createDriverInterface();
471
- }
472
- }
473
- // For iOS devices, ONLY check for existing WDA session - DON'T create new one
474
- else if (deviceInfo.platform === 'ios') {
475
- await this.checkExistingWDASession();
476
- }
477
-
478
- // Start real-time touch event monitoring
479
- this.startTouchMonitoring();
480
-
481
- this.broadcastCommanderStatus();
482
-
483
- return {
484
- success: true,
485
- commanderDevice: this.commanderDevice,
486
- sessionId: this.sessionId
487
- };
488
- }
489
-
490
- async checkExistingWDASession() {
491
- const axios = require('axios');
492
- const device = this.commanderDevice;
493
- const port = device.port || 8100;
494
-
495
- try {
496
- // Check for existing WDA session first - ALWAYS, even for promoted followers
497
- this.logger.info('Checking for existing WDA session...');
498
-
499
- const wasRecentFollower = this.wasRecentlyFollower || false;
500
- if (wasRecentFollower) {
501
- this.logger.info(`🔄 Device ${device.name} was recently a follower - attempting to reuse existing WDA session`);
502
- }
503
-
504
- // ALWAYS try to find and reuse existing sessions first
505
- try {
506
- const existingSessionId = await this.sessionManager.getActiveSession(device.name, device.port || 8100);
507
-
508
- if (existingSessionId) {
509
- this.logger.info(`✅ Found existing WDA session: ${existingSessionId.substring(0, 8)}... - testing viability`);
510
-
511
- // Test if the session actually works
512
- const sessionWorks = await this.validateSessionWorking(device.port || 8100);
513
- if (sessionWorks) {
514
- this.commanderDriver = this.createDriverInterface();
515
- this.logger.info(`✅ Reusing healthy existing WDA session for ${wasRecentFollower ? 'promoted follower' : 'commander'}`);
516
- await this.getDeviceWindowSize(device.port || 8100);
517
- return;
518
- } else {
519
- this.logger.warn(`⚠️ Existing session found but not responding - will create fresh session`);
520
- }
521
- } else {
522
- this.logger.info('No existing session found - will create new one');
523
- }
524
- } catch (sessionCheckError) {
525
- this.logger.warn(`Session check failed: ${sessionCheckError.message} - will create new session`);
526
- }
527
-
528
- // GENIUS: First check global session manager
529
- const existingSession = global.sessionManager?.getByName(device.name);
530
- if (existingSession && existingSession.platform === 'ios') {
531
- this.logger.info(`✅ Found global WDA session: ${existingSession.sessionId}`);
532
-
533
- // CRITICAL: Test if this session actually works for screenshots before using it
534
- const sessionWorks = await this.validateSessionWorking(port);
535
- if (sessionWorks) {
536
- this.commanderDriver = this.createDriverInterface();
537
- this.logger.info(`✅ Using validated global WDA session: ${existingSession.sessionId}`);
538
- await this.getDeviceWindowSize(port);
539
- return;
540
- } else {
541
- this.logger.warn(`⚠️ Global session found but screenshot test failed - creating fresh session`);
542
- await this.createWDASession();
543
- return;
544
- }
545
- }
546
-
547
- // Fallback: check connectedDevices for backward compatibility
548
- const globalDevice = this.connectedDevices.find(d => d.name === device.name || d.udid === device.udid);
549
- if (globalDevice && globalDevice.sessionId) {
550
- // FIXED: Use safe session registration (don't override for Quick actions)
551
- global.sessionManager?.getOrRegisterSession(device.udid, globalDevice.sessionId, port, device.name, null, 'ios');
552
- this.commanderDriver = this.createDriverInterface();
553
- this.logger.info(`✅ Registered and using WDA session: ${globalDevice.sessionId}`);
554
- await this.getDeviceWindowSize(port);
555
- return;
556
- }
557
-
558
- // If no global session, check /status endpoint for existing session
559
- try {
560
- const statusResponse = await axios.get(`http://localhost:${port}/status`, { timeout: 5000 });
561
- const sessionId = statusResponse.data?.sessionId || statusResponse.data?.value?.sessionId;
562
-
563
- if (sessionId) {
564
- // Register globally to ensure it's available everywhere
565
- this.registerSessionGlobally(device, sessionId, port);
566
- this.commanderDriver = this.createDriverInterface();
567
- this.logger.info(`✅ Found and registered WDA session: ${sessionId.substring(0, 8)}...`);
568
- await this.getDeviceWindowSize(port);
569
- return;
570
- }
571
- } catch (statusError) {
572
- this.logger.debug(`WDA /status endpoint check failed: ${statusError.message}`);
573
- }
574
-
575
- // Try to get existing session from /sessions endpoint
576
- try {
577
- const sessionsResponse = await axios.get(`http://localhost:${port}/sessions`, { timeout: 3000 });
578
- const sessions = sessionsResponse.data?.value;
579
-
580
- if (sessions && Array.isArray(sessions) && sessions.length > 0) {
581
- const sessionId = sessions[0].id;
582
- // Register globally
583
- this.registerSessionGlobally(device, sessionId, port);
584
- this.commanderDriver = this.createDriverInterface();
585
- this.logger.info(`✅ Found and registered session from /sessions: ${sessionId.substring(0, 8)}...`);
586
- await this.getDeviceWindowSize(port);
587
- return;
588
- }
589
- } catch (e) {
590
- this.logger.debug('No sessions found at /sessions endpoint');
591
- }
592
-
593
- // If no existing session found, create a new one (this should work since we validated WDA availability)
594
- this.logger.info('⚠️ No existing WDA session found - creating new session for commander mode');
595
-
596
- // CRITICAL DEBUG: Log the complete device state for troubleshooting
597
- this.logger.info(`🔍 Device promotion debug info:`, {
598
- name: device.name,
599
- udid: device.udid,
600
- port: device.port,
601
- platform: device.platform
602
- });
603
-
604
- // Check if this device was recently a follower
605
- const wasFollower = this.followerDevices && Array.from(this.followerDevices).some(f => f.name === device.name);
606
- if (wasFollower) {
607
- this.logger.info(`🔄 Device ${device.name} was recently a follower, ensuring clean session setup`);
608
- }
609
-
610
- try {
611
- await this.createWDASession();
612
- this.commanderDriver = this.createDriverInterface();
613
- this.logger.info('✅ New WDA session created for commander mode');
614
-
615
- // CRITICAL: Force immediate session validation after creation
616
- await this.validateSessionWorking(device.port || 8100);
617
-
618
- } catch (createError) {
619
- this.logger.error(`❌ Failed to create WDA session: ${createError.message}`);
620
- this.logger.warn('💡 Commander mode will work with limited functionality');
621
-
622
- // EMERGENCY FALLBACK: Try one more time after a delay
623
- setTimeout(async () => {
624
- try {
625
- this.logger.info(`🔧 Emergency retry: Creating WDA session for ${device.name}`);
626
- await this.createWDASession();
627
- this.commanderDriver = this.createDriverInterface();
628
- this.logger.info('✅ Emergency retry successful');
629
- } catch (retryError) {
630
- this.logger.error(`❌ Emergency retry failed: ${retryError.message}`);
631
- }
632
- }, 2000);
633
- }
634
-
635
- } catch (error) {
636
- this.logger.error(`❌ Failed to check WDA status: ${error.message}`);
637
- this.logger.warn('💡 Commander mode will continue with limited functionality');
638
- // Continue anyway - since WDA was validated before we got here, this is likely a temporary issue
639
- }
640
- }
641
-
642
- // CRITICAL: Validate that a WDA session is actually working for screenshots
643
- async validateSessionWorking(port) {
644
- const axios = require('axios');
645
-
646
- try {
647
- this.logger.info(`🧪 Fast validating WDA session on port ${port}...`);
648
-
649
- // Test screenshot endpoint with faster timeout
650
- const screenshotResponse = await axios.get(`http://localhost:${port}/screenshot`, {
651
- timeout: 4000, // Reduced from 8000ms to 4000ms for faster validation
652
- validateStatus: () => true // Don't throw on any status
653
- });
654
-
655
- if (screenshotResponse.status === 200) {
656
- const imageData = screenshotResponse.data;
657
- if (imageData && (typeof imageData === 'string' && imageData.length > 100)) {
658
- this.logger.info(`✅ WDA session validation successful - screenshot working`);
659
- return true;
660
- } else {
661
- this.logger.warn(`⚠️ WDA session responds but image data invalid (length: ${imageData?.length || 0})`);
662
- return false;
663
- }
664
- } else {
665
- this.logger.warn(`⚠️ WDA session validation failed - status ${screenshotResponse.status}`);
666
- return false;
667
- }
668
-
669
- } catch (error) {
670
- this.logger.warn(`⚠️ WDA session validation failed: ${error.message}`);
671
- return false;
672
- }
673
- }
674
-
675
- async startTouchMonitoring() {
676
- if (this.touchMonitorInterval) {
677
- clearInterval(this.touchMonitorInterval);
678
- }
679
-
680
- this.logger.info('🎯 Starting optimized screen mirroring and touch monitoring');
681
-
682
- // Pre-warm sessions for faster startup
683
- await this.preWarmSessions();
684
-
685
- // Start screen streaming (2 FPS for responsive interaction)
686
- this.startScreenStreaming();
687
- }
688
-
689
- /**
690
- * Pre-warm sessions to reduce initial mirroring delay
691
- */
692
- async preWarmSessions() {
693
- try {
694
- this.logger.info('⚡ Pre-warming sessions for instant mirroring...');
695
-
696
- // Pre-warm commander device session
697
- if (this.commanderDevice) {
698
- if (this.commanderDevice.platform === 'android') {
699
- await this.getAndroidSession(this.commanderDevice);
700
- } else if (this.commanderDevice.platform === 'ios') {
701
- const port = this.commanderDevice.port || 8100;
702
- try {
703
- await this.sessionManager.getLiveSessionId(this.commanderDevice.name, port);
704
- this.logger.info(`✅ Pre-warmed iOS session for ${this.commanderDevice.name}:${port}`);
705
- } catch (error) {
706
- this.logger.debug(`Pre-warm failed for ${this.commanderDevice.name}, will discover during streaming`);
707
- }
708
- }
709
- }
710
-
711
- // Pre-warm follower device sessions and cache screenshot URLs
712
- if (this.followerDevices && this.followerDevices.size > 0) {
713
- for (const follower of this.followerDevices) {
714
- try {
715
- if (follower.platform === 'android') {
716
- const session = await this.getAndroidSession(follower);
717
- if (session) {
718
- // Pre-cache screenshot URL for faster streaming
719
- follower._cachedPort = session.port;
720
- follower._cachedScreenshotUrl = `http://localhost:${session.port}/wd/hub/session/${session.sessionId}/screenshot`;
721
- this.logger.info(`✅ Pre-cached Android follower: ${follower.name}:${session.port}`);
722
- }
723
- } else if (follower.platform === 'ios') {
724
- const port = follower.port || 8100;
725
- try {
726
- await this.sessionManager.getLiveSessionId(follower.name, port);
727
- // Pre-cache screenshot URL for faster streaming
728
- follower._cachedPort = port;
729
- follower._cachedScreenshotUrl = `http://localhost:${port}/screenshot`;
730
- this.logger.info(`✅ Pre-cached iOS follower: ${follower.name}:${port}`);
731
- } catch (error) {
732
- // Try alternative port for iOS
733
- const altPort = port === 8100 ? 8101 : 8100;
734
- try {
735
- await this.sessionManager.getLiveSessionId(follower.name, altPort);
736
- follower.port = altPort;
737
- follower._cachedPort = altPort;
738
- follower._cachedScreenshotUrl = `http://localhost:${altPort}/screenshot`;
739
- this.logger.info(`✅ Pre-cached iOS follower on alt port: ${follower.name}:${altPort}`);
740
- } catch (altError) {
741
- this.logger.debug(`Pre-warm failed for follower ${follower.name} on both ports`);
742
- }
743
- }
744
- }
745
- } catch (error) {
746
- this.logger.debug(`Pre-warm failed for follower ${follower.name}, will discover during streaming`);
747
- }
748
- }
749
- }
750
-
751
- this.logger.info('⚡ Session pre-warming complete');
752
- } catch (error) {
753
- this.logger.warn(`Pre-warming failed: ${error.message}, continuing with normal startup`);
754
- }
755
- }
756
-
757
- async createWDASession() {
758
- const axios = require('axios');
759
- const device = this.commanderDevice;
760
- const port = device.port || 8100;
761
-
762
- try {
763
- this.logger.info('No existing session found, creating new one...');
764
-
765
- // Create new session
766
- const createResponse = await axios.post(`http://localhost:${port}/session`, {
767
- capabilities: {
768
- alwaysMatch: {},
769
- firstMatch: [{}]
770
- }
771
- }, { timeout: 10000 });
772
-
773
- const newSessionId = createResponse.data?.sessionId || createResponse.data?.value?.sessionId;
774
-
775
- if (newSessionId) {
776
- // Register globally to ensure it's available everywhere
777
- this.registerSessionGlobally(device, newSessionId, port);
778
- this.logger.info(`✅ WDA session created and registered: ${newSessionId.substring(0, 8)}...`);
779
- await this.getDeviceWindowSize(port);
780
- } else {
781
- throw new Error('Failed to create WDA session - no session ID returned');
782
- }
783
-
784
- } catch (error) {
785
- this.logger.error(`❌ WDA session creation failed: ${error.message}`);
786
- throw error;
787
- }
788
- }
789
-
790
- async startScreenStreaming() {
791
- if (this.screenStreamInterval) {
792
- clearInterval(this.screenStreamInterval);
793
- }
794
-
795
- const axios = require('axios');
796
- this.logger.info('📺 Starting optimized screen streaming at 2 FPS');
797
-
798
- // CRITICAL DEBUG: Check device state before starting streaming
799
- if (!this.commanderDevice) {
800
- this.logger.error('❌ Cannot start screen streaming: no commander device set');
801
- return;
802
- }
803
-
804
- this.logger.info(`📱 Starting screen streaming for: "${this.commanderDevice.name}" on port ${this.commanderDevice.port}`);
805
-
806
- // Pre-validate session and cache endpoint URL for faster streaming
807
- const device = this.commanderDevice;
808
- let cachedScreenshotUrl = null;
809
- let cachedPort = null;
810
-
811
- try {
812
- if (device.platform === 'android') {
813
- const session = await this.getAndroidSession(device);
814
- if (session) {
815
- cachedPort = session.port;
816
- cachedScreenshotUrl = `http://localhost:${cachedPort}/wd/hub/session/${session.sessionId}/screenshot`;
817
- this.logger.info(`✅ Pre-cached Android screenshot URL: ${cachedScreenshotUrl.substring(0, 80)}...`);
818
- }
819
- } else if (device.platform === 'ios') {
820
- cachedPort = device.port || 8100;
821
- try {
822
- const sessionId = await this.sessionManager.getLiveSessionId(device.name, cachedPort);
823
- cachedScreenshotUrl = `http://localhost:${cachedPort}/screenshot`;
824
- this.logger.info(`✅ Pre-validated iOS WDA session on port ${cachedPort}`);
825
- } catch (error) {
826
- this.logger.info(`⚡ Will try session discovery during first screenshot - continuing optimistically`);
827
- cachedScreenshotUrl = `http://localhost:${cachedPort}/screenshot`;
828
- }
829
- }
830
- } catch (error) {
831
- this.logger.warn(`⚠️ Session pre-validation failed, will discover during streaming: ${error.message}`);
832
- }
833
-
834
- let lastScreenHash = null;
835
- let frameCount = 0;
836
- let consecutiveErrors = 0;
837
-
838
- // Stream at 2 FPS (every 500ms) with optimized error handling
839
- this.screenStreamInterval = setInterval(async () => {
840
- try {
841
- const device = this.commanderDevice;
842
-
843
- if (!device) {
844
- this.logger.error('❌ Commander device is null during streaming');
845
- return;
846
- }
847
-
848
- // Use cached screenshot URL first, or fallback to session discovery
849
- let screenshotUrl = cachedScreenshotUrl;
850
- let port = cachedPort;
851
-
852
- // If we don't have cached URL, do fast session lookup
853
- if (!screenshotUrl) {
854
- if (device.platform === 'android') {
855
- // For Android UIAutomator2, use session-based endpoint with global session manager
856
- const session = await this.getAndroidSession(device);
857
-
858
- if (!session) {
859
- consecutiveErrors++;
860
- if (consecutiveErrors <= 3) { // Only log for first few errors
861
- this.logger.warn(`⚠️ No Android UIAutomator2 session found for ${device.name} (attempt ${consecutiveErrors})`);
862
- }
863
- return;
864
- }
865
-
866
- port = session.port;
867
- screenshotUrl = `http://localhost:${port}/wd/hub/session/${session.sessionId}/screenshot`;
868
- // Cache for future use
869
- cachedPort = port;
870
- cachedScreenshotUrl = screenshotUrl;
871
-
872
- } else if (device.platform === 'ios') {
873
- // iOS WDA uses simple screenshot endpoint with faster port detection
874
- port = device.port || cachedPort || 8100;
875
- screenshotUrl = `http://localhost:${port}/screenshot`;
876
-
877
- // Only do session validation if we haven't cached it yet
878
- if (!cachedScreenshotUrl) {
879
- try {
880
- // Quick session check with reduced timeout
881
- const sessionId = await this.sessionManager.getLiveSessionId(device.name, port);
882
- // Cache successful session info
883
- cachedPort = port;
884
- cachedScreenshotUrl = screenshotUrl;
885
- } catch (error) {
886
- // Try one alternative port quickly
887
- const altPort = port === 8100 ? 8101 : 8100;
888
- try {
889
- await this.sessionManager.getLiveSessionId(device.name, altPort);
890
- port = altPort;
891
- screenshotUrl = `http://localhost:${altPort}/screenshot`;
892
- // Cache successful session info
893
- cachedPort = port;
894
- cachedScreenshotUrl = screenshotUrl;
895
- device.port = altPort; // Update device port
896
- } catch (altError) {
897
- consecutiveErrors++;
898
- if (consecutiveErrors <= 3) {
899
- this.logger.warn(`⚠️ No WDA session found for ${device.name} on ports ${port} or ${altPort} (attempt ${consecutiveErrors})`);
900
- }
901
- return;
902
- }
903
- }
904
- }
905
- }
906
- }
907
-
908
- // Get screenshot from WDA/UIAutomator2 with optimized timeout
909
- const response = await axios.get(screenshotUrl, {
910
- timeout: 3000, // Reduced timeout for faster response
911
- validateStatus: () => true // Don't throw on any status
912
- });
913
-
914
- if (response.status !== 200) {
915
- consecutiveErrors++;
916
- // Clear cache if we have errors - might be stale
917
- if (consecutiveErrors >= 2) {
918
- cachedScreenshotUrl = null;
919
- cachedPort = null;
920
- }
921
-
922
- if (consecutiveErrors <= 3) { // Only log for first few errors
923
- this.logger.warn(`⚠️ Screenshot failed with status ${response.status} for ${device.name} (attempt ${consecutiveErrors})`);
924
- if (response.status === 404) {
925
- this.logger.debug(`Session might be expired - will retry session discovery`);
926
- } else if (response.status === 500) {
927
- this.logger.debug(`Device might be locked or disconnected`);
928
- }
929
- }
930
- return;
931
- }
932
-
933
- // Reset error counter on success
934
- consecutiveErrors = 0;
935
-
936
- const base64Image = response.data?.value || response.data;
937
-
938
- if (!base64Image || typeof base64Image !== 'string' || base64Image.length < 100) {
939
- this.logger.debug(`❌ Invalid image data from ${device.name}: length=${base64Image?.length}`);
940
- return;
941
- }
942
-
943
- // Detect screen changes for auto-refresh
944
- const currentHash = base64Image.substring(0, 100); // Simple hash using first 100 chars
945
- const screenChanged = lastScreenHash && lastScreenHash !== currentHash;
946
- lastScreenHash = currentHash;
947
- frameCount++;
948
-
949
- // Get device window size for coordinate scaling
950
- const windowSize = await this.getDeviceWindowSize(port);
951
-
952
- // Broadcast commander screen to all connected clients
953
- this.broadcast({
954
- type: 'screen_frame',
955
- image: `data:image/png;base64,${base64Image}`,
956
- deviceName: device.name,
957
- windowSize: windowSize
958
- });
959
-
960
- // Stream follower devices screens
961
- this.streamFollowerScreens();
962
-
963
- // DISABLED: Auto-refresh locators on screen change (too aggressive)
964
- // Only refresh manually or after user clicks
965
- // if (screenChanged && frameCount > 5) { // Wait for 5 frames before detecting changes
966
- // this.logger.info('📍 Screen changed - auto-refreshing locators...');
967
- // this.broadcast({
968
- // type: 'locators_reloading',
969
- // message: 'Locators are reloading...'
970
- // });
971
- // this.cachedElements = [];
972
- // this.lastCacheTime = 0;
973
- // this.cacheAllElements().catch(err =>
974
- // this.logger.warn(`Auto-refresh failed: ${err.message}`)
975
- // );
976
- // }
977
-
978
- // DISABLED: Load locators only once after first frame (too aggressive)
979
- // Only load locators manually when user requests via UI
980
- // if (this.cachedElements.length === 0 && !this.isCaching && Date.now() - this.lastCacheTime > 5000) {
981
- // this.broadcast({
982
- // type: 'locators_loading',
983
- // message: 'Loading locators...'
984
- // });
985
- // this.cacheAllElements().catch(err =>
986
- // this.logger.warn(`Background locator loading failed: ${err.message}`)
987
- // );
988
- // }
989
-
990
- } catch (error) {
991
- // CRITICAL: Better error handling for commander screen streaming
992
- if (error.message.includes('socket hang up')) {
993
- this.logger.error(`❌ Commander WDA connection lost (socket hang up) - attempting recovery`);
994
-
995
- // Try to recover the session
996
- setTimeout(async () => {
997
- try {
998
- this.logger.info('🔧 Attempting commander session recovery...');
999
- await this.checkExistingWDASession();
1000
- } catch (recoveryError) {
1001
- this.logger.error(`❌ Session recovery failed: ${recoveryError.message}`);
1002
- }
1003
- }, 2000);
1004
-
1005
- } else if (error.code === 'ECONNREFUSED') {
1006
- this.logger.error(`❌ Commander WDA not responding - connection refused`);
1007
- } else {
1008
- this.logger.error(`❌ Commander screenshot error: ${error.message}`);
1009
- }
1010
- }
1011
- }, 500);
1012
- }
1013
-
1014
- async streamFollowerScreens() {
1015
- if (!this.followerDevices || this.followerDevices.size === 0) return;
1016
-
1017
- const axios = require('axios');
1018
-
1019
- // Stream each follower device screen with optimized session handling
1020
- for (const follower of this.followerDevices) {
1021
- try {
1022
- // Use cached screenshot URL if available, otherwise do fast discovery
1023
- let screenshotUrl = follower._cachedScreenshotUrl;
1024
- let port = follower._cachedPort;
1025
-
1026
- if (!screenshotUrl) {
1027
- // Fast session discovery for followers
1028
- if (follower.platform === 'android') {
1029
- const session = await this.getAndroidSession(follower);
1030
-
1031
- if (!session || !session.port || !session.sessionId) {
1032
- follower._consecutiveErrors = (follower._consecutiveErrors || 0) + 1;
1033
- if (follower._consecutiveErrors <= 2) { // Reduced error logging
1034
- this.logger.debug(`⚠️ No UIAutomator2 session for follower: ${follower.name} (attempt ${follower._consecutiveErrors})`);
1035
- }
1036
- continue;
1037
- }
1038
-
1039
- port = session.port;
1040
- screenshotUrl = `http://localhost:${port}/wd/hub/session/${session.sessionId}/screenshot`;
1041
-
1042
- // Cache for future use
1043
- follower._cachedPort = port;
1044
- follower._cachedScreenshotUrl = screenshotUrl;
1045
-
1046
- } else if (follower.platform === 'ios') {
1047
- port = follower.port || 8100;
1048
- screenshotUrl = `http://localhost:${port}/screenshot`;
1049
-
1050
- // Quick session validation for iOS followers (only if not cached)
1051
- if (!follower._cachedScreenshotUrl) {
1052
- try {
1053
- await this.sessionManager.getLiveSessionId(follower.name, port);
1054
- // Cache successful session info
1055
- follower._cachedPort = port;
1056
- follower._cachedScreenshotUrl = screenshotUrl;
1057
- } catch (error) {
1058
- // Try one alternative port quickly
1059
- const altPort = port === 8100 ? 8101 : 8100;
1060
- try {
1061
- await this.sessionManager.getLiveSessionId(follower.name, altPort);
1062
- port = altPort;
1063
- screenshotUrl = `http://localhost:${altPort}/screenshot`;
1064
- follower.port = altPort;
1065
- follower._cachedPort = port;
1066
- follower._cachedScreenshotUrl = screenshotUrl;
1067
- } catch (altError) {
1068
- follower._consecutiveErrors = (follower._consecutiveErrors || 0) + 1;
1069
- if (follower._consecutiveErrors <= 2) {
1070
- this.logger.debug(`⚠️ No WDA session for follower ${follower.name} on ports ${port} or ${altPort}`);
1071
- }
1072
- continue;
1073
- }
1074
- }
1075
- }
1076
- }
1077
- }
1078
-
1079
- // Get screenshot with optimized timeout
1080
- const response = await axios.get(screenshotUrl, {
1081
- timeout: 3000, // Reduced from 5000ms to 3000ms for faster follower streaming
1082
- validateStatus: () => true // Don't throw on any status
1083
- });
1084
-
1085
- if (response.status !== 200) {
1086
- follower._consecutiveErrors = (follower._consecutiveErrors || 0) + 1;
1087
-
1088
- // Clear cache if we have persistent errors
1089
- if (follower._consecutiveErrors >= 2) {
1090
- follower._cachedScreenshotUrl = null;
1091
- follower._cachedPort = null;
1092
- }
1093
-
1094
- if (follower._consecutiveErrors <= 2) {
1095
- this.logger.debug(`⚠️ Follower ${follower.name} returned status ${response.status} (attempt ${follower._consecutiveErrors})`);
1096
- }
1097
- continue;
1098
- }
1099
-
1100
- // Reset error counter on success
1101
- follower._consecutiveErrors = 0;
1102
-
1103
- const base64Image = response.data?.value || response.data;
1104
-
1105
- if (base64Image && typeof base64Image === 'string' && base64Image.length > 100) {
1106
- this.broadcast({
1107
- type: 'follower_screen',
1108
- deviceId: follower.name,
1109
- image: `data:image/png;base64,${base64Image}`
1110
- });
1111
- this.logger.debug(`📸 Streamed follower screen: ${follower.name}`);
1112
- } else {
1113
- this.logger.warn(`⚠️ No image data for follower: ${follower.name}`);
1114
- }
1115
- } catch (error) {
1116
- if (error.message.includes('socket hang up')) {
1117
- this.logger.warn(`⚠️ Follower ${follower.name} WDA connection lost - session may need refresh`);
1118
-
1119
- // For followers, try to refresh their session
1120
- if (follower.platform === 'ios') {
1121
- setTimeout(async () => {
1122
- try {
1123
- this.logger.info(`🔧 Refreshing WDA session for follower ${follower.name}...`);
1124
- await this.checkFollowerWDASession(follower);
1125
- } catch (refreshError) {
1126
- this.logger.warn(`⚠️ Failed to refresh follower session: ${refreshError.message}`);
1127
- }
1128
- }, 3000);
1129
- }
1130
-
1131
- } else {
1132
- this.logger.warn(`⚠️ Follower ${follower.name} screenshot error: ${error.message}`);
1133
- }
1134
- }
1135
- }
1136
- }
1137
-
1138
- async getDeviceWindowSize(port) {
1139
- try {
1140
- const axios = require('axios');
1141
- let size = null;
1142
-
1143
- // Try dedicated window size endpoint first (most reliable for logical points)
1144
- const wdaSessionId = await this.getWDASessionId();
1145
- if (wdaSessionId) {
1146
- try {
1147
- const res = await axios.get(`http://localhost:${port}/session/${wdaSessionId}/window/size`, { timeout: 1000 });
1148
- if (res.data?.value) {
1149
- size = res.data.value;
1150
- this.logger.debug(`Window size from /window/size: ${size.width}x${size.height}`);
1151
- }
1152
- } catch (e) { /* ignore */ }
1153
- }
1154
-
1155
- // Fallback to /status if session endpoint failed
1156
- if (!size) {
1157
- const response = await axios.get(`http://localhost:${port}/status`, { timeout: 1000 });
1158
- if (response.data?.value?.screenSize) {
1159
- size = response.data.value.screenSize;
1160
- this.logger.debug(`Window size from /status: ${size.width}x${size.height}`);
1161
- }
1162
- }
1163
-
1164
- if (size && size.width > 0) {
1165
- this.lastWindowSize = {
1166
- width: size.width,
1167
- height: size.height
1168
- };
1169
- }
1170
-
1171
- return this.lastWindowSize;
1172
- } catch (error) {
1173
- return this.lastWindowSize;
1174
- }
1175
- }
1176
-
1177
- async handleScreenClick(x, y, windowSize) {
1178
- // GENIUS: Queue system for handling rapid clicks
1179
- return new Promise((resolve, reject) => {
1180
- const clickRequest = { x, y, windowSize, resolve, reject, timestamp: Date.now() };
1181
-
1182
- // Limit queue size to prevent memory issues
1183
- if (this.clickQueue.length >= this.maxQueueSize) {
1184
- this.logger.warn(`⚠️ Click queue full (${this.maxQueueSize}), dropping oldest click`);
1185
- const dropped = this.clickQueue.shift();
1186
- dropped.reject(new Error('Click dropped due to queue overflow'));
1187
- }
1188
-
1189
- this.clickQueue.push(clickRequest);
1190
- this.processClickQueue();
1191
- });
1192
- }
1193
-
1194
- async processClickQueue() {
1195
- if (this.processingClick || this.clickQueue.length === 0) {
1196
- return;
1197
- }
1198
-
1199
- this.processingClick = true;
1200
-
1201
- try {
1202
- while (this.clickQueue.length > 0) {
1203
- const clickRequest = this.clickQueue.shift();
1204
- const age = Date.now() - clickRequest.timestamp;
1205
-
1206
- // Skip very old clicks (>2 seconds)
1207
- if (age > 2000) {
1208
- this.logger.warn(`⚠️ Skipping stale click (${age}ms old)`);
1209
- clickRequest.reject(new Error('Click too old'));
1210
- continue;
1211
- }
1212
-
1213
- try {
1214
- this.logger.info(`🔄 Processing queued click (${this.clickQueue.length} remaining)`);
1215
- const result = await this._handleScreenClickInternal(clickRequest.x, clickRequest.y, clickRequest.windowSize);
1216
- clickRequest.resolve(result);
1217
- } catch (error) {
1218
- clickRequest.reject(error);
1219
- }
1220
-
1221
- // Small delay between clicks to prevent overwhelming the system
1222
- if (this.clickQueue.length > 0) {
1223
- await new Promise(resolve => setTimeout(resolve, 200));
1224
- }
1225
- }
1226
- } finally {
1227
- this.processingClick = false;
1228
- }
1229
- }
1230
-
1231
- async _handleScreenClickInternal(x, y, windowSize) {
1232
- if (!this.isActive) {
1233
- throw new Error('Commander mode not active');
1234
- }
1235
-
1236
- if (!this.commanderDevice) {
1237
- throw new Error('No commander device available');
1238
- }
1239
-
1240
- this.logger.info('');
1241
- this.logger.info('═══════════════════════════════════════════════════════');
1242
- this.logger.info(`🖱️ CLICK RECEIVED: (${x}, ${y})`);
1243
- this.logger.info('═══════════════════════════════════════════════════════');
1244
-
1245
- // Determine if incoming coordinates are physical or logical
1246
- // windowSize parameter is what the frontend THINKS is the total size
1247
- let logicalX = x;
1248
- let logicalY = y;
1249
-
1250
- // If the windowSize provided by frontend is physical resolution, convert to logical
1251
- if (windowSize && windowSize.width > 600) { // Likely physical
1252
- const scaleX = this.lastWindowSize.width / windowSize.width;
1253
- const scaleY = this.lastWindowSize.height / windowSize.height;
1254
- logicalX = Math.round(x * scaleX);
1255
- logicalY = Math.round(y * scaleY);
1256
- this.logger.info(`📐 Converting physical (${x}, ${y}) to logical (${logicalX}, ${logicalY})`);
1257
- this.logger.info(`📏 Scale factors: X=${scaleX.toFixed(3)}, Y=${scaleY.toFixed(3)}`);
1258
- this.logger.info(`📱 Frontend size: ${windowSize.width}x${windowSize.height}, Device size: ${this.lastWindowSize.width}x${this.lastWindowSize.height}`);
1259
- } else {
1260
- this.logger.info(`📍 Using logical coordinates: (${logicalX}, ${logicalY})`);
1261
- this.logger.info(`📏 Frontend size: ${windowSize?.width || 'unknown'}x${windowSize?.height || 'unknown'}`);
1262
- }
1263
-
1264
- this.logger.info(`🎯 Target: ${this.commanderDevice.name}`);
1265
-
1266
- // 🚀 SIMPLE APP LAUNCH DETECTION: Check home screen FIRST
1267
- const port = this.commanderDevice.port || 8100;
1268
-
1269
- // REVOLUTIONARY: Get live session ID for pre-checks
1270
- let liveSessionId = null;
1271
- let isOnHomeScreen = false;
1272
-
1273
- try {
1274
- liveSessionId = await this.sessionManager.getSessionWithRetry(this.commanderDevice.name, port);
1275
-
1276
- try {
1277
- const preAppRes = await axios.get(`http://localhost:${port}/session/${liveSessionId}/wda/activeAppInfo`, { timeout: 2000 });
1278
- const preAppId = preAppRes.data?.value?.bundleId;
1279
- isOnHomeScreen = preAppId === 'com.apple.springboard';
1280
- this.logger.info(`🏠 SIMPLE CHECK: ${preAppId} ${isOnHomeScreen ? '(HOME SCREEN - WILL SKIP FOLLOWER COORDS)' : '(IN APP - NORMAL FLOW)'}`);
1281
- } catch (preAppError) {
1282
- this.logger.debug(`🏠 Pre-check failed: ${preAppError.message}`);
1283
- isOnHomeScreen = false;
1284
- }
1285
- } catch (sessionError) {
1286
- this.logger.warn(`⚠️ Failed to get live session for pre-check: ${sessionError.message}`);
1287
- isOnHomeScreen = false;
1288
- }
1289
-
1290
- try {
1291
- // ===============================
1292
- // STEP -1: REVOLUTIONARY SESSION MANAGEMENT
1293
- // ===============================
1294
-
1295
- // ALWAYS get fresh live session ID - no more stored sessions!
1296
- const liveSession = await this.sessionManager.getSessionWithRetry(this.commanderDevice.name, port);
1297
- this.logger.info(`🎯 Using live WDA session: ${liveSession.substring(0, 8)}... for ${this.commanderDevice.name}`);
1298
-
1299
- let isSessionHealthy = false;
1300
-
1301
- // Quick health check using window size (fast and reliable)
1302
- try {
1303
- const windowRes = await axios.get(`http://localhost:${port}/session/${liveSession}/window/size`, { timeout: 2000 });
1304
- if (windowRes.data?.value) {
1305
- this.logger.info(`✅ Session healthy: ${liveSession.substring(0, 8)}...`);
1306
- isSessionHealthy = true;
1307
- // Store window size for coordinate scaling
1308
- this.lastWindowSize = windowRes.data.value;
1309
- }
1310
- } catch (healthError) {
1311
- this.logger.warn(`⚠️ Session health check failed: ${healthError.message}`);
1312
-
1313
- // For 404 errors, try getting a fresh session
1314
- if (healthError.response?.status === 404) {
1315
- this.logger.warn(`🔄 Session expired (404), getting fresh session...`);
1316
- try {
1317
- const freshSession = await this.sessionManager.getSessionWithRetry(this.commanderDevice.name, port);
1318
-
1319
- // Test the fresh session
1320
- const windowRes = await axios.get(`http://localhost:${port}/session/${freshSession}/window/size`, { timeout: 2000 });
1321
- if (windowRes.data?.value) {
1322
- this.logger.info(`✅ Fresh session healthy: ${freshSession.substring(0, 8)}...`);
1323
- isSessionHealthy = true;
1324
- this.lastWindowSize = windowRes.data.value;
1325
- }
1326
- } catch (recoveryError) {
1327
- this.logger.error(`❌ Session recovery failed: ${recoveryError.message}`);
1328
- }
1329
- }
1330
- }
1331
-
1332
- // If no session is healthy, use default window size
1333
- if (!isSessionHealthy) {
1334
- this.logger.warn(`⚠️ No healthy session found, using default window size`);
1335
- this.lastWindowSize = { width: 414, height: 896 };
1336
- }
1337
-
1338
- // ===============================
1339
- // STEP 0: NO AUTO LOCATOR CACHING - MANUAL ONLY
1340
- // ===============================
1341
- // Cache age checking disabled - only manual refresh via UI
1342
- // const cacheAge = Date.now() - this.lastCacheTime;
1343
- // const needsRefresh = this.cachedElements.length === 0 || cacheAge > 30000;
1344
-
1345
- // DISABLED: Auto-refresh on clicks - use existing cache or user can manually refresh
1346
- // if (needsRefresh && !this.isCaching && this.cachedElements.length < 3) {
1347
- // this.logger.info(`🔄 Refreshing locators before click (${this.cachedElements.length} cached, ${cacheAge}ms old)`);
1348
- // await this.fastCacheAllElements();
1349
- // } else if (this.cachedElements.length > 0) {
1350
- // this.logger.debug(`⚡ Skipping cache refresh for speed (${this.cachedElements.length} elements, ${cacheAge}ms old)`);
1351
- // }
1352
-
1353
- // ===============================
1354
- // 🎯 STEP 1: INTELLIGENT LOCATOR DETECTION (NEW GENIUS METHOD)
1355
- // ===============================
1356
- let intelligentLocator = null;
1357
-
1358
- if (isOnHomeScreen) {
1359
- // 🚀 SIMPLE: Skip intelligent locator for app launches, use coordinates directly
1360
- this.logger.info(`🚀 SIMPLE: Skipping intelligent locator - home screen app launch, using coordinates`);
1361
- intelligentLocator = { method: 'coordinates', x: logicalX, y: logicalY };
1362
- } else {
1363
- this.logger.info(`🎯 INTELLIGENT LOCATOR: Finding smart locator for (${logicalX}, ${logicalY})...`);
1364
-
1365
- if (this.useIntelligentLocators) {
1366
- try {
1367
- // CRITICAL FIX: Ensure cache is fresh for current screen state
1368
- // Check cache age and refresh if it's older than a few seconds
1369
- // PERFORMANCE: Optimize cache management for commander speed - SAFER VERSION
1370
- const cacheAge = this.intelligentLocator.getCacheAge(this.commanderDevice.name);
1371
-
1372
- // More conservative cache strategy to prevent issues
1373
- const hasValidCache = cacheAge !== null && cacheAge < 15000; // REDUCED: 15 seconds (was 20)
1374
- const needsRefresh = !hasValidCache;
1375
-
1376
- if (needsRefresh) {
1377
- this.logger.info(`🔄 Cache is stale (age: ${cacheAge}ms) - refreshing before locator search...`);
1378
- } else {
1379
- this.logger.debug(`⚡ Using existing cache (age: ${cacheAge}ms) for commander speed`);
1380
- }
1381
-
1382
- // Get intelligent locator - only force refresh if truly needed
1383
- intelligentLocator = await this.intelligentLocator.getIntelligentLocator(
1384
- this.commanderDevice.name,
1385
- logicalX,
1386
- logicalY,
1387
- needsRefresh // Force refresh if cache is stale
1388
- );
1389
-
1390
- if (intelligentLocator.method === 'locator') {
1391
- this.logger.info(`✅ INTELLIGENT: Found ${intelligentLocator.locator.strategy} locator with ${(intelligentLocator.confidence * 100).toFixed(1)}% confidence`);
1392
- this.logger.info(` Strategy: ${intelligentLocator.locator.strategy} = "${intelligentLocator.locator.selector}"`);
1393
- this.logger.info(` Element: ${intelligentLocator.element.elementType} "${intelligentLocator.element.text || intelligentLocator.element.accessibilityLabel || 'No text'}"`);
1394
- } else {
1395
- this.logger.warn(`⚠️ INTELLIGENT: Falling back to coordinates (${intelligentLocator.method})`);
1396
- this.coordinateFallbackCount++;
1397
- }
1398
-
1399
- } catch (error) {
1400
- this.logger.error(`❌ INTELLIGENT: Locator detection failed: ${error.message}`);
1401
- intelligentLocator = { method: 'coordinates', x: logicalX, y: logicalY };
1402
- this.coordinateFallbackCount++;
1403
- }
1404
- } else {
1405
- // Traditional method for compatibility
1406
- this.logger.info(`🔍 TRADITIONAL: Finding element at target location...`);
1407
- const preClickElement = this.findClosestElement(logicalX, logicalY);
1408
-
1409
- if (preClickElement) {
1410
- intelligentLocator = {
1411
- method: 'locator',
1412
- locator: {
1413
- strategy: 'accessibility_id',
1414
- selector: preClickElement.accessibilityId || preClickElement.label || preClickElement.name
1415
- },
1416
- fallbackCoords: { x: logicalX, y: logicalY },
1417
- element: preClickElement
1418
- };
1419
- this.logger.info(`✅ TRADITIONAL: Found element: ${preClickElement.label}`);
1420
- } else {
1421
- intelligentLocator = { method: 'coordinates', x: logicalX, y: logicalY };
1422
- this.logger.warn(`⚠️ TRADITIONAL: No element found, using coordinates`);
1423
- }
1424
- }
1425
- } // Close the else block for non-home screen processing
1426
-
1427
- // ===============================
1428
- // STEP 2: EXECUTE CLICK ON COMMANDER
1429
- // ===============================
1430
- // Get fresh live session using universal session manager
1431
- const device = this.commanderDevice;
1432
-
1433
- // REVOLUTIONARY: Always get current live session
1434
- let currentSession;
1435
- try {
1436
- currentSession = await this.sessionManager.getSessionWithRetry(device.name, port);
1437
- this.logger.info(`📱 Using WDA session: ${currentSession.substring(0, 8)}... on port ${port}`);
1438
- } catch (sessionError) {
1439
- throw new Error(`No valid WDA session found for ${device.name}`);
1440
- }
1441
-
1442
- let retryAttempted = false;
1443
-
1444
- // Execute tap on commander using LOGICAL coordinates - AWAIT to ensure it completes
1445
- try {
1446
- this.logger.info(`⏳ Executing tap on commander device...`);
1447
- this.logger.info(`📍 Final tap coordinates: (${logicalX}, ${logicalY})`);
1448
-
1449
- // Simple coordinate validation - only prevent obviously dangerous coordinates
1450
- let safeX = Math.round(logicalX);
1451
- let safeY = Math.round(logicalY);
1452
-
1453
- const deviceHeight = this.lastWindowSize?.height || 896;
1454
- const deviceWidth = this.lastWindowSize?.width || 414;
1455
-
1456
- // Basic bounds checking only (don't modify unless clearly outside device bounds)
1457
- if (safeX < 0 || safeX > deviceWidth || safeY < 0 || safeY > deviceHeight) {
1458
- this.logger.warn(`⚠️ Coordinates outside device bounds, clamping: (${safeX}, ${safeY})`);
1459
- safeX = Math.max(10, Math.min(safeX, deviceWidth - 10));
1460
- safeY = Math.max(50, Math.min(safeY, deviceHeight - 100)); // Avoid control center
1461
- }
1462
-
1463
- // CRITICAL FIX: Use current session from above
1464
- await axios.post(
1465
- `http://localhost:${port}/session/${currentSession}/actions`,
1466
- {
1467
- actions: [{
1468
- type: 'pointer',
1469
- id: 'finger1',
1470
- parameters: { pointerType: 'touch' },
1471
- actions: [
1472
- { type: 'pointerMove', duration: 0, x: safeX, y: safeY },
1473
- { type: 'pointerDown', button: 0 },
1474
- { type: 'pause', duration: 100 },
1475
- { type: 'pointerUp', button: 0 }
1476
- ]
1477
- }]
1478
- },
1479
- { timeout: 15000 }
1480
- );
1481
- this.logger.info(`✅ Tap executed on commander: ${this.commanderDevice.name}`);
1482
-
1483
- // 🚀 SIMPLE APP LAUNCH DETECTION: Check if app changed after tap
1484
- setTimeout(async () => {
1485
- // Only check for app launch if we were originally on home screen
1486
- if (!isOnHomeScreen) {
1487
- this.logger.debug('📱 Skipping app detection - was not on home screen');
1488
- return;
1489
- }
1490
-
1491
- try {
1492
- const currentAppRes = await axios.get(`http://localhost:${port}/session/${currentSession}/wda/activeAppInfo`, { timeout: 3000 });
1493
- const currentAppId = currentAppRes.data?.value?.bundleId;
1494
-
1495
- if (currentAppId && currentAppId !== 'com.apple.springboard') {
1496
- this.logger.info(`🚀 SIMPLE: App launched on commander: ${currentAppId} - launching on followers...`);
1497
-
1498
- // Launch same app on all followers immediately
1499
- for (const follower of this.followerDevices) {
1500
- if (follower.platform === 'ios') {
1501
- try {
1502
- const followerPort = follower.port || 8100;
1503
-
1504
- // Get fresh session ID using universal session manager
1505
- let followerSessionId = null;
1506
- try {
1507
- followerSessionId = await this.sessionManager.getSessionWithRetry(follower.name, followerPort);
1508
- } catch (sessionError) {
1509
- this.logger.warn(`⚠️ Could not get session for follower ${follower.name}: ${sessionError.message}`);
1510
- continue; // Skip this follower
1511
- }
1512
-
1513
- if (followerSessionId) {
1514
- this.logger.info(`🚀 Auto-launching ${currentAppId} on follower: ${follower.name}`);
1515
- await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/wda/apps/launch`, {
1516
- bundleId: currentAppId
1517
- }, { timeout: 15000 });
1518
- this.logger.info(`✅ App ${currentAppId} launched on ${follower.name}`);
1519
- } else {
1520
- this.logger.warn(`⚠️ No valid session found for ${follower.name}`);
1521
- }
1522
- } catch (launchError) {
1523
- this.logger.warn(`⚠️ Failed to launch ${currentAppId} on ${follower.name}: ${launchError.message}`);
1524
- }
1525
- }
1526
- }
1527
-
1528
- // 🚀 REFRESH CACHE AFTER APP LAUNCH: Now that app is launched, refresh locators for new app
1529
- setTimeout(() => {
1530
- this.logger.info('🔄 Refreshing locators after successful app launch...');
1531
- this.intelligentLocator.invalidateCache(this.commanderDevice.name);
1532
- // Refresh in background for next interactions in the new app
1533
- this.intelligentLocator.getCachedLocators(this.commanderDevice.name, true).catch(() => {});
1534
- for (const follower of this.followerDevices) {
1535
- this.intelligentLocator.getCachedLocators(follower.name, true).catch(() => {});
1536
- }
1537
- }, 2000); // Wait 2s for app to fully load
1538
-
1539
- }
1540
- } catch (appCheckError) {
1541
- // Ignore errors in app detection - not critical
1542
- this.logger.debug(`📱 App check failed: ${appCheckError.message}`);
1543
- }
1544
- }, 1000); // Wait 1 second for app to launch (reduced from 2s)
1545
-
1546
- // 🎯 OPTIMIZED CACHE INVALIDATION: Less aggressive, smarter cache management
1547
- if (!isOnHomeScreen) {
1548
- this.logger.info('🔄 Invalidating locator cache after successful tap to ensure fresh screen state');
1549
- this.intelligentLocator.invalidateCache(this.commanderDevice.name);
1550
-
1551
- // PERFORMANCE: Make background refresh safer with better error handling
1552
- setImmediate(() => {
1553
- // Safer async background refresh
1554
- (async () => {
1555
- try {
1556
- this.logger.debug('🔄 Background refresh of locators after regular tap...');
1557
-
1558
- // Refresh commander cache with timeout protection
1559
- const refreshCommander = this.intelligentLocator.getCachedLocators(this.commanderDevice.name, true)
1560
- .catch(err => {
1561
- this.logger.debug(`Commander refresh failed: ${err.message}`);
1562
- return null;
1563
- });
1564
-
1565
- await refreshCommander;
1566
-
1567
- // Also refresh follower caches in background with individual error handling
1568
- if (this.followerDevices && this.followerDevices.size > 0) {
1569
- const refreshPromises = Array.from(this.followerDevices).map(async (follower) => {
1570
- try {
1571
- await this.intelligentLocator.getCachedLocators(follower.name, true);
1572
- } catch (err) {
1573
- this.logger.debug(`Background refresh failed for ${follower.name}: ${err.message}`);
1574
- }
1575
- });
1576
-
1577
- await Promise.allSettled(refreshPromises);
1578
- }
1579
-
1580
- this.logger.debug('✅ Background locator refresh completed');
1581
- } catch (refreshErr) {
1582
- this.logger.debug(`Background refresh error: ${refreshErr.message}`);
1583
- }
1584
- })().catch(err => {
1585
- // Final safety net to prevent unhandled promise rejections
1586
- this.logger.debug(`Background refresh async error: ${err.message}`);
1587
- });
1588
- });
1589
- } else {
1590
- this.logger.info('🚀 Skipping cache refresh - app launch detected, will refresh after app stabilizes');
1591
- }
1592
- } catch (err) {
1593
- // GENIUS: Smarter session handling - avoid recreating sessions that just need a status sync
1594
- if (err.response && err.response.status === 404 && !retryAttempted) {
1595
- this.logger.warn(`⚠️ Session might be expired (404), attempting smart recovery...`);
1596
-
1597
- // First, try to get a valid session from WDA status
1598
- let sessionRecovered = false;
1599
- try {
1600
- const port = this.commanderDevice.port || 8100;
1601
- const statusRes = await axios.get(`http://localhost:${port}/status`, { timeout: 5000 });
1602
- const activeSession = statusRes.data?.sessionId;
1603
-
1604
- if (activeSession && activeSession !== currentSession) {
1605
- // WDA has an active session, just update our reference
1606
- this.logger.info(`🔄 Found active WDA session: ${activeSession.substring(0, 8)}...`);
1607
- this.sessionManager.updateSession(this.commanderDevice.name, activeSession);
1608
- currentSession = activeSession;
1609
- sessionRecovered = true;
1610
- this.logger.info(`✅ Session recovered without recreation`);
1611
- }
1612
- } catch (statusError) {
1613
- this.logger.debug(`Status check failed: ${statusError.message}`);
1614
- }
1615
-
1616
- // Only recreate if we couldn't recover an existing session
1617
- if (!sessionRecovered) {
1618
- // Before recreating, try one more recovery approach - check if device is still responsive
1619
- try {
1620
- const port = this.commanderDevice.port || 8100;
1621
- const healthRes = await axios.get(`http://localhost:${port}/wda/healthcheck`, { timeout: 2000 });
1622
- if (healthRes.status === 200) {
1623
- // WDA is healthy, just create a new session without full recreation
1624
- this.logger.info(`🔄 WDA is healthy, creating new session without full restart...`);
1625
- const newSessionRes = await axios.post(`http://localhost:${port}/session`, {
1626
- capabilities: {
1627
- alwaysMatch: {},
1628
- firstMatch: [{}]
1629
- }
1630
- }, { timeout: 10000 });
1631
-
1632
- const newSessionId = newSessionRes.data?.sessionId || newSessionRes.data?.value?.sessionId;
1633
- if (newSessionId) {
1634
- this.logger.info(`✅ New session created: ${newSessionId.substring(0, 8)}...`);
1635
- this.commanderDevice.sessionId = newSessionId;
1636
- if (global.sessionManager) {
1637
- global.sessionManager.updateSession(this.commanderDevice.udid, newSessionId);
1638
- }
1639
- currentSession = newSessionId;
1640
- sessionRecovered = true;
1641
- }
1642
- }
1643
- } catch (healthError) {
1644
- this.logger.debug(`Health check failed: ${healthError.message}`);
1645
- }
1646
- }
1647
-
1648
- // If session recovery failed, use universal session manager as last resort
1649
- if (!sessionRecovered) {
1650
- this.logger.warn(`⚠️ No active session found, getting fresh session...`);
1651
- try {
1652
- currentSession = await this.sessionManager.getSessionWithRetry(device.name, port);
1653
- sessionRecovered = true;
1654
- } catch (recreateError) {
1655
- this.logger.error(`❌ Session recovery failed: ${recreateError.message}`);
1656
- throw new Error(`Commander tap failed: ${err.message}`);
1657
- }
1658
- }
1659
-
1660
- // Retry the tap with recovered/new session
1661
- if (sessionRecovered) {
1662
- // Retry the tap with new session - use simple coordinate validation
1663
- try {
1664
- await axios.post(
1665
- `http://localhost:${port}/session/${currentSession}/actions`,
1666
- {
1667
- actions: [{
1668
- type: 'pointer',
1669
- id: 'finger1',
1670
- parameters: { pointerType: 'touch' },
1671
- actions: [
1672
- { type: 'pointerMove', duration: 0, x: safeX, y: safeY },
1673
- { type: 'pointerDown', button: 0 },
1674
- { type: 'pause', duration: 100 },
1675
- { type: 'pointerUp', button: 0 }
1676
- ]
1677
- }]
1678
- },
1679
- { timeout: 15000 }
1680
- );
1681
- this.logger.info(`✅ Tap executed on commander: ${this.commanderDevice.name} (after session recovery)`);
1682
-
1683
- // 🎯 CACHE INVALIDATION: Force fresh locators after successful recovery tap
1684
- this.logger.info('🔄 Invalidating locator cache after successful recovery tap');
1685
- this.intelligentLocator.invalidateCache(this.commanderDevice.name);
1686
-
1687
- // 🎯 OPTIMIZED CACHE REFRESH: Schedule refresh but don't block execution
1688
- setTimeout(async () => {
1689
- try {
1690
- this.logger.info('🔄 Background refresh after recovery tap...');
1691
- await this.intelligentLocator.getCachedLocators(this.commanderDevice.name, true);
1692
-
1693
- // Background refresh for followers
1694
- // OPTIMIZATION: Skip duplicate follower refresh - already handled in intelligent actions
1695
- this.logger.info('ℹ️ Follower caches managed in intelligent actions - skipping duplicate refresh');
1696
-
1697
- this.logger.info('✅ Background refresh completed (recovery)');
1698
- } catch (refreshErr) {
1699
- // Ignore background refresh errors
1700
- }
1701
- }, 500);
1702
- } catch (retryErr) {
1703
- this.logger.error(`❌ Commander tap failed even after session recovery: ${retryErr.message}`);
1704
- throw new Error(`Commander tap failed: ${retryErr.message}`);
1705
- }
1706
- } else {
1707
- this.logger.error(`❌ Could not recover WDA session for commander`);
1708
- throw new Error(`Commander tap failed: ${err.message}`);
1709
- }
1710
- } else {
1711
- this.logger.error(`❌ Commander tap failed: ${err.message}`);
1712
- throw new Error(`Commander tap failed: ${err.message}`);
1713
- }
1714
- }
1715
-
1716
- // ===============================
1717
- // 🎯 STEP 4: BROADCAST INTELLIGENT LOCATORS TO FOLLOWERS (NEW GENIUS METHOD)
1718
- // ===============================
1719
- const followerDevices = Array.from(this.followerDevices);
1720
- if (followerDevices.length > 0) {
1721
-
1722
- if (isOnHomeScreen) {
1723
- this.logger.info('🏠 SIMPLE: On home screen - skipping follower broadcast, will detect app launch instead...');
1724
- // Skip immediate coordinate broadcast, let app detection handle it
1725
- } else {
1726
- this.logger.info('');
1727
- this.logger.info(`📡 Broadcasting INTELLIGENT locators to ${followerDevices.length} follower(s):`);
1728
- for (const follower of followerDevices) {
1729
- this.logger.info(` • ${follower.name} (${follower.platform})`);
1730
- this.executeIntelligentActionOnFollower(follower, intelligentLocator, logicalX, logicalY)
1731
- .catch(e => this.logger.error(`Follower ${follower.name} intelligent action failed: ${e.message}`));
1732
- }
1733
- }
1734
- } else {
1735
- this.logger.info('ℹ️ No followers connected');
1736
- }
1737
-
1738
- // Log performance metrics
1739
- const metrics = this.intelligentLocator.getPerformanceMetrics();
1740
- this.logger.info(`📊 Intelligent Locator Metrics: ${metrics.accuracyRate} accuracy, ${metrics.cacheHitRate} cache hit rate`);
1741
-
1742
- this.logger.info('═══════════════════════════════════════════════════════');
1743
- this.logger.info('');
1744
- return {
1745
- success: true,
1746
- devicesCount: 1 + followerDevices.length,
1747
- intelligentLocator: intelligentLocator,
1748
- method: intelligentLocator.method,
1749
- metrics: metrics
1750
- };
1751
- } catch (error) {
1752
- this.logger.error('Screen click handling error:', error.message);
1753
- throw error;
1754
- }
1755
- }
1756
-
1757
- // Faster caching method that avoids heavy API calls
1758
- async fastCacheAllElements() {
1759
- const wdaSessionId = await this.getWDASessionId();
1760
- if (!wdaSessionId || !this.commanderDevice) return;
1761
- if (this.isCaching) return;
1762
-
1763
- this.isCaching = true;
1764
- const startTime = Date.now();
1765
-
1766
- try {
1767
- const device = this.commanderDevice;
1768
- const port = device.port || 8100;
1769
- const sessionId = wdaSessionId;
1770
-
1771
- this.logger.info('🔍 Starting fast locator detection...');
1772
-
1773
- // Try page source first (fastest method)
1774
- try {
1775
- const sourceRes = await axios.get(
1776
- `http://localhost:${port}/session/${sessionId}/source`,
1777
- { timeout: 3000 } // Reduced timeout for speed
1778
- );
1779
-
1780
- if (sourceRes.data?.value) {
1781
- const pageSource = sourceRes.data.value;
1782
- const allElements = this.parsePageSourceFast(pageSource);
1783
-
1784
- if (allElements.length > 0) {
1785
- this.cachedElements = allElements;
1786
- this.lastCacheTime = Date.now();
1787
-
1788
- const elapsed = Date.now() - startTime;
1789
- this.logger.info(`✅ Fast cached ${allElements.length} elements in ${elapsed}ms`);
1790
- return;
1791
- }
1792
- }
1793
- } catch (pageSourceErr) {
1794
- this.logger.warn(`Page source method failed, falling back to element query`);
1795
-
1796
- // Fallback: Quick element query for essential elements only - SPEED OPTIMIZED
1797
- const elementTypes = ['XCUIElementTypeButton']; // Only buttons for speed
1798
- const allElements = [];
1799
-
1800
- for (const elementType of elementTypes) {
1801
- try {
1802
- const query = { using: 'class name', value: elementType };
1803
- const elementsRes = await axios.post(
1804
- `http://localhost:${port}/session/${sessionId}/elements`,
1805
- query,
1806
- { timeout: 2000 } // Quick timeout
1807
- );
1808
-
1809
- if (elementsRes.data?.value && Array.isArray(elementsRes.data.value)) {
1810
- const elements = elementsRes.data.value.slice(0, 10); // Further reduced for speed
1811
-
1812
- for (const elem of elements) {
1813
- const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
1814
- if (!elementId) continue;
1815
-
1816
- try {
1817
- const rectRes = await axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/rect`, { timeout: 500 });
1818
-
1819
- if (rectRes.data?.value) {
1820
- const rect = rectRes.data.value;
1821
- if (rect.width > 5 && rect.height > 5) { // Skip tiny elements
1822
- allElements.push({
1823
- type: elementType,
1824
- label: 'Unlabeled', // Skip label for speed
1825
- name: 'Unlabeled',
1826
- accessibilityId: 'Unlabeled',
1827
- x: Math.round(rect.x),
1828
- y: Math.round(rect.y),
1829
- width: Math.round(rect.width),
1830
- height: Math.round(rect.height),
1831
- centerX: Math.round(rect.x + rect.width / 2),
1832
- centerY: Math.round(rect.y + rect.height / 2)
1833
- });
1834
- }
1835
- }
1836
- } catch (err) {
1837
- // Skip this element, continue with others
1838
- continue;
1839
- }
1840
- }
1841
- }
1842
- } catch (queryErr) {
1843
- this.logger.warn(`Query ${elementType} failed: ${queryErr.message}`);
1844
- continue;
1845
- }
1846
- }
1847
-
1848
- this.cachedElements = allElements;
1849
- this.lastCacheTime = Date.now();
1850
-
1851
- const elapsed = Date.now() - startTime;
1852
- this.logger.info(`✅ Cached ${allElements.length} elements in ${elapsed}ms`);
1853
- }
1854
- } catch (error) {
1855
- this.logger.error(`Fast cache failed: ${error.message}`);
1856
- } finally {
1857
- this.isCaching = false;
1858
- }
1859
- }
1860
-
1861
- async cacheAllElements() {
1862
- const wdaSessionId = await this.getWDASessionId();
1863
- if (!wdaSessionId || !this.commanderDevice) return;
1864
- if (this.isCaching) return;
1865
-
1866
- this.isCaching = true;
1867
- const startTime = Date.now();
1868
-
1869
- try {
1870
- const axios = require('axios');
1871
- const device = this.commanderDevice;
1872
- const port = device.port || 8100;
1873
- const sessionId = wdaSessionId;
1874
-
1875
- this.logger.info('🔍 Starting fast locator detection...');
1876
-
1877
- // Use page source for MUCH faster element detection
1878
- try {
1879
- const sourceRes = await axios.get(
1880
- `http://localhost:${port}/session/${sessionId}/source`,
1881
- { timeout: 8000 }
1882
- );
1883
-
1884
- if (sourceRes.data?.value) {
1885
- const pageSource = sourceRes.data.value;
1886
- const allElements = this.parsePageSourceFast(pageSource);
1887
-
1888
- if (allElements.length > 0) {
1889
- this.cachedElements = allElements;
1890
- this.lastCacheTime = Date.now();
1891
-
1892
- const elapsed = Date.now() - startTime;
1893
- this.logger.info(`✅ Fast cached ${allElements.length} elements in ${elapsed}ms`);
1894
-
1895
- // Send all at once from page source (fastest method)
1896
- this.broadcast({
1897
- type: 'cached_elements',
1898
- elements: allElements.map(e => ({
1899
- label: e.label || e.name,
1900
- type: e.type.replace('XCUIElementType', ''),
1901
- x: e.x,
1902
- y: e.y,
1903
- width: e.width,
1904
- height: e.height
1905
- })),
1906
- total: allElements.length,
1907
- timestamp: Date.now()
1908
- });
1909
-
1910
- // Broadcast completion
1911
- this.broadcast({
1912
- type: 'locators_loaded',
1913
- total: allElements.length
1914
- });
1915
-
1916
- this.isCaching = false;
1917
- return;
1918
- }
1919
- }
1920
- } catch (err) {
1921
- this.logger.warn('Page source method failed, falling back to element query');
1922
- }
1923
-
1924
- // Optimized queries - focus on most important element types only
1925
- const queries = [
1926
- { using: 'class name', value: 'XCUIElementTypeButton' },
1927
- { using: 'class name', value: 'XCUIElementTypeStaticText' }
1928
- ];
1929
-
1930
- const allElements = [];
1931
- const MAX_ELEMENTS = 20; // Further reduced for speed
1932
-
1933
- // Process queries sequentially with timeout protection
1934
- for (const query of queries) {
1935
- if (allElements.length >= MAX_ELEMENTS) break;
1936
-
1937
- try {
1938
- const elementsRes = await axios.post(
1939
- `http://localhost:${port}/session/${sessionId}/elements`,
1940
- query,
1941
- { timeout: 3000 } // Reduced timeout
1942
- );
1943
-
1944
- if (elementsRes.data?.value && Array.isArray(elementsRes.data.value)) {
1945
- const elements = elementsRes.data.value.slice(0, MAX_ELEMENTS - allElements.length);
1946
-
1947
- // Process elements in smaller batches to reduce session load
1948
- const chunkSize = 5; // Smaller chunks for better performance
1949
- for (let i = 0; i < elements.length && allElements.length < MAX_ELEMENTS; i += chunkSize) {
1950
- const chunk = elements.slice(i, i + chunkSize);
1951
- const chunkPromises = chunk.map(async (elem) => {
1952
- const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
1953
- if (!elementId) return null;
1954
-
1955
- try {
1956
- // Only get rect - skip label to reduce API calls
1957
- const rectRes = await axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/rect`, { timeout: 800 });
1958
-
1959
- if (rectRes.data?.value) {
1960
- const rect = rectRes.data.value;
1961
-
1962
- // Skip very small elements that are not useful
1963
- if (rect.width < 10 || rect.height < 10) return null;
1964
-
1965
- // Get label only for buttons (most important)
1966
- let label = 'Unlabeled';
1967
- if (query.value === 'XCUIElementTypeButton') {
1968
- try {
1969
- const labelRes = await axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/attribute/label`, { timeout: 500 });
1970
- label = labelRes.data?.value || 'Unlabeled';
1971
- } catch (labelErr) {
1972
- // Skip this element if we can't get its label for buttons
1973
- return null;
1974
- }
1975
- }
1976
-
1977
- return {
1978
- type: query.value,
1979
- label: label,
1980
- name: label || 'Unlabeled',
1981
- accessibilityId: label,
1982
- x: Math.round(rect.x),
1983
- y: Math.round(rect.y),
1984
- width: Math.round(rect.width),
1985
- height: Math.round(rect.height),
1986
- centerX: Math.round(rect.x + rect.width / 2),
1987
- centerY: Math.round(rect.y + rect.height / 2)
1988
- };
1989
- }
1990
- } catch (err) { return null; }
1991
- return null;
1992
- });
1993
-
1994
- const chunkResults = await Promise.all(chunkPromises);
1995
- const validElements = chunkResults.filter(e => e !== null);
1996
-
1997
- if (validElements.length > 0) {
1998
- typeElements.push(...validElements);
1999
- allElements.push(...validElements);
2000
-
2001
- // Broadcast incrementally for instant UI feedback
2002
- this.broadcast({
2003
- type: 'cached_elements',
2004
- elements: validElements.map(e => ({
2005
- label: e.label || e.name,
2006
- type: e.type.replace('XCUIElementType', ''),
2007
- x: e.x,
2008
- y: e.y,
2009
- width: e.width,
2010
- height: e.height
2011
- })),
2012
- total: allElements.length,
2013
- timestamp: Date.now()
2014
- });
2015
- }
2016
- }
2017
- }
2018
- } catch (err) {
2019
- this.logger.warn(`Query ${query.value} failed: ${err.message}`);
2020
- }
2021
- }
2022
-
2023
- this.cachedElements = allElements;
2024
- this.lastCacheTime = Date.now();
2025
-
2026
- const elapsed = Date.now() - startTime;
2027
- this.logger.info(`✅ Cached ${allElements.length} elements in ${elapsed}ms`);
2028
-
2029
- // Broadcast completion
2030
- this.broadcast({
2031
- type: 'locators_loaded',
2032
- total: allElements.length
2033
- });
2034
-
2035
- } finally {
2036
- this.isCaching = false;
2037
- }
2038
- }
2039
-
2040
- // GENIUS: Enhanced session recovery with minimal disruption
2041
- async recoverOrCreateSession(device, originalSessionId = null) {
2042
- const port = device.port || 8100;
2043
-
2044
- this.logger.info(`🔍 Smart session recovery for ${device.name}...`);
2045
-
2046
- // STEP 1: Check if WDA server itself is responsive
2047
- try {
2048
- const statusRes = await axios.get(`http://localhost:${port}/status`, { timeout: 2000 });
2049
- const serverSessionId = statusRes.data?.sessionId;
2050
-
2051
- if (serverSessionId && serverSessionId !== originalSessionId) {
2052
- // WDA has a VALID session that's different from ours - use it!
2053
- this.logger.info(`✅ Found active WDA session: ${serverSessionId.substring(0, 8)}...`);
2054
-
2055
- // Test the session works
2056
- try {
2057
- await axios.get(`http://localhost:${port}/session/${serverSessionId}/window/size`, { timeout: 2000 });
2058
-
2059
- // Update our references
2060
- device.sessionId = serverSessionId;
2061
- if (global.sessionManager) {
2062
- global.sessionManager.updateSession(device.udid, serverSessionId);
2063
- }
2064
-
2065
- this.logger.info(`✅ Session recovered without recreation`);
2066
- return serverSessionId;
2067
- } catch (testErr) {
2068
- this.logger.debug(`Found session ${serverSessionId} doesn't work: ${testErr.message}`);
2069
- }
2070
- }
2071
- } catch (statusErr) {
2072
- this.logger.debug(`WDA status check failed: ${statusErr.message}`);
2073
- }
2074
-
2075
- // STEP 2: Try creating a NEW session without killing existing ones
2076
- try {
2077
- this.logger.info(`🔄 Creating fresh WDA session (keeping WDA running)...`);
2078
- const sessionRes = await axios.post(`http://localhost:${port}/session`, {
2079
- capabilities: {
2080
- alwaysMatch: {},
2081
- firstMatch: [{}]
2082
- }
2083
- }, { timeout: 15000 });
2084
-
2085
- const newSessionId = sessionRes.data?.sessionId || sessionRes.data?.value?.sessionId;
2086
- if (newSessionId) {
2087
- this.logger.info(`✅ Fresh session created: ${newSessionId.substring(0, 8)}...`);
2088
-
2089
- // Update references
2090
- device.sessionId = newSessionId;
2091
- if (global.sessionManager) {
2092
- global.sessionManager.updateSession(device.udid, newSessionId);
2093
- }
2094
-
2095
- return newSessionId;
2096
- }
2097
- } catch (createErr) {
2098
- this.logger.warn(`⚠️ Fresh session creation failed: ${createErr.message}`);
2099
- }
2100
-
2101
- // STEP 3: Last resort - full WDA restart (only if nothing else worked)
2102
- this.logger.warn(`🔄 All gentle approaches failed, doing full session recreation...`);
2103
- await this.recreateWDASession();
2104
- return this.getWDASessionId();
2105
- }
2106
-
2107
- async recreateWDASession() {
2108
- if (!this.commanderDevice) {
2109
- throw new Error('No commander device set');
2110
- }
2111
-
2112
- this.logger.info(`🔄 Recreating WDA session for ${this.commanderDevice.name}...`);
2113
-
2114
- try {
2115
- const device = this.commanderDevice;
2116
- const port = device.port || 8100;
2117
-
2118
- // Store current app before session recreation
2119
- let currentAppId = null;
2120
- try {
2121
- const oldSessionId = this.getWDASessionId();
2122
- if (oldSessionId) {
2123
- const appRes = await axios.get(`http://localhost:${port}/session/${oldSessionId}/wda/activeAppInfo`, { timeout: 2000 });
2124
- currentAppId = appRes.data?.value?.bundleId;
2125
- this.logger.info(`📱 Current app before session recreation: ${currentAppId || 'unknown'}`);
2126
- }
2127
- } catch (appCheckError) {
2128
- this.logger.warn(`⚠️ Could not detect current app: ${appCheckError.message}`);
2129
- }
2130
-
2131
- // Remove the old session from global manager
2132
- if (global.sessionManager) {
2133
- global.sessionManager.deleteSession(device.udid);
2134
- this.logger.info('🗑️ Removed stale session from global manager');
2135
- }
2136
-
2137
- // Create a completely new WDA session
2138
- await this.createWDASession();
2139
-
2140
- // Verify the new session works
2141
- const newSessionId = this.getWDASessionId();
2142
- if (newSessionId) {
2143
- // Test the session with a simple status call
2144
- await axios.get(`http://localhost:${port}/session/${newSessionId}/window/size`, { timeout: 3000 });
2145
-
2146
- // Smart app restoration - restore the ACTUAL app that was running
2147
- if (currentAppId && currentAppId !== 'com.apple.springboard') {
2148
- try {
2149
- this.logger.info(`🔄 Restoring original app: ${currentAppId}`);
2150
- await axios.post(`http://localhost:${port}/session/${newSessionId}/wda/apps/launch`, {
2151
- bundleId: currentAppId
2152
- }, { timeout: 10000 }); // Increased from 5s to 10s for app restoration
2153
-
2154
- // Wait longer for app to fully stabilize after session recreation
2155
- await new Promise(resolve => setTimeout(resolve, 4000)); // Increased from 2.5s to 4s
2156
- this.logger.info(`✅ Original app restored: ${currentAppId}`);
2157
-
2158
- // Verify app actually launched
2159
- const verifyRes = await axios.get(`http://localhost:${port}/session/${newSessionId}/wda/activeAppInfo`, { timeout: 3000 });
2160
- const actualApp = verifyRes.data?.value?.bundleId;
2161
- if (actualApp === currentAppId) {
2162
- this.logger.info(`✅ App restoration verified: ${actualApp}`);
2163
- } else {
2164
- this.logger.warn(`⚠️ App restoration failed: expected ${currentAppId}, got ${actualApp}`);
2165
- }
2166
- } catch (appRestoreError) {
2167
- this.logger.warn(`⚠️ Could not restore original app ${currentAppId}: ${appRestoreError.message}`);
2168
- // Don't try to restore anything else - leave in home screen
2169
- }
2170
- } else {
2171
- this.logger.info(`ℹ️ No specific app to restore (was in: ${currentAppId || 'home screen'})`);
2172
- }
2173
-
2174
- this.logger.info(`✅ WDA session recreated successfully: ${newSessionId.substring(0, 8)}...`);
2175
- return true;
2176
- }
2177
- } catch (error) {
2178
- this.logger.error(`❌ Failed to recreate WDA session: ${error.message}`);
2179
- throw error;
2180
- }
2181
-
2182
- return false;
2183
- }
2184
-
2185
- parsePageSourceFast(pageSource) {
2186
- // Parse XML page source quickly - focus on truly interactive elements
2187
- const elements = [];
2188
- // Focus on most reliable interactive elements for speed and accuracy
2189
- const xmlRegex = /<XCUIElementType(Button|TextField|Switch|Link|StaticText)[^>]*>/g;
2190
-
2191
- let match;
2192
- while ((match = xmlRegex.exec(pageSource)) !== null) {
2193
- const elementTag = match[0];
2194
-
2195
- // Extract attributes
2196
- const xMatch = elementTag.match(/x="([^"]+)"/);
2197
- const yMatch = elementTag.match(/y="([^"]+)"/);
2198
- const widthMatch = elementTag.match(/width="([^"]+)"/);
2199
- const heightMatch = elementTag.match(/height="([^"]+)"/);
2200
- const labelMatch = elementTag.match(/label="([^"]+)"/);
2201
- const nameMatch = elementTag.match(/name="([^"]+)"/);
2202
- const enabledMatch = elementTag.match(/enabled="([^"]+)"/);
2203
-
2204
- if (xMatch && yMatch && widthMatch && heightMatch) {
2205
- const x = parseFloat(xMatch[1]);
2206
- const y = parseFloat(yMatch[1]);
2207
- const width = parseFloat(widthMatch[1]);
2208
- const height = parseFloat(heightMatch[1]);
2209
- const label = labelMatch?.[1] || nameMatch?.[1] || `${match[1]}_${Math.round(x)}_${Math.round(y)}`;
2210
- const enabled = enabledMatch?.[1] !== 'false';
2211
- const elementType = `XCUIElementType${match[1]}`;
2212
-
2213
- // Fast filtering - keep it simple for speed
2214
- if (width > 8 && height > 8 && enabled) {
2215
- // Quick relevance check for speed
2216
- if (elementType.includes('Button') || elementType.includes('TextField') ||
2217
- elementType.includes('Switch') || elementType.includes('Link')) {
2218
- // Always include truly interactive elements
2219
- elements.push({
2220
- type: elementType,
2221
- label: label,
2222
- name: label,
2223
- accessibilityId: label,
2224
- x: Math.round(x),
2225
- y: Math.round(y),
2226
- width: Math.round(width),
2227
- height: Math.round(height),
2228
- centerX: Math.round(x + width / 2),
2229
- centerY: Math.round(y + height / 2)
2230
- });
2231
- } else if (elementType.includes('StaticText') && width > 30 && height > 15 && label && label.length > 2) {
2232
- // Only include meaningful StaticText elements
2233
- elements.push({
2234
- type: elementType,
2235
- label: label,
2236
- name: label,
2237
- accessibilityId: label,
2238
- x: Math.round(x),
2239
- y: Math.round(y),
2240
- width: Math.round(width),
2241
- height: Math.round(height),
2242
- centerX: Math.round(x + width / 2),
2243
- centerY: Math.round(y + height / 2)
2244
- });
2245
- }
2246
- }
2247
- }
2248
- }
2249
-
2250
- return elements;
2251
- }
2252
-
2253
- findClosestElement(x, y) {
2254
- if (this.cachedElements.length === 0) return null;
2255
-
2256
- // FAST STRATEGY: Find elements containing the click point
2257
- const containingElements = this.cachedElements.filter(elem => {
2258
- return x >= elem.x && x <= elem.x + elem.width &&
2259
- y >= elem.y && y <= elem.y + elem.height;
2260
- });
2261
-
2262
- // If we found elements containing the point, return the smallest one (most specific)
2263
- if (containingElements.length > 0) {
2264
- // Sort by area (smallest first = most specific element)
2265
- containingElements.sort((a, b) => (a.width * a.height) - (b.width * b.height));
2266
- const bestElement = containingElements[0];
2267
-
2268
- // Build strategies quickly - defer sibling detection
2269
- bestElement.locatorStrategies = [];
2270
- this.buildFastLocatorStrategies(bestElement);
2271
-
2272
- return bestElement;
2273
- }
2274
-
2275
- // ENHANCED STRATEGY: Search for nearby elements with expanded tolerance
2276
- const nearbyElements = this.cachedElements.filter(elem => {
2277
- const dx = elem.centerX - x;
2278
- const dy = elem.centerY - y;
2279
- const distance = Math.sqrt(dx * dx + dy * dy);
2280
- return distance <= 80; // Increased from 60px for better coverage
2281
- });
2282
-
2283
- if (nearbyElements.length > 0) {
2284
- // Enhanced sorting with sibling text detection
2285
- nearbyElements.forEach(elem => {
2286
- const dx = elem.centerX - x;
2287
- const dy = elem.centerY - y;
2288
- elem.distance = Math.sqrt(dx * dx + dy * dy);
2289
-
2290
- // Smart priority boosts
2291
- if (elem.label && elem.label !== 'Unlabeled' && elem.label.length > 2) {
2292
- elem.distance *= 0.7; // Strong preference for labeled elements
2293
- }
2294
- if (elem.type.includes('Button')) {
2295
- elem.distance *= 0.8; // Prefer buttons
2296
- }
2297
- if (elem.type.includes('Cell')) {
2298
- elem.distance *= 0.85; // Table cells are often clickable
2299
- }
2300
-
2301
- // GENIUS: Look for nearby text elements that could describe this element
2302
- const nearbyText = this.cachedElements.find(textElem => {
2303
- if (textElem.type === 'XCUIElementTypeStaticText' && textElem.label && textElem.label !== 'Unlabeled') {
2304
- const textDistance = Math.sqrt(
2305
- Math.pow(textElem.centerX - elem.centerX, 2) +
2306
- Math.pow(textElem.centerY - elem.centerY, 2)
2307
- );
2308
- return textDistance < 60; // Text within 60px could be related
2309
- }
2310
- return false;
2311
- });
2312
-
2313
- if (nearbyText) {
2314
- elem.associatedText = nearbyText.label;
2315
- elem.distance *= 0.6; // Big boost for elements with associated text
2316
- }
2317
- });
2318
-
2319
- nearbyElements.sort((a, b) => a.distance - b.distance);
2320
- const bestNearby = nearbyElements[0];
2321
-
2322
- // Build enhanced strategies with sibling info
2323
- bestNearby.locatorStrategies = [];
2324
- this.buildEnhancedLocatorStrategies(bestNearby);
2325
-
2326
- return bestNearby;
2327
- }
2328
-
2329
- return null;
2330
- }
2331
-
2332
- // Enhanced strategy building with sibling detection for better accuracy
2333
- buildEnhancedLocatorStrategies(element) {
2334
- const strategies = [];
2335
-
2336
- // Strategy 1: Direct element locators (highest priority)
2337
- if (element.accessibilityId && element.accessibilityId !== 'Unlabeled') {
2338
- strategies.push({
2339
- strategy: 'accessibility id',
2340
- value: element.accessibilityId,
2341
- priority: 1
2342
- });
2343
- }
2344
-
2345
- if (element.label && element.label !== 'Unlabeled' && element.label !== element.accessibilityId) {
2346
- strategies.push({
2347
- strategy: 'name',
2348
- value: element.label,
2349
- priority: 2
2350
- });
2351
- }
2352
-
2353
- // Strategy 2: Use associated text if available (very good accuracy)
2354
- if (element.associatedText) {
2355
- strategies.push({
2356
- strategy: 'name',
2357
- value: element.associatedText,
2358
- priority: 1.5 // Between accessibility id and direct label
2359
- });
2360
-
2361
- // Also try accessibility id with associated text
2362
- strategies.push({
2363
- strategy: 'accessibility id',
2364
- value: element.associatedText,
2365
- priority: 1.5
2366
- });
2367
- }
2368
-
2369
- // Strategy 3: Class name (lower priority)
2370
- if (element.type) {
2371
- strategies.push({
2372
- strategy: 'class name',
2373
- value: element.type,
2374
- priority: 5
2375
- });
2376
- }
2377
-
2378
- // Sort by priority (lower number = higher priority)
2379
- strategies.sort((a, b) => a.priority - b.priority);
2380
-
2381
- // Store the locator strategies on the element
2382
- element.locatorStrategies = strategies;
2383
-
2384
- return strategies;
2385
- }
2386
-
2387
- // Enhanced strategy building with sibling detection for better accuracy
2388
- buildFastLocatorStrategies(element) {
2389
- const strategies = [];
2390
-
2391
- // Strategy 1: Direct element locators
2392
-
2393
- // Strategy 1: Direct element locators
2394
- if (element.accessibilityId && element.accessibilityId !== 'Unlabeled') {
2395
- strategies.push({
2396
- strategy: 'accessibility id',
2397
- value: element.accessibilityId,
2398
- priority: 1
2399
- });
2400
- }
2401
-
2402
- if (element.label && element.label !== 'Unlabeled' && element.label !== element.accessibilityId) {
2403
- strategies.push({
2404
- strategy: 'name',
2405
- value: element.label,
2406
- priority: 2
2407
- });
2408
- }
2409
-
2410
- // Strategy 2: Class name (lower priority)
2411
- if (element.type) {
2412
- strategies.push({
2413
- strategy: 'class name',
2414
- value: element.type,
2415
- priority: 5
2416
- });
2417
- }
2418
-
2419
- // Strategy 3: Sibling-based strategies for better accuracy
2420
- const siblingStrategies = this.buildSiblingStrategies(element);
2421
- strategies.push(...siblingStrategies);
2422
-
2423
- element.locatorStrategies = strategies;
2424
- }
2425
-
2426
- // Build sibling-based locator strategies
2427
- buildSiblingStrategies(element) {
2428
- const strategies = [];
2429
-
2430
- // Find nearby labeled elements that can help identify this element
2431
- const nearbyElements = this.cachedElements.filter(elem => {
2432
- if (elem === element) return false;
2433
-
2434
- const dx = Math.abs(elem.centerX - element.centerX);
2435
- const dy = Math.abs(elem.centerY - element.centerY);
2436
- const distance = Math.sqrt(dx * dx + dy * dy);
2437
-
2438
- // Look for nearby elements with good labels
2439
- return distance <= 100 && elem.label && elem.label !== 'Unlabeled' && elem.label.length > 2;
2440
- });
2441
-
2442
- // Create sibling-based strategies
2443
- for (const sibling of nearbyElements.slice(0, 3)) { // Limit to 3 for performance
2444
- if (sibling.accessibilityId && sibling.accessibilityId !== 'Unlabeled') {
2445
- // Create XPath strategy using sibling
2446
- const xpath = this.buildSiblingXPath(element, sibling);
2447
- if (xpath) {
2448
- strategies.push({
2449
- strategy: 'xpath',
2450
- value: xpath,
2451
- priority: 3,
2452
- siblingBased: true,
2453
- siblingLabel: sibling.label
2454
- });
2455
- }
2456
- }
2457
- }
2458
-
2459
- return strategies;
2460
- }
2461
-
2462
- // Build XPath using sibling elements
2463
- buildSiblingXPath(targetElement, siblingElement) {
2464
- try {
2465
- // Simple sibling-based XPath strategies
2466
- if (siblingElement.accessibilityId) {
2467
- // Target element following sibling
2468
- const siblingClass = siblingElement.type.replace('XCUIElementType', '');
2469
- const targetClass = targetElement.type.replace('XCUIElementType', '');
2470
-
2471
- // Try "following" axis
2472
- return `//XCUIElementType${siblingClass}[@name="${siblingElement.accessibilityId}"]/following::XCUIElementType${targetClass}[1]`;
2473
- }
2474
- } catch (error) {
2475
- this.logger.debug(`Sibling XPath creation failed: ${error.message}`);
2476
- }
2477
-
2478
- return null;
2479
- }
2480
-
2481
- // Enhanced strategy building with siblings - called only when broadcasting to followers
2482
- buildEnhancedLocatorStrategies(element, clickX, clickY) {
2483
- // Start with fast strategies
2484
- this.buildFastLocatorStrategies(element);
2485
-
2486
- // Add sibling strategies only if we don't have good direct strategies
2487
- const hasGoodDirectStrategy = element.locatorStrategies.some(s =>
2488
- s.strategy === 'accessibility id' || (s.strategy === 'name' && s.value.length > 3)
2489
- );
2490
-
2491
- if (!hasGoodDirectStrategy) {
2492
- // Only do expensive sibling detection when needed
2493
- const siblings = this.findSiblingElements(element, clickX, clickY);
2494
-
2495
- siblings.forEach((sibling, index) => {
2496
- const siblingLabel = sibling.element.label;
2497
- if (siblingLabel && siblingLabel !== 'Unlabeled' && siblingLabel.length > 2) {
2498
-
2499
- // Simple sibling strategies (avoid complex XPath)
2500
- element.locatorStrategies.push({
2501
- strategy: 'accessibility id',
2502
- value: siblingLabel,
2503
- priority: 4 + index,
2504
- siblingBased: true,
2505
- siblingLabel: siblingLabel
2506
- });
2507
-
2508
- // Only add XPath for very short, common labels
2509
- if (siblingLabel.length < 15) {
2510
- element.locatorStrategies.push({
2511
- strategy: 'xpath',
2512
- value: `//*[contains(@label, '${siblingLabel}')]`,
2513
- priority: 6 + index,
2514
- siblingBased: true,
2515
- siblingLabel: siblingLabel
2516
- });
2517
- }
2518
- }
2519
- });
2520
-
2521
- // Log sibling enhancement
2522
- const siblingStrategies = element.locatorStrategies.filter(s => s.siblingBased);
2523
- if (siblingStrategies.length > 0) {
2524
- this.logger.info(`🔗 Enhanced with ${siblingStrategies.length} sibling strategies: ${siblingStrategies.map(s => s.siblingLabel).join(', ')}`);
2525
- }
2526
- }
2527
-
2528
- // Limit to top 4 strategies for speed
2529
- element.locatorStrategies = element.locatorStrategies
2530
- .sort((a, b) => a.priority - b.priority)
2531
- .slice(0, 4);
2532
- }
2533
-
2534
- // Find sibling elements that might provide better locators
2535
- findSiblingElements(element, clickX, clickY) {
2536
- const siblings = [];
2537
- const searchRadius = 100; // Search area around the element
2538
-
2539
- // Look for text elements near the clicked element
2540
- this.cachedElements.forEach(candidate => {
2541
- if (candidate === element) return; // Skip self
2542
-
2543
- // Calculate distance between elements (not click point)
2544
- const dx = Math.abs(candidate.centerX - element.centerX);
2545
- const dy = Math.abs(candidate.centerY - element.centerY);
2546
- const distance = Math.sqrt(dx * dx + dy * dy);
2547
-
2548
- if (distance <= searchRadius) {
2549
- // Prefer StaticText elements with meaningful labels
2550
- if (candidate.type.includes('StaticText') &&
2551
- candidate.label &&
2552
- candidate.label !== 'Unlabeled' &&
2553
- candidate.label.length > 2) {
2554
-
2555
- // Check if it's likely a label for this element (vertical or horizontal alignment)
2556
- const isVerticallyAligned = Math.abs(candidate.centerX - element.centerX) < 30;
2557
- const isHorizontallyAligned = Math.abs(candidate.centerY - element.centerY) < 20;
2558
- const isNearby = distance < 60;
2559
-
2560
- if ((isVerticallyAligned || isHorizontallyAligned) && isNearby) {
2561
- siblings.push({
2562
- element: candidate,
2563
- distance: distance,
2564
- relationship: isVerticallyAligned ? 'vertical' : 'horizontal'
2565
- });
2566
- }
2567
- }
2568
- }
2569
- });
2570
-
2571
- // Sort siblings by distance and return best ones
2572
- return siblings.sort((a, b) => a.distance - b.distance).slice(0, 3);
2573
- }
2574
-
2575
- async executeOnFollowerWithLocators(follower, locators, fallbackX, fallbackY) {
2576
- // Use session lock to prevent concurrent WDA access
2577
- return this.withSessionLock(follower.name, async () => {
2578
- return this._executeOnFollowerWithLocators(follower, locators, fallbackX, fallbackY);
2579
- });
2580
- }
2581
-
2582
- async _executeOnFollowerWithLocators(follower, locators, fallbackX, fallbackY) {
2583
- const axios = require('axios');
2584
- const followerPort = follower.port || 8100;
2585
-
2586
- // OPTIMIZATION: Build enhanced strategies with siblings only when needed for followers
2587
- if (locators && locators.element) {
2588
- // Start with fast strategies only
2589
- this.buildFastLocatorStrategies(locators.element);
2590
- // Update locators with basic strategies first
2591
- locators.locatorStrategies = locators.element.locatorStrategies;
2592
- }
2593
-
2594
- this.logger.info('');
2595
- this.logger.info(` 📲 Processing follower: ${follower.name}`);
2596
-
2597
- // GENIUS: Use global session approach - get session from connectedDevices
2598
- let followerSessionId = null;
2599
-
2600
- // First, try to get sessionId from the follower object itself (may be wdaSessionId for iOS)
2601
- if (follower.wdaSessionId) {
2602
- followerSessionId = follower.wdaSessionId;
2603
- this.logger.info(` ✓ Session found on follower object: ${followerSessionId.substring(0, 8)}...`);
2604
- } else if (follower.sessionId) {
2605
- followerSessionId = follower.sessionId;
2606
- this.logger.info(` ✓ Session found on follower object: ${followerSessionId.substring(0, 8)}...`);
2607
- }
2608
-
2609
- // If not found on follower object, find the follower in global connectedDevices array
2610
- if (!followerSessionId) {
2611
- this.logger.info(` - Checking global connectedDevices...`);
2612
- const globalDevice = this.connectedDevices.find(d => d.name === follower.name || d.udid === follower.udid);
2613
- if (globalDevice) {
2614
- // Check both sessionId (Android) and wdaSessionId (iOS)
2615
- followerSessionId = globalDevice.wdaSessionId || globalDevice.sessionId;
2616
- if (followerSessionId) {
2617
- this.logger.info(` ✓ Session found in global devices: ${followerSessionId.substring(0, 8)}...`);
2618
- }
2619
- }
2620
- }
2621
-
2622
- // If no global session, try to get from WDA status endpoint
2623
- if (!followerSessionId) {
2624
- this.logger.info(` - Querying WDA /status endpoint on port ${followerPort}...`);
2625
- try {
2626
- const followerStatusRes = await axios.get(`http://localhost:${followerPort}/status`, { timeout: 2000 });
2627
- followerSessionId = followerStatusRes.data?.sessionId || followerStatusRes.data?.value?.sessionId;
2628
-
2629
- // If we found a session, store it globally for reuse
2630
- if (followerSessionId) {
2631
- this.logger.info(` ✓ Session obtained from /status: ${followerSessionId.substring(0, 8)}...`);
2632
- const globalDevice = this.connectedDevices.find(d => d.name === follower.name);
2633
- if (globalDevice) {
2634
- if (follower.platform === 'ios') {
2635
- globalDevice.wdaSessionId = followerSessionId;
2636
- } else {
2637
- globalDevice.sessionId = followerSessionId;
2638
- }
2639
- }
2640
- }
2641
- } catch (error) {
2642
- this.logger.error(` ✗ Failed to get session from /status: ${error.message}`);
2643
- return;
2644
- }
2645
- }
2646
-
2647
- if (!followerSessionId) {
2648
- this.logger.warn(` ⚠️ No active session for follower ${follower.name}, attempting to create one...`);
2649
-
2650
- // Last resort: Try to create a new session for the follower
2651
- try {
2652
- this.logger.info(` 🔧 Creating new WDA session for follower ${follower.name}...`);
2653
- const sessionResponse = await axios.post(`http://localhost:${followerPort}/session`, {
2654
- capabilities: {
2655
- alwaysMatch: {},
2656
- firstMatch: [{}]
2657
- }
2658
- }, { timeout: 10000 });
2659
-
2660
- followerSessionId = sessionResponse.data?.sessionId || sessionResponse.data?.value?.sessionId;
2661
- if (followerSessionId) {
2662
- this.logger.info(` ✅ Created new session for follower: ${followerSessionId.substring(0, 8)}...`);
2663
-
2664
- // Store the session
2665
- if (follower.platform === 'ios') {
2666
- follower.wdaSessionId = followerSessionId;
2667
- } else {
2668
- follower.sessionId = followerSessionId;
2669
- }
2670
-
2671
- // Store in global connectedDevices and session manager
2672
- const globalDevice = this.connectedDevices.find(d => d.name === follower.name);
2673
- if (globalDevice) {
2674
- if (follower.platform === 'ios') {
2675
- globalDevice.wdaSessionId = followerSessionId;
2676
- } else {
2677
- globalDevice.sessionId = followerSessionId;
2678
- }
2679
- }
2680
-
2681
- // Register with global session manager if available
2682
- if (global.sessionManager && follower.udid) {
2683
- global.sessionManager.registerSession(follower.udid, followerSessionId, followerPort, follower.name, null, follower.platform);
2684
- }
2685
- } else {
2686
- this.logger.error(` ✗ Failed to create session - no sessionId in response`);
2687
- return;
2688
- }
2689
- } catch (createError) {
2690
- this.logger.error(` ✗ Failed to create session for follower: ${createError.message}`);
2691
- return;
2692
- }
2693
- }
2694
-
2695
- this.logger.info(` 📱 Using WDA session: ${followerSessionId.substring(0, 8)}... on port ${followerPort}`);
2696
-
2697
- // Validate session is still active before trying locators
2698
- try {
2699
- this.logger.info(` ✓ Validating WDA session is active...`);
2700
- const statusCheck = await axios.get(`http://localhost:${followerPort}/status`, { timeout: 2000 });
2701
- if (statusCheck.data?.sessionId !== followerSessionId) {
2702
- this.logger.warn(` ⚠️ Session ID mismatch - WDA has different session now`);
2703
- this.logger.warn(` Expected: ${followerSessionId.substring(0, 8)}...`);
2704
- this.logger.warn(` Current: ${statusCheck.data?.sessionId?.substring(0, 8)}...`);
2705
-
2706
- // Update to the current active session
2707
- followerSessionId = statusCheck.data?.sessionId;
2708
- if (follower.platform === 'ios') {
2709
- follower.wdaSessionId = followerSessionId;
2710
- } else {
2711
- follower.sessionId = followerSessionId;
2712
- }
2713
- this.logger.info(` ✓ Updated to current session: ${followerSessionId.substring(0, 8)}...`);
2714
-
2715
- // Try to sync follower to same app as commander
2716
- try {
2717
- // Get commander's current app
2718
- const commanderPort = this.commanderDevice.port || 8100;
2719
- const commanderSessionId = this.getWDASessionId();
2720
-
2721
- if (commanderSessionId) {
2722
- const commanderAppRes = await axios.get(`http://localhost:${commanderPort}/session/${commanderSessionId}/wda/activeAppInfo`, { timeout: 3000 });
2723
- const commanderAppId = commanderAppRes.data?.value?.bundleId;
2724
-
2725
- if (commanderAppId && commanderAppId !== 'com.apple.springboard') {
2726
- this.logger.info(` 🔄 Syncing follower to commander app: ${commanderAppId}`);
2727
- await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/wda/apps/launch`, {
2728
- bundleId: commanderAppId
2729
- }, { timeout: 20000 }); // Increased from 15s to 20s for app launch
2730
-
2731
- // Wait longer for app to load and stabilize
2732
- await new Promise(resolve => setTimeout(resolve, 3000)); // Increased from 1.5s to 3s
2733
- this.logger.info(` ✅ Follower synced to ${commanderAppId}`);
2734
- } else {
2735
- this.logger.info(` ℹ️ Commander in home screen, not syncing follower app`);
2736
- }
2737
- }
2738
- } catch (syncError) {
2739
- this.logger.warn(` ⚠️ Could not sync follower app: ${syncError.message}`);
2740
- }
2741
- }
2742
- } catch (statusErr) {
2743
- this.logger.warn(` ⚠️ Could not validate session: ${statusErr.message}`);
2744
- // Continue anyway - maybe the /status endpoint is not available
2745
- }
2746
-
2747
- let tapExecuted = false;
2748
- let sessionInvalid = false;
2749
-
2750
- // Try locator strategies in priority order
2751
- if (locators?.locatorStrategies && locators.locatorStrategies.length > 0) {
2752
- // Sort by priority
2753
- const strategies = [...locators.locatorStrategies].sort((a, b) => a.priority - b.priority);
2754
-
2755
- this.logger.info(` 🔎 Trying ${strategies.length} locator strategies:`);
2756
-
2757
- for (const strategy of strategies) {
2758
- if (tapExecuted) break;
2759
-
2760
- try {
2761
- const strategyDesc = strategy.siblingBased ?
2762
- `${strategy.strategy}="${strategy.value}" (sibling: "${strategy.siblingLabel}")` :
2763
- `${strategy.strategy}="${strategy.value}"`;
2764
- this.logger.info(` [${strategy.priority}] Trying: ${strategyDesc}`);
2765
-
2766
- const elementRes = await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element`, {
2767
- using: strategy.strategy,
2768
- value: strategy.value
2769
- }, { timeout: 4000 }); // Slightly increased for XPath queries
2770
-
2771
- if (elementRes.data?.value) {
2772
- const elementId = elementRes.data.value.ELEMENT || elementRes.data.value['element-6066-11e4-a52e-4f735466cecf'];
2773
- const successMsg = strategy.siblingBased ?
2774
- `Element found via sibling "${strategy.siblingLabel}": ${elementId.substring(0, 16)}...` :
2775
- `Element found: ${elementId.substring(0, 16)}...`;
2776
- this.logger.info(` ✓ ${successMsg}`);
2777
-
2778
- this.logger.info(` ⏳ Executing click...`);
2779
- await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element/${elementId}/click`, {}, { timeout: 3000 });
2780
-
2781
- const clickSuccessMsg = strategy.siblingBased ?
2782
- `SUCCESS: Follower ${follower.name} tapped using sibling strategy (${strategy.siblingLabel})` :
2783
- `SUCCESS: Follower ${follower.name} tapped using ${strategy.strategy}`;
2784
- this.logger.info(` ✅ ${clickSuccessMsg}`);
2785
- tapExecuted = true;
2786
- }
2787
- } catch (error) {
2788
- // Check if session is invalid (404)
2789
- if (error.response?.status === 404) {
2790
- sessionInvalid = true;
2791
- this.logger.warn(` ✗ Session invalid (404 error)`);
2792
- this.logger.warn(` ✗ WDA endpoint returned 404 - session may have expired`);
2793
- this.logger.debug(` Response: ${JSON.stringify(error.response?.data)?.substring(0, 200)}`);
2794
- break; // Stop trying strategies, move to fallback
2795
- } else if (error.response?.status === 400) {
2796
- // Bad request - strategy/value might be invalid
2797
- this.logger.warn(` ✗ Bad request (400) - strategy or value format invalid`);
2798
- this.logger.debug(` Response: ${JSON.stringify(error.response?.data)?.substring(0, 200)}`);
2799
- } else if (error.code === 'ECONNREFUSED') {
2800
- // Can't reach WDA server
2801
- this.logger.error(` ✗ ECONNREFUSED - WDA server not responding on port ${followerPort}`);
2802
- sessionInvalid = true;
2803
- break;
2804
- } else {
2805
- this.logger.debug(` ✗ ${strategy.strategy} strategy failed: ${error.message}`);
2806
- }
2807
- }
2808
- }
2809
- } else if (locators && (locators.accessibilityId || locators.label)) {
2810
- // Fallback for old format
2811
- const strategy = locators.accessibilityId ? 'accessibility id' : 'name';
2812
- const value = locators.accessibilityId || locators.label;
2813
-
2814
- this.logger.info(` 🔎 Using fallback locator: ${strategy}="${value}"`);
2815
-
2816
- try {
2817
- const elementRes = await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element`, {
2818
- using: strategy,
2819
- value: value
2820
- }, { timeout: 3000 });
2821
-
2822
- if (elementRes.data?.value) {
2823
- const elementId = elementRes.data.value.ELEMENT || elementRes.data.value['element-6066-11e4-a52e-4f735466cecf'];
2824
- this.logger.info(` ✓ Element found: ${elementId.substring(0, 16)}...`);
2825
-
2826
- this.logger.info(` ⏳ Executing click...`);
2827
- await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element/${elementId}/click`, {}, { timeout: 3000 });
2828
- this.logger.info(` ✅ Tap executed!`);
2829
- this.logger.info(` ✅ SUCCESS: Follower ${follower.name} tapped using ${strategy}`);
2830
- tapExecuted = true;
2831
- }
2832
- } catch (error) {
2833
- // Check if session is invalid (404)
2834
- if (error.response?.status === 404) {
2835
- sessionInvalid = true;
2836
- this.logger.warn(` ✗ Session invalid (404), will use coordinate fallback`);
2837
- } else {
2838
- this.logger.debug(` ✗ Locator-based tap failed: ${error.message}`);
2839
- }
2840
- }
2841
- } else {
2842
- this.logger.info(` ℹ️ No locator strategies available, will use coordinate fallback`);
2843
- }
2844
-
2845
- // FALLBACK: If no locator worked and we have element context, try enhanced strategies
2846
- if (!tapExecuted && !sessionInvalid && locators?.element && locators.locatorStrategies && locators.locatorStrategies.length > 0) {
2847
- this.logger.info(` 🔄 Primary locators failed, trying enhanced sibling strategies...`);
2848
-
2849
- // Build enhanced strategies with siblings
2850
- this.buildEnhancedLocatorStrategies(locators.element, fallbackX, fallbackY);
2851
- const enhancedStrategies = locators.element.locatorStrategies.filter(s => s.siblingBased);
2852
-
2853
- if (enhancedStrategies.length > 0) {
2854
- this.logger.info(` 🔗 Trying ${enhancedStrategies.length} sibling-based strategies...`);
2855
-
2856
- for (const strategy of enhancedStrategies.slice(0, 2)) { // Limit to 2 for speed
2857
- if (tapExecuted) break;
2858
-
2859
- try {
2860
- const elementRes = await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element`, {
2861
- using: strategy.strategy,
2862
- value: strategy.value
2863
- }, { timeout: 3000 });
2864
-
2865
- if (elementRes.data?.value) {
2866
- const elementId = elementRes.data.value.ELEMENT || elementRes.data.value['element-6066-11e4-a52e-4f735466cecf'];
2867
- this.logger.info(` ✅ Sibling strategy worked: ${strategy.siblingLabel}`);
2868
-
2869
- await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element/${elementId}/click`, {}, { timeout: 3000 });
2870
- this.logger.info(` ✅ SUCCESS: Enhanced locator tap executed on ${follower.name}`);
2871
- tapExecuted = true;
2872
- }
2873
- } catch (enhancedErr) {
2874
- this.logger.debug(` ✗ Enhanced strategy failed: ${enhancedErr.message}`);
2875
- }
2876
- }
2877
- }
2878
- }
2879
-
2880
- if (!tapExecuted) {
2881
- try {
2882
- // GENIUS: Smart session recovery - avoid recreating sessions unless absolutely necessary
2883
- if (sessionInvalid) {
2884
- this.logger.info(` 🔄 Session was invalid (404) - MUST recover before coordinate tap`);
2885
-
2886
- // Step 1: Check if WDA has any active session we can use
2887
- try {
2888
- const statusRes = await axios.get(`http://localhost:${followerPort}/status`, { timeout: 2000 });
2889
- const activeSession = statusRes.data?.sessionId;
2890
-
2891
- if (activeSession && activeSession !== followerSessionId) {
2892
- // WDA has a different session - update our reference
2893
- this.logger.info(` ✅ Found active WDA session: ${activeSession.substring(0, 8)}...`);
2894
- followerSessionId = activeSession;
2895
-
2896
- // Update follower object
2897
- if (follower.platform === 'ios') {
2898
- follower.wdaSessionId = activeSession;
2899
- } else {
2900
- follower.sessionId = activeSession;
2901
- }
2902
-
2903
- // Update global session manager
2904
- if (global.sessionManager) {
2905
- global.sessionManager.updateSession(follower.udid, activeSession);
2906
- }
2907
-
2908
- this.logger.info(` ✅ Session recovered without recreation: ${activeSession.substring(0, 8)}...`);
2909
-
2910
- // Quick test to make sure it works
2911
- await axios.get(`http://localhost:${followerPort}/session/${activeSession}/window/size`, { timeout: 2000 });
2912
- this.logger.debug(` ✓ Session validated and ready`);
2913
-
2914
- } else if (!activeSession) {
2915
- this.logger.error(` ✗ Cannot proceed - WDA has no active session and creation would close apps`);
2916
- this.logger.warn(` 🚫 Skipping ${follower.name} to prevent app disruption`);
2917
- return;
2918
- } else {
2919
- this.logger.debug(` ℹ️ Session ID matches, session should be valid now`);
2920
- }
2921
- } catch (statusError) {
2922
- this.logger.error(` ✗ Cannot proceed - session is still invalid and WDA status failed`);
2923
- this.logger.warn(` 🚫 Skipping ${follower.name} to prevent further issues`);
2924
- return;
2925
- }
2926
- }
2927
-
2928
- // Coordinate fallback after session recovery
2929
- if (!tapExecuted && followerSessionId) {
2930
- // Simple coordinate validation for follower
2931
- let safeFallbackX = Math.round(fallbackX);
2932
- let safeFallbackY = Math.round(fallbackY);
2933
-
2934
- const deviceHeight = 896; // iPhone default
2935
- const deviceWidth = 414;
2936
-
2937
- // Basic bounds checking only
2938
- if (safeFallbackX < 0 || safeFallbackX > deviceWidth || safeFallbackY < 0 || safeFallbackY > deviceHeight) {
2939
- this.logger.warn(`⚠️ FOLLOWER: Coordinates outside device bounds, clamping: (${safeFallbackX}, ${safeFallbackY})`);
2940
- safeFallbackX = Math.max(10, Math.min(safeFallbackX, deviceWidth - 10));
2941
- safeFallbackY = Math.max(50, Math.min(safeFallbackY, deviceHeight - 100));
2942
- }
2943
-
2944
- this.logger.info(` 🎯 Fallback: Using coordinate-based tap at (${safeFallbackX}, ${safeFallbackY})`);
2945
- this.logger.info(` Session: ${followerSessionId.substring(0, 8)}...`);
2946
-
2947
- try {
2948
- await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/actions`, {
2949
- actions: [{
2950
- type: 'pointer',
2951
- id: 'finger1',
2952
- parameters: { pointerType: 'touch' },
2953
- actions: [
2954
- { type: 'pointerMove', duration: 0, x: safeFallbackX, y: safeFallbackY },
2955
- { type: 'pointerDown', button: 0 },
2956
- { type: 'pause', duration: 100 },
2957
- { type: 'pointerUp', button: 0 }
2958
- ]
2959
- }]
2960
- }, { timeout: 15000 });
2961
- this.logger.info(` ✅ SUCCESS: Fallback tap executed on ${follower.name}`);
2962
- } catch (fallbackErr) {
2963
- if (fallbackErr.response?.status === 404) {
2964
- this.logger.error(` ✗ FAILED: Coordinate tap got 404 - session is still invalid`);
2965
- this.logger.error(` Port: ${followerPort}, Session: ${followerSessionId.substring(0, 8)}...`);
2966
- } else {
2967
- this.logger.error(` ✗ FAILED: Coordinate tap failed: ${fallbackErr.message}`);
2968
- }
2969
- }
2970
- }
2971
- } catch (error) {
2972
- this.logger.error(` ✗ FAILED: Follower ${follower.name} tap execution failed: ${error.message}`);
2973
- }
2974
- }
2975
- }
2976
-
2977
- async addFollower(deviceName, deviceInfo) {
2978
- if (!this.isActive) {
2979
- throw new Error('Commander mode is not active');
2980
- }
2981
-
2982
- // FIXED: Prevent duplicate followers - check if follower already exists
2983
- const existingFollower = Array.from(this.followerDevices).find(f => f.name === deviceName);
2984
- if (existingFollower) {
2985
- this.logger.info(`✓ Follower ${deviceName} already exists, skipping duplicate addition`);
2986
- return {
2987
- success: true,
2988
- follower: deviceName,
2989
- totalFollowers: this.followerDevices.size,
2990
- isDuplicate: true
2991
- };
2992
- }
2993
-
2994
- const followerDevice = {
2995
- name: deviceName,
2996
- ...deviceInfo
2997
- };
2998
-
2999
- // For Android devices, ensure session info is loaded and cache screenshot URL
3000
- if (deviceInfo.platform === 'android') {
3001
- // GENIUS: Use sessionManager for reliable session lookup
3002
- const session = global.sessionManager?.getSession(followerDevice) || global.androidSessions?.get(deviceInfo.udid);
3003
-
3004
- if (session && session.port && session.sessionId) {
3005
- followerDevice.port = session.port;
3006
- followerDevice.sessionId = session.sessionId;
3007
-
3008
- // Pre-cache screenshot URL for instant mirroring
3009
- followerDevice._cachedPort = session.port;
3010
- followerDevice._cachedScreenshotUrl = `http://localhost:${session.port}/wd/hub/session/${session.sessionId}/screenshot`;
3011
-
3012
- this.logger.info(`✅ Android follower session found and cached: port ${session.port}, sessionId ${session.sessionId.substring(0, 8)}...`);
3013
- } else {
3014
- this.logger.warn(`⚠️ No UIAutomator2 session found for Android follower ${deviceName}`);
3015
- }
3016
- }
3017
- // For iOS devices, check for existing WDA session and cache screenshot URL
3018
- else if (deviceInfo.platform === 'ios') {
3019
- await this.checkFollowerWDASession(followerDevice);
3020
-
3021
- // Pre-cache iOS screenshot URL if port is available
3022
- if (followerDevice.port) {
3023
- followerDevice._cachedPort = followerDevice.port;
3024
- followerDevice._cachedScreenshotUrl = `http://localhost:${followerDevice.port}/screenshot`;
3025
- this.logger.info(`✅ iOS follower screenshot URL cached: ${followerDevice.name}:${followerDevice.port}`);
3026
- }
3027
- }
3028
-
3029
- this.followerDevices.add(followerDevice);
3030
- this.logger.info(`Follower added: ${deviceName} (${deviceInfo.platform})`);
3031
- this.broadcastCommanderStatus();
3032
-
3033
- return {
3034
- success: true,
3035
- follower: deviceName,
3036
- totalFollowers: this.followerDevices.size
3037
- };
3038
- }
3039
-
3040
- async checkFollowerWDASession(followerDevice) {
3041
- const axios = require('axios');
3042
- const port = followerDevice.port || 8100;
3043
-
3044
- try {
3045
- this.logger.info(`Fast checking WDA session on follower ${followerDevice.name}...`);
3046
-
3047
- // Check /status for existing session with reduced timeout
3048
- const statusResponse = await axios.get(`http://localhost:${port}/status`, { timeout: 3000 }); // Reduced from 5000ms
3049
- const sessionId = statusResponse.data?.sessionId || statusResponse.data?.value?.sessionId;
3050
-
3051
- if (sessionId) {
3052
- followerDevice.wdaSessionId = sessionId;
3053
- followerDevice.port = port; // Ensure port is set
3054
- this.logger.info(`✅ Follower using existing session: ${sessionId}`);
3055
- return;
3056
- }
3057
-
3058
- // Try to get existing session from /sessions endpoint with reduced timeout
3059
- try {
3060
- const sessionsResponse = await axios.get(`http://localhost:${port}/sessions`, { timeout: 2000 }); // Reduced from 3000ms
3061
- const sessions = sessionsResponse.data?.value;
3062
-
3063
- if (sessions && Array.isArray(sessions) && sessions.length > 0) {
3064
- followerDevice.wdaSessionId = sessions[0].id;
3065
- followerDevice.port = port; // Ensure port is set
3066
- this.logger.info(`✅ Follower found session from /sessions: ${followerDevice.wdaSessionId}`);
3067
- return;
3068
- }
3069
- } catch (e) {
3070
- this.logger.debug('No sessions found at /sessions endpoint for follower');
3071
- }
3072
-
3073
- // Try alternative port quickly if first port failed
3074
- if (port === 8100) {
3075
- const altPort = 8101;
3076
- try {
3077
- this.logger.debug(`Trying alternative port ${altPort} for follower ${followerDevice.name}...`);
3078
- const altStatusResponse = await axios.get(`http://localhost:${altPort}/status`, { timeout: 2000 });
3079
- const altSessionId = altStatusResponse.data?.sessionId || altStatusResponse.data?.value?.sessionId;
3080
-
3081
- if (altSessionId) {
3082
- followerDevice.wdaSessionId = altSessionId;
3083
- followerDevice.port = altPort;
3084
- this.logger.info(`✅ Follower found session on alternative port ${altPort}: ${altSessionId}`);
3085
- return;
3086
- }
3087
- } catch (altError) {
3088
- this.logger.debug(`Alternative port ${altPort} also failed for follower`);
3089
- }
3090
- }
3091
-
3092
- this.logger.warn(`⚠️ No existing WDA session found for follower ${followerDevice.name}`);
3093
-
3094
- // OPTIMIZATION: Create session in background to avoid blocking UI
3095
- this.createFollowerSessionInBackground(followerDevice, port);
3096
-
3097
- this.logger.warn(`💡 Follower will work for screen mirroring when WDA becomes available`);
3098
-
3099
- } catch (error) {
3100
- this.logger.warn(`⚠️ Error checking follower WDA session: ${error.message}`);
3101
- this.logger.warn(`💡 Follower will work for screen mirroring when session becomes available`);
3102
- }
3103
- }
3104
-
3105
- /**
3106
- * Create follower WDA session in background without blocking startup
3107
- */
3108
- async createFollowerSessionInBackground(followerDevice, port) {
3109
- // Don't block the main thread - create session asynchronously
3110
- setTimeout(async () => {
3111
- try {
3112
- this.logger.info(`🔧 Creating WDA session for follower ${followerDevice.name} in background...`);
3113
-
3114
- const axios = require('axios');
3115
- const device = this.connectedDevices.find(d => d.name === followerDevice.name);
3116
- if (!device) {
3117
- throw new Error('Device not found');
3118
- }
3119
-
3120
- const sessionResponse = await axios.post(`http://localhost:${port}/session`, {
3121
- capabilities: {
3122
- alwaysMatch: {},
3123
- firstMatch: [{}]
3124
- }
3125
- }, { timeout: 10000 });
3126
-
3127
- const sessionId = sessionResponse.data?.sessionId || sessionResponse.data?.value?.sessionId;
3128
- if (!sessionId) {
3129
- throw new Error('No sessionId in response');
3130
- }
3131
-
3132
- // Update follower device with new session
3133
- followerDevice.wdaSessionId = sessionId;
3134
- followerDevice.port = port;
3135
-
3136
- // Pre-cache screenshot URL for instant mirroring
3137
- followerDevice._cachedPort = port;
3138
- followerDevice._cachedScreenshotUrl = `http://localhost:${port}/screenshot`;
3139
-
3140
- // Register with global session manager if available
3141
- if (global.sessionManager) {
3142
- global.sessionManager.registerSession(device.udid, sessionId, port, followerDevice.name, null, 'ios');
3143
- }
3144
-
3145
- // Store in global connectedDevices for persistence
3146
- const globalDevice = this.connectedDevices.find(d => d.name === followerDevice.name);
3147
- if (globalDevice) {
3148
- globalDevice.wdaSessionId = sessionId;
3149
- }
3150
-
3151
- this.logger.info(`✅ Background session created for follower: ${followerDevice.name} (${sessionId.substring(0, 8)}...)`);
3152
-
3153
- // Notify UI that follower is now fully ready
3154
- this.broadcast({
3155
- type: 'follower_session_ready',
3156
- deviceName: followerDevice.name,
3157
- sessionId: sessionId.substring(0, 8) + '...'
3158
- });
3159
-
3160
- } catch (sessionError) {
3161
- this.logger.warn(`⚠️ Background session creation failed for follower ${followerDevice.name}: ${sessionError.message}`);
3162
- // Retry once after 5 seconds
3163
- setTimeout(() => {
3164
- this.createFollowerSessionInBackground(followerDevice, port);
3165
- }, 5000);
3166
- }
3167
- }, 1000); // Start after 1 second delay to not block initial setup
3168
- }
3169
-
3170
- removeFollower(deviceName, preserveSession = true) {
3171
- const follower = Array.from(this.followerDevices).find(f => f.name === deviceName);
3172
- if (follower) {
3173
- this.followerDevices.delete(follower);
3174
-
3175
- // CRITICAL: Clear any follower-specific state that might conflict with commander role
3176
- this.logger.info(`🧹 Removing follower: ${deviceName} (clearing follower state)`);
3177
-
3178
- // Only preserve session info if explicitly requested (for role changes, not disconnects)
3179
- if (preserveSession && (follower.sessionId || follower.wdaSessionId)) {
3180
- this.logger.info(`📝 Preserving session info from follower ${deviceName} for potential commander use`);
3181
-
3182
- // Update the device in connectedDevices with session info
3183
- const globalDevice = this.connectedDevices.find(d => d.name === deviceName);
3184
- if (globalDevice) {
3185
- if (follower.sessionId) globalDevice.sessionId = follower.sessionId;
3186
- if (follower.wdaSessionId) globalDevice.wdaSessionId = follower.wdaSessionId;
3187
- if (follower.port) globalDevice.port = follower.port;
3188
-
3189
- this.logger.info(`📝 Session preserved: ${globalDevice.sessionId || globalDevice.wdaSessionId}`);
3190
- }
3191
- } else if (!preserveSession) {
3192
- this.logger.info(`🧹 Device ${deviceName} disconnecting - not preserving session`);
3193
- }
3194
-
3195
- this.broadcastCommanderStatus();
3196
- return { success: true };
3197
- }
3198
- return { success: false, error: 'Follower not found' };
3199
- }
3200
-
3201
- stopTouchMonitoring() {
3202
- this.logger.info('🛑 Stopping touch monitoring and screen streaming...');
3203
-
3204
- // Stop screen streaming
3205
- if (this.screenStreamInterval) {
3206
- clearInterval(this.screenStreamInterval);
3207
- this.screenStreamInterval = null;
3208
- }
3209
-
3210
- // Stop touch monitoring
3211
- if (this.touchMonitorInterval) {
3212
- clearInterval(this.touchMonitorInterval);
3213
- this.touchMonitorInterval = null;
3214
- }
3215
-
3216
- this.logger.info('✅ Touch monitoring stopped');
3217
- }
3218
-
3219
- stopCommanderMode() {
3220
- if (!this.isActive) return { success: false };
3221
-
3222
- this.logger.info('🛑 Stopping commander mode and cleaning up...');
3223
-
3224
- this.stopTouchMonitoring();
3225
-
3226
- // Clear commander device
3227
- const commanderName = this.commanderDevice?.name;
3228
- this.commanderDevice = null;
3229
-
3230
- // Clear all followers
3231
- this.followerDevices.clear();
3232
-
3233
- // Reset state
3234
- this.isActive = false;
3235
- // Note: We don't reset wdaSessionId anymore - it's managed globally
3236
- this.cachedElements = [];
3237
- this.lastCacheTime = 0;
3238
- this.sessionId = null;
3239
-
3240
- this.broadcastCommanderStatus();
3241
-
3242
- this.logger.info(`✅ Commander mode stopped. Removed device: ${commanderName}`);
3243
-
3244
- return {
3245
- success: true,
3246
- removedCommander: commanderName,
3247
- removedFollowers: 0
3248
- };
3249
- }
3250
-
3251
- async refreshLocators(fullScan = false) {
3252
- if (!this.isActive) {
3253
- throw new Error('Commander mode not active');
3254
- }
3255
-
3256
- this.logger.info(`🔄 ${fullScan ? 'Full' : 'Fast'} locator refresh requested`);
3257
- this.cachedElements = [];
3258
- this.lastCacheTime = 0;
3259
-
3260
- if (fullScan) {
3261
- await this.cacheAllElementsFullScan();
3262
- } else {
3263
- await this.cacheAllElements();
3264
- }
3265
-
3266
- return {
3267
- success: true,
3268
- elementsCount: this.cachedElements.length
3269
- };
3270
- }
3271
-
3272
- /**
3273
- * Background locator refresh - runs in a separate thread after locator is found
3274
- * This ensures UI elements stay current without blocking the current click operation
3275
- */
3276
- async refreshLocatorsInBackground() {
3277
- if (!this.isActive || !this.commanderDevice) {
3278
- this.logger.debug('⚠️ Commander not active, skipping background refresh');
3279
- return;
3280
- }
3281
-
3282
- try {
3283
- this.logger.info('🔄 Background: Starting automatic locator refresh...');
3284
-
3285
- // Add a small delay to avoid interfering with the current operation
3286
- await new Promise(resolve => setTimeout(resolve, 250));
3287
-
3288
- // Check if we're not in the middle of another caching operation
3289
- if (this.isCaching) {
3290
- this.logger.debug('⚠️ Background: Caching already in progress, skipping');
3291
- return;
3292
- }
3293
-
3294
- // Perform a fast refresh (not full scan) to keep it lightweight
3295
- const previousCount = this.cachedElements.length;
3296
- await this.fastCacheAllElements();
3297
- const newCount = this.cachedElements.length;
3298
-
3299
- this.logger.info(`✅ Background: Locators refreshed (${previousCount} → ${newCount} elements)`);
3300
-
3301
- // Broadcast refresh completion to UI if there are significant changes
3302
- const elementCountDiff = Math.abs(newCount - previousCount);
3303
- if (elementCountDiff > 5) {
3304
- this.broadcast({
3305
- type: 'locators_auto_refreshed',
3306
- message: `Locators auto-refreshed: ${newCount} elements available`,
3307
- elementsCount: newCount,
3308
- changeCount: elementCountDiff
3309
- });
3310
- this.logger.info(`📡 Background: Notified UI of significant changes (+/- ${elementCountDiff} elements)`);
3311
- }
3312
-
3313
- } catch (error) {
3314
- this.logger.warn(`⚠️ Background: Auto-refresh failed: ${error.message}`);
3315
- // Don't throw error since this is background operation
3316
- }
3317
- }
3318
-
3319
- /**
3320
- * Schedule a background refresh - safer approach than setImmediate
3321
- */
3322
- scheduleBackgroundRefresh() {
3323
- if (this._refreshTimeout) {
3324
- clearTimeout(this._refreshTimeout);
3325
- }
3326
-
3327
- this.logger.info('🔄 Near locator found - scheduling auto-refresh...');
3328
-
3329
- this._refreshTimeout = setTimeout(() => {
3330
- this.refreshLocatorsInBackground()
3331
- .catch(error => {
3332
- this.logger.warn(`Background locator refresh failed: ${error.message}`);
3333
- });
3334
- }, 500); // Wait 500ms to avoid interfering with current click
3335
- }
3336
-
3337
- async getAllLocatorsForInspector() {
3338
- // Use session lock to prevent concurrent access with click operations
3339
- return this.withSessionLock(this.commanderDevice?.name || 'commander', async () => {
3340
- return this._getAllLocatorsForInspector();
3341
- });
3342
- }
3343
-
3344
- async _getAllLocatorsForInspector() {
3345
- // Reset session recovery flag for this call
3346
- this._sessionRecoveryAttempted = false;
3347
-
3348
- // Independent locator fetch for inspector - doesn't use cache
3349
- if (!this.isActive || !this.commanderDevice) {
3350
- throw new Error('Commander mode not active');
3351
- }
3352
-
3353
- let wdaSessionId = this.getWDASessionId();
3354
-
3355
- // If session ID is not available, try to get it from the status endpoint
3356
- if (!wdaSessionId && this.commanderDevice.platform === 'ios') {
3357
- try {
3358
- const axios = require('axios');
3359
- const port = this.commanderDevice.port || 8100;
3360
- const statusRes = await axios.get(`http://localhost:${port}/status`, { timeout: 3000 });
3361
-
3362
- if (statusRes.data?.sessionId) {
3363
- wdaSessionId = statusRes.data.sessionId;
3364
- this.logger.debug(`🔍 Inspector: Retrieved WDA session from /status: ${wdaSessionId}`);
3365
-
3366
- // Register this session for future use
3367
- global.sessionManager?.getOrRegisterSession(
3368
- this.commanderDevice.udid,
3369
- wdaSessionId,
3370
- port,
3371
- this.commanderDevice.name,
3372
- null,
3373
- 'ios'
3374
- );
3375
- }
3376
- } catch (err) {
3377
- this.logger.debug(`Could not get WDA session from /status: ${err.message}`);
3378
- }
3379
- }
3380
-
3381
- // If still no session ID, we'll try to proceed anyway - the endpoint may not require it
3382
- if (!wdaSessionId && this.commanderDevice.platform === 'ios') {
3383
- this.logger.warn('⚠️ No WDA session ID available, will attempt without it');
3384
- }
3385
-
3386
- const startTime = Date.now();
3387
- this.logger.info('🔍 Inspector: Starting independent locator fetch...');
3388
-
3389
- try {
3390
- const axios = require('axios');
3391
- const device = this.commanderDevice;
3392
- const port = device.port || 8100;
3393
- const sessionId = wdaSessionId;
3394
-
3395
- // Try page source method first (fastest)
3396
- try {
3397
- // Skip page source method if no session ID - it won't work
3398
- if (!sessionId) {
3399
- this.logger.debug('Skipping page source method (no session ID), will try element finding');
3400
- throw new Error('No session ID available');
3401
- }
3402
-
3403
- const sourceRes = await axios.get(
3404
- `http://localhost:${port}/session/${sessionId}/source`,
3405
- { timeout: 8000 }
3406
- );
3407
-
3408
- if (sourceRes.data?.value) {
3409
- const pageSource = sourceRes.data.value;
3410
- const allElements = this.parsePageSourceFast(pageSource);
3411
-
3412
- if (allElements.length > 0) {
3413
- const elapsed = Date.now() - startTime;
3414
- this.logger.info(`✅ Inspector: Loaded ${allElements.length} elements in ${elapsed}ms (page source)`);
3415
-
3416
- return {
3417
- success: true,
3418
- elements: allElements
3419
- .filter(e => {
3420
- // Filter out overly large background elements that cover entire screen
3421
- const isEntireScreen = (e.width >= 350 && e.height >= 700); // Most of screen
3422
- const isContainer = e.type?.includes('View') || e.type?.includes('ScrollView') || e.type?.includes('CollectionView') || e.type?.includes('TableView');
3423
- const hasNoText = !e.label && !e.name && !e.value;
3424
-
3425
- // Exclude large containers without useful labels
3426
- if (isEntireScreen && isContainer && hasNoText) {
3427
- return false;
3428
- }
3429
-
3430
- // Exclude elements smaller than 10x10 pixels (too small to click)
3431
- if (e.width < 10 || e.height < 10) {
3432
- return false;
3433
- }
3434
-
3435
- return true;
3436
- })
3437
- .map(e => ({
3438
- label: e.label || e.name,
3439
- type: e.type.replace('XCUIElementType', ''),
3440
- x: e.x,
3441
- y: e.y,
3442
- width: e.width,
3443
- height: e.height,
3444
- centerX: e.centerX,
3445
- centerY: e.centerY
3446
- })),
3447
- method: 'page_source'
3448
- };
3449
- }
3450
- }
3451
- } catch (err) {
3452
- // If no session ID or 404 error, session is invalid - try to recover once
3453
- if ((err.message === 'No session ID available' || (err.response && err.response.status === 404)) && !this._sessionRecoveryAttempted) {
3454
- this.logger.warn(`⚠️ Inspector: Page source failed (${err.message || 'status ' + err.response.status}), attempting recovery...`);
3455
- this._sessionRecoveryAttempted = true;
3456
-
3457
- try {
3458
- // Use smart recovery instead of full recreation
3459
- const newSessionId = await this.recoverOrCreateSession(this.commanderDevice, wdaSessionId);
3460
-
3461
- if (newSessionId && newSessionId !== sessionId) {
3462
- this.logger.info(`✅ Inspector: Session recovered, retrying with ${newSessionId.substring(0, 8)}...`);
3463
-
3464
- // Retry page source with new session
3465
- const retryRes = await axios.get(
3466
- `http://localhost:${port}/session/${newSessionId}/source`,
3467
- { timeout: 8000 }
3468
- );
3469
-
3470
- if (retryRes.data?.value) {
3471
- const pageSource = retryRes.data.value;
3472
- const allElements = this.parsePageSourceFast(pageSource);
3473
-
3474
- if (allElements.length > 0) {
3475
- const elapsed = Date.now() - startTime;
3476
- this.logger.info(`✅ Inspector: Recovered ${allElements.length} elements in ${elapsed}ms (page source after recovery)`);
3477
-
3478
- return {
3479
- success: true,
3480
- elements: allElements.map(e => ({
3481
- label: e.label || e.name,
3482
- type: e.type.replace('XCUIElementType', ''),
3483
- x: e.x,
3484
- y: e.y,
3485
- width: e.width,
3486
- height: e.height,
3487
- centerX: e.centerX,
3488
- centerY: e.centerY
3489
- })),
3490
- method: 'page_source_recovered'
3491
- };
3492
- }
3493
- }
3494
- }
3495
- } catch (recoveryErr) {
3496
- this.logger.error(`❌ Inspector: Page source session recovery failed: ${recoveryErr.message}`);
3497
- }
3498
- }
3499
-
3500
- this.logger.warn('Inspector: Page source failed, using element query');
3501
- }
3502
-
3503
- // Fallback: Full element query (comprehensive)
3504
- const queries = [
3505
- { using: 'class name', value: 'XCUIElementTypeButton' },
3506
- { using: 'class name', value: 'XCUIElementTypeStaticText' },
3507
- { using: 'class name', value: 'XCUIElementTypeTextField' },
3508
- { using: 'class name', value: 'XCUIElementTypeCell' },
3509
- { using: 'class name', value: 'XCUIElementTypeImage' },
3510
- { using: 'class name', value: 'XCUIElementTypeSwitch' },
3511
- { using: 'class name', value: 'XCUIElementTypeLink' }
3512
- ];
3513
-
3514
- const allElements = [];
3515
- const MAX_ELEMENTS = 50;
3516
-
3517
- for (const query of queries) {
3518
- try {
3519
- const elementsRes = await axios.post(
3520
- `http://localhost:${port}/session/${sessionId}/elements`,
3521
- query,
3522
- { timeout: 5000 }
3523
- );
3524
-
3525
- if (elementsRes.data?.value && Array.isArray(elementsRes.data.value)) {
3526
- const elements = elementsRes.data.value.slice(0, MAX_ELEMENTS);
3527
-
3528
- const chunkSize = 15;
3529
- for (let i = 0; i < elements.length; i += chunkSize) {
3530
- const chunk = elements.slice(i, i + chunkSize);
3531
- const chunkPromises = chunk.map(async (elem) => {
3532
- const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
3533
- if (!elementId) return null;
3534
-
3535
- try {
3536
- const [rectRes, labelRes] = await Promise.all([
3537
- axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/rect`, { timeout: 1000 }),
3538
- axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/attribute/label`, { timeout: 1000 }).catch(() => ({ data: { value: null } }))
3539
- ]);
3540
-
3541
- if (rectRes.data?.value) {
3542
- const rect = rectRes.data.value;
3543
- const label = labelRes.data?.value;
3544
-
3545
- if (rect.width > 3 && rect.height > 3) {
3546
- return {
3547
- type: query.value,
3548
- label: label || 'Unlabeled',
3549
- name: label || 'Unlabeled',
3550
- x: Math.round(rect.x),
3551
- y: Math.round(rect.y),
3552
- width: Math.round(rect.width),
3553
- height: Math.round(rect.height),
3554
- centerX: Math.round(rect.x + rect.width / 2),
3555
- centerY: Math.round(rect.y + rect.height / 2)
3556
- };
3557
- }
3558
- }
3559
- } catch (err) { return null; }
3560
- return null;
3561
- });
3562
-
3563
- const chunkResults = await Promise.all(chunkPromises);
3564
- const validElements = chunkResults.filter(e => e !== null);
3565
- allElements.push(...validElements);
3566
- }
3567
- }
3568
- } catch (err) {
3569
- // If 404 error, session is invalid - try to recover once
3570
- if (err.response && err.response.status === 404 && !this._sessionRecoveryAttempted) {
3571
- this.logger.warn(`⚠️ Inspector: Session expired (404), attempting recovery...`);
3572
- this._sessionRecoveryAttempted = true;
3573
-
3574
- try {
3575
- // Use smart recovery instead of full recreation
3576
- const newSessionId = await this.recoverOrCreateSession(this.commanderDevice, sessionId);
3577
-
3578
- if (newSessionId && newSessionId !== sessionId) {
3579
- this.logger.info(`✅ Inspector: Session recovered, retrying with ${newSessionId.substring(0, 8)}...`);
3580
-
3581
- // Retry the query with new session
3582
- const retryRes = await axios.post(
3583
- `http://localhost:${port}/session/${newSessionId}/elements`,
3584
- query,
3585
- { timeout: 5000 }
3586
- );
3587
-
3588
- if (retryRes.data?.value && Array.isArray(retryRes.data.value)) {
3589
- // Process the retry results - simplified version
3590
- const elements = retryRes.data.value.slice(0, MAX_ELEMENTS);
3591
- for (const elem of elements.slice(0, 10)) { // Limit for quick recovery
3592
- const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
3593
- if (elementId) {
3594
- try {
3595
- const rectRes = await axios.get(`http://localhost:${port}/session/${newSessionId}/element/${elementId}/rect`, { timeout: 1000 });
3596
- if (rectRes.data?.value && rectRes.data.value.width > 3) {
3597
- const rect = rectRes.data.value;
3598
- allElements.push({
3599
- type: query.value,
3600
- label: 'Recovered Element',
3601
- name: 'Recovered Element',
3602
- x: Math.round(rect.x),
3603
- y: Math.round(rect.y),
3604
- width: Math.round(rect.width),
3605
- height: Math.round(rect.height),
3606
- centerX: Math.round(rect.x + rect.width / 2),
3607
- centerY: Math.round(rect.y + rect.height / 2)
3608
- });
3609
- }
3610
- } catch (e) { /* ignore individual element errors during recovery */ }
3611
- }
3612
- }
3613
- this.logger.info(`✅ Inspector: Recovered ${allElements.length} elements after session recovery`);
3614
- continue; // Skip the normal error logging since we recovered
3615
- }
3616
- }
3617
- } catch (recoveryErr) {
3618
- this.logger.error(`❌ Inspector: Session recovery failed: ${recoveryErr.message}`);
3619
- }
3620
- }
3621
-
3622
- this.logger.warn(`Inspector query ${query.value} failed: ${err.message}`);
3623
- }
3624
- }
3625
-
3626
- const elapsed = Date.now() - startTime;
3627
- this.logger.info(`✅ Inspector: Loaded ${allElements.length} elements in ${elapsed}ms (element query)`);
3628
-
3629
- return {
3630
- success: true,
3631
- elements: allElements.map(e => ({
3632
- label: e.label || e.name,
3633
- type: e.type.replace('XCUIElementType', ''),
3634
- x: e.x,
3635
- y: e.y,
3636
- width: e.width,
3637
- height: e.height,
3638
- centerX: e.centerX,
3639
- centerY: e.centerY
3640
- })),
3641
- method: 'element_query'
3642
- };
3643
-
3644
- } catch (error) {
3645
- this.logger.error(`Inspector locator fetch failed: ${error.message}`);
3646
- throw error;
3647
- }
3648
- }
3649
-
3650
- async cacheAllElementsFullScan() {
3651
- // Full comprehensive scan for manual refresh
3652
- const wdaSessionId = await this.getWDASessionId();
3653
- if (!wdaSessionId || !this.commanderDevice) return;
3654
- if (this.isCaching) return;
3655
-
3656
- this.isCaching = true;
3657
- const startTime = Date.now();
3658
-
3659
- try {
3660
- const axios = require('axios');
3661
- const device = this.commanderDevice;
3662
- const port = device.port || 8100;
3663
- const sessionId = wdaSessionId;
3664
-
3665
- this.logger.info('🔍 Starting FULL locator scan...');
3666
-
3667
- // Fast and accurate queries - back to original working set
3668
- const queries = [
3669
- { using: 'class name', value: 'XCUIElementTypeButton' },
3670
- { using: 'class name', value: 'XCUIElementTypeTextField' },
3671
- { using: 'class name', value: 'XCUIElementTypeStaticText' }
3672
- ];
3673
-
3674
- const allElements = [];
3675
- const MAX_ELEMENTS = 100; // Full scan - more elements
3676
-
3677
- for (const query of queries) {
3678
- try {
3679
- const elementsRes = await axios.post(
3680
- `http://localhost:${port}/session/${sessionId}/elements`,
3681
- query,
3682
- { timeout: 3000 }
3683
- );
3684
-
3685
- if (elementsRes.data?.value && Array.isArray(elementsRes.data.value)) {
3686
- const elements = elementsRes.data.value.slice(0, MAX_ELEMENTS);
3687
-
3688
- const chunkSize = 15;
3689
- for (let i = 0; i < elements.length; i += chunkSize) {
3690
- const chunk = elements.slice(i, i + chunkSize);
3691
- const chunkPromises = chunk.map(async (elem) => {
3692
- const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
3693
- if (!elementId) return null;
3694
-
3695
- try {
3696
- const [rectRes, labelRes] = await Promise.all([
3697
- axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/rect`, { timeout: 1500 }),
3698
- axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/attribute/label`, { timeout: 1500 }).catch(() => ({ data: { value: null } }))
3699
- ]);
3700
-
3701
- if (rectRes.data?.value) {
3702
- const rect = rectRes.data.value;
3703
- const label = labelRes.data?.value;
3704
-
3705
- // Simple fast filtering - back to original working logic
3706
- if (rect.width > 3 && rect.height > 3) {
3707
- return {
3708
- type: query.value,
3709
- label: label || 'Unlabeled',
3710
- name: label || 'Unlabeled',
3711
- accessibilityId: label,
3712
- x: Math.round(rect.x),
3713
- y: Math.round(rect.y),
3714
- width: Math.round(rect.width),
3715
- height: Math.round(rect.height),
3716
- centerX: Math.round(rect.x + rect.width / 2),
3717
- centerY: Math.round(rect.y + rect.height / 2)
3718
- };
3719
- }
3720
- }
3721
- } catch (err) { return null; }
3722
- return null;
3723
- });
3724
-
3725
- const chunkResults = await Promise.all(chunkPromises);
3726
- const validElements = chunkResults.filter(e => e !== null);
3727
-
3728
- if (validElements.length > 0) {
3729
- allElements.push(...validElements);
3730
-
3731
- // Send progress updates
3732
- this.broadcast({
3733
- type: 'cached_elements',
3734
- elements: validElements.map(e => ({
3735
- label: e.label || e.name,
3736
- type: e.type.replace('XCUIElementType', ''),
3737
- x: e.x,
3738
- y: e.y,
3739
- width: e.width,
3740
- height: e.height
3741
- })),
3742
- total: allElements.length,
3743
- timestamp: Date.now()
3744
- });
3745
- }
3746
- }
3747
- }
3748
- } catch (err) {
3749
- this.logger.warn(`Full scan query ${query.value} failed: ${err.message}`);
3750
- }
3751
- }
3752
-
3753
- this.cachedElements = allElements;
3754
- this.lastCacheTime = Date.now();
3755
-
3756
- const elapsed = Date.now() - startTime;
3757
- this.logger.info(`✅ Full scan completed: ${allElements.length} elements in ${elapsed}ms`);
3758
-
3759
- } finally {
3760
- this.isCaching = false;
3761
- }
3762
- }
3763
-
3764
- broadcastCommanderStatus() {
3765
- // CRITICAL FIX: Proper null safety checks
3766
- const commanderName = this.isActive && this.commanderDevice?.name ? this.commanderDevice.name : null;
3767
-
3768
- // Debug logging to verify device name casing
3769
- if (commanderName) {
3770
- this.logger.info(`📡 Broadcasting commander status for device: "${commanderName}"`);
3771
- }
3772
-
3773
- const status = {
3774
- type: 'commander_status',
3775
- isActive: this.isActive,
3776
- commander: commanderName,
3777
- followers: Array.from(this.followerDevices || []).map(f => ({
3778
- deviceId: f?.name || 'unknown',
3779
- name: f?.name || 'unknown',
3780
- platform: f?.platform || 'unknown',
3781
- port: f?.port || 0
3782
- })),
3783
- followerCount: this.followerDevices ? this.followerDevices.size : 0,
3784
- sessionId: this.sessionId
3785
- };
3786
- this.broadcast(status);
3787
- }
3788
-
3789
- broadcast(message) {
3790
- if (this.wss?.clients) {
3791
- this.wss.clients.forEach(client => {
3792
- if (client.readyState === 1) client.send(JSON.stringify(message));
3793
- });
3794
- }
3795
- }
3796
-
3797
- async executeAppAction(appAction, appId) {
3798
- const results = [];
3799
-
3800
- // Execute on commander device first
3801
- if (this.commanderDevice) {
3802
- try {
3803
- if (this.commanderDevice.platform === 'ios') {
3804
- // iOS commander app control via WDA
3805
- const session = await this.getWDASessionId();
3806
- if (session) {
3807
- const wdaUrl = `http://localhost:${this.commanderDevice.port || 8100}/session/${session}`;
3808
- const axios = require('axios');
3809
-
3810
- switch (appAction) {
3811
- case 'openApp':
3812
- case 'launchApp':
3813
- await axios.post(`${wdaUrl}/wda/apps/launch`, { bundleId: appId });
3814
- break;
3815
- case 'closeApp':
3816
- case 'terminateApp':
3817
- await axios.post(`${wdaUrl}/wda/apps/terminate`, { bundleId: appId });
3818
- break;
3819
- case 'activateApp':
3820
- await axios.post(`${wdaUrl}/wda/apps/activate`, { bundleId: appId });
3821
- break;
3822
- }
3823
- results.push({ device: this.commanderDevice.name, success: true });
3824
- this.logger.info(`✅ App action ${appAction} executed on commander: ${this.commanderDevice.name}`);
3825
- }
3826
- } else if (this.commanderDevice.platform === 'android') {
3827
- // Android commander app control via UIAutomator2
3828
- const session = sessionManager.getByName(this.commanderDevice.name);
3829
- if (session?.sessionId && session?.port) {
3830
- const axios = require('axios');
3831
- const baseUrl = `http://localhost:${session.port}/session/${session.sessionId}`;
3832
-
3833
- switch (appAction) {
3834
- case 'openApp':
3835
- case 'launchApp':
3836
- await axios.post(`${baseUrl}/appium/device/activate_app`, {
3837
- json: { appId: appId }
3838
- });
3839
- break;
3840
- case 'closeApp':
3841
- case 'terminateApp':
3842
- await axios.post(`${baseUrl}/appium/device/terminate_app`, {
3843
- json: { appId: appId }
3844
- });
3845
- break;
3846
- }
3847
- results.push({ device: this.commanderDevice.name, success: true });
3848
- this.logger.info(`✅ App action ${appAction} executed on commander: ${this.commanderDevice.name}`);
3849
- }
3850
- }
3851
- } catch (error) {
3852
- this.logger.error(`❌ App action ${appAction} failed on commander: ${error.message}`);
3853
- results.push({ device: this.commanderDevice.name, success: false, error: error.message });
3854
- }
3855
- }
3856
-
3857
- // Broadcast to followers
3858
- if (this.followerDevices.size > 0) {
3859
- const broadcastResult = await this.broadcastCommand({
3860
- action: 'app-control',
3861
- appAction: appAction,
3862
- appId: appId
3863
- });
3864
- results.push(...broadcastResult.results || []);
3865
- }
3866
-
3867
- return {
3868
- success: true,
3869
- results: results,
3870
- executedOn: results.length
3871
- };
3872
- }
3873
-
3874
- async broadcastCommand(commandData) {
3875
- if (!this.isActive || this.followerDevices.size === 0) {
3876
- this.logger.info('No followers to broadcast to');
3877
- return { success: true, broadcastCount: 0 };
3878
- }
3879
-
3880
- this.logger.info(`📢 Broadcasting command to ${this.followerDevices.size} followers`);
3881
-
3882
- const results = [];
3883
-
3884
- for (const follower of this.followerDevices) {
3885
- try {
3886
- if (commandData.action === 'app-control') {
3887
- // Handle app control commands for followers
3888
- const { appAction, appId } = commandData;
3889
-
3890
- if (follower.platform === 'ios') {
3891
- // iOS follower app control via WDA
3892
- const session = this.checkFollowerWDASession(follower);
3893
- if (session?.sessionId) {
3894
- const wdaUrl = `http://localhost:${follower.port || 8100}/session/${session.sessionId}`;
3895
- const axios = require('axios');
3896
-
3897
- switch (appAction) {
3898
- case 'openApp':
3899
- case 'launchApp':
3900
- await axios.post(`${wdaUrl}/wda/apps/launch`, { bundleId: appId });
3901
- break;
3902
- case 'closeApp':
3903
- case 'terminateApp':
3904
- await axios.post(`${wdaUrl}/wda/apps/terminate`, { bundleId: appId });
3905
- break;
3906
- case 'activateApp':
3907
- await axios.post(`${wdaUrl}/wda/apps/activate`, { bundleId: appId });
3908
- break;
3909
- }
3910
- }
3911
- } else if (follower.platform === 'android') {
3912
- // Android follower app control via UIAutomator2
3913
- const session = await this.getAndroidSession(follower);
3914
- if (session?.sessionId && session?.port) {
3915
- const axios = require('axios');
3916
- const baseUrl = `http://localhost:${session.port}/session/${session.sessionId}`;
3917
-
3918
- switch (appAction) {
3919
- case 'openApp':
3920
- case 'launchApp':
3921
- await axios.post(`${baseUrl}/appium/device/activate_app`, {
3922
- json: { appId: appId }
3923
- });
3924
- break;
3925
- case 'closeApp':
3926
- case 'terminateApp':
3927
- await axios.post(`${baseUrl}/appium/device/terminate_app`, {
3928
- json: { appId: appId }
3929
- });
3930
- break;
3931
- }
3932
- }
3933
- }
3934
-
3935
- results.push({ device: follower.name, success: true });
3936
- this.logger.info(`✅ App control broadcast to ${follower.name}: ${appAction} ${appId}`);
3937
- } else {
3938
- // Handle general commands (home, swipe, etc.) by using shell scripts
3939
- const command = commandData.action;
3940
- this.logger.info(`📢 Broadcasting command '${command}' to follower ${follower.name} via shell script`);
3941
-
3942
- // Use the existing executeCommand function that calls shell scripts
3943
- // This is the same system that /api/execute uses
3944
- const { spawn } = require('child_process');
3945
- const executeCommand = require('util').promisify(require('child_process').exec);
3946
-
3947
- try {
3948
- // Get the correct script path based on platform and connection type
3949
- let scriptPath;
3950
- if (follower.platform === 'ios') {
3951
- scriptPath = follower.connectionType === 'wireless'
3952
- ? './connect_ios_wireless_multi_final.sh'
3953
- : './connect_ios_usb_multi_final.sh';
3954
- } else if (follower.platform === 'android') {
3955
- scriptPath = follower.connectionType === 'wireless'
3956
- ? './connect_android_wireless_multi_final.sh'
3957
- : './connect_android_usb_multi_final.sh';
3958
- }
3959
-
3960
- if (scriptPath) {
3961
- // Execute command on follower using the same shell script system
3962
- const result = await executeCommand(`${scriptPath} "${follower.name}" "${command}"`);
3963
- this.logger.info(`✅ Command '${command}' broadcast to follower ${follower.name}`);
3964
- results.push({ device: follower.name, success: true });
3965
- } else {
3966
- throw new Error(`No script found for platform: ${follower.platform}`);
3967
- }
3968
- } catch (error) {
3969
- this.logger.error(`❌ Failed to broadcast command '${command}' to ${follower.name}: ${error.message}`);
3970
- results.push({ device: follower.name, success: false, error: error.message });
3971
- }
3972
- }
3973
- } catch (error) {
3974
- this.logger.error(`❌ Failed to broadcast to ${follower.name}: ${error.message}`);
3975
- results.push({ device: follower.name, success: false, error: error.message });
3976
- }
3977
- }
3978
-
3979
- const successCount = results.filter(r => r.success).length;
3980
- this.logger.info(`📢 Broadcast complete: ${successCount}/${results.length} followers updated`);
3981
-
3982
- return {
3983
- success: true,
3984
- broadcastCount: successCount,
3985
- results
3986
- };
3987
- }
3988
-
3989
- /**
3990
- * GENIUS: Get specific device window size for coordinate scaling with caching
3991
- */
3992
- async getDeviceWindowSizeForScaling(deviceName, port) {
3993
- // Check cache first
3994
- const cacheKey = `${deviceName}:${port}`;
3995
- const cached = this.deviceSizeCache.get(cacheKey);
3996
- if (cached && (Date.now() - cached.timestamp) < 30000) { // Cache for 30 seconds
3997
- this.logger.debug(`📦 Using cached size for ${deviceName}: ${cached.size.width}×${cached.size.height}`);
3998
- return cached.size;
3999
- }
4000
-
4001
- try {
4002
- const session = global.sessionManager?.getByName(deviceName);
4003
- if (!session?.sessionId) {
4004
- this.logger.warn(`⚠️ No session found for device: ${deviceName}`);
4005
- return null;
4006
- }
4007
-
4008
- const response = await axios.get(`http://localhost:${port}/session/${session.sessionId}/window/size`, {
4009
- timeout: 3000
4010
- });
4011
-
4012
- if (response.data?.value) {
4013
- const size = response.data.value;
4014
- this.logger.debug(`📱 Device ${deviceName} window size: ${size.width}×${size.height}`);
4015
-
4016
- // Cache the result
4017
- this.deviceSizeCache.set(cacheKey, {
4018
- size: size,
4019
- timestamp: Date.now()
4020
- });
4021
-
4022
- return size;
4023
- }
4024
- } catch (error) {
4025
- this.logger.warn(`⚠️ Failed to get window size for ${deviceName}: ${error.message}`);
4026
- }
4027
- return null;
4028
- }
4029
-
4030
- /**
4031
- * GENIUS: Scale coordinates from commander to follower device
4032
- */
4033
- scaleCoordinatesForDevice(commanderCoords, commanderSize, followerSize) {
4034
- if (!commanderSize || !followerSize) {
4035
- this.logger.warn(`⚠️ Missing device sizes for coordinate scaling`);
4036
- return commanderCoords; // Fallback to original coordinates
4037
- }
4038
-
4039
- // Prevent division by zero
4040
- if (commanderSize.width === 0 || commanderSize.height === 0) {
4041
- this.logger.warn(`⚠️ Invalid commander size: ${commanderSize.width}×${commanderSize.height}`);
4042
- return commanderCoords;
4043
- }
4044
-
4045
- const scaleX = followerSize.width / commanderSize.width;
4046
- const scaleY = followerSize.height / commanderSize.height;
4047
-
4048
- const scaledX = Math.round(commanderCoords.x * scaleX);
4049
- const scaledY = Math.round(commanderCoords.y * scaleY);
4050
-
4051
- // Ensure coordinates are within bounds
4052
- const boundedX = Math.max(0, Math.min(scaledX, followerSize.width - 1));
4053
- const boundedY = Math.max(0, Math.min(scaledY, followerSize.height - 1));
4054
-
4055
- this.logger.info(`📐 Coordinate scaling: ${commanderCoords.x},${commanderCoords.y} → ${boundedX},${boundedY}`);
4056
- this.logger.info(` Commander: ${commanderSize.width}×${commanderSize.height}`);
4057
- this.logger.info(` Follower: ${followerSize.width}×${followerSize.height}`);
4058
- this.logger.info(` Scale: ${scaleX.toFixed(3)}×${scaleY.toFixed(3)}`);
4059
-
4060
- return { x: boundedX, y: boundedY };
4061
- }
4062
-
4063
- // General action broadcasting (tap, type, etc.) with intelligent coordinate transformation
4064
- async broadcastAction({ action, locator, input, coordinates }) {
4065
- if (!this.isActive || this.followerDevices.size === 0) {
4066
- this.logger.info('No followers to broadcast action to');
4067
- return { success: true, broadcastCount: 0 };
4068
- }
4069
-
4070
- this.logger.info(`📢 Broadcasting action '${action}' to ${this.followerDevices.size} followers`);
4071
-
4072
- // GENIUS: Get commander device screen size for coordinate scaling
4073
- let commanderSize = null;
4074
- if (coordinates && this.commanderDevice) {
4075
- const commanderPort = this.commanderDevice.port || 8100;
4076
- commanderSize = await this.getDeviceWindowSizeForScaling(this.commanderDevice.name, commanderPort);
4077
- if (commanderSize) {
4078
- this.logger.info(`📱 Commander screen size: ${commanderSize.width}×${commanderSize.height}`);
4079
- }
4080
- }
4081
-
4082
- const results = [];
4083
- const axios = require('axios');
4084
-
4085
- for (const follower of this.followerDevices) {
4086
- try {
4087
- const port = follower.port || (follower.platform === 'ios' ? 8100 : 4723);
4088
-
4089
- // GENIUS: Get follower screen size and scale coordinates if needed
4090
- let scaledCoordinates = coordinates;
4091
- if (coordinates && commanderSize) {
4092
- const followerSize = await this.getDeviceWindowSizeForScaling(follower.name, port);
4093
- if (followerSize) {
4094
- scaledCoordinates = this.scaleCoordinatesForDevice(coordinates, commanderSize, followerSize);
4095
- } else {
4096
- this.logger.warn(`⚠️ Could not get screen size for follower ${follower.name}, using original coordinates`);
4097
- }
4098
- }
4099
-
4100
- if (follower.platform === 'ios') {
4101
- // iOS WebDriverAgent actions
4102
- // GENIUS: Get session from global manager for each follower
4103
- const followerSession = global.sessionManager?.getByName(follower.name);
4104
- const sessionId = followerSession?.sessionId || this.getWDASessionId(); // Fallback to commander's session
4105
- const baseUrl = `http://localhost:${port}`;
4106
-
4107
- switch (action) {
4108
- case 'tap':
4109
- if (scaledCoordinates) {
4110
- try {
4111
- this.logger.info(`🎯 Tapping follower ${follower.name} at scaled coordinates: (${scaledCoordinates.x}, ${scaledCoordinates.y})`);
4112
- await axios.post(`${baseUrl}/session/${sessionId}/wda/tap/0`, {
4113
- x: scaledCoordinates.x,
4114
- y: scaledCoordinates.y
4115
- });
4116
- } catch (tapErr) {
4117
- // If 404 error, follower session is invalid - try with fresh session
4118
- if (tapErr.response?.status === 404) {
4119
- this.logger.warn(`⚠️ Follower ${follower.name} session expired (404), attempting recovery...`);
4120
- try {
4121
- // Try to get a fresh session for the follower
4122
- const newFollowerSession = global.sessionManager?.getByName(follower.name);
4123
- const newSessionId = newFollowerSession?.sessionId;
4124
-
4125
- if (newSessionId && newSessionId !== sessionId) {
4126
- // Retry with new session and scaled coordinates
4127
- this.logger.info(`🔄 Retry tapping ${follower.name} with recovered session at: (${scaledCoordinates.x}, ${scaledCoordinates.y})`);
4128
- await axios.post(`${baseUrl}/session/${newSessionId}/wda/tap/0`, {
4129
- x: scaledCoordinates.x,
4130
- y: scaledCoordinates.y
4131
- });
4132
- this.logger.info(`✅ Follower ${follower.name} tap succeeded after session recovery`);
4133
- } else {
4134
- throw tapErr; // Re-throw if no new session available
4135
- }
4136
- } catch (recoveryErr) {
4137
- throw new Error(`Follower tap failed: ${recoveryErr.message}`);
4138
- }
4139
- } else {
4140
- throw tapErr;
4141
- }
4142
- }
4143
- } else if (locator) {
4144
- // Find element and tap
4145
- const elementRes = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
4146
- using: 'accessibility id',
4147
- value: locator
4148
- });
4149
- const elementId = elementRes.data.value.ELEMENT;
4150
- await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/click`);
4151
- }
4152
- break;
4153
-
4154
- case 'type':
4155
- if (input) {
4156
- await axios.post(`${baseUrl}/session/${sessionId}/wda/keys`, {
4157
- value: input.split('')
4158
- });
4159
- }
4160
- break;
4161
-
4162
- case 'swipe':
4163
- if (scaledCoordinates && scaledCoordinates.fromX !== undefined) {
4164
- // Scale swipe coordinates too
4165
- let scaledSwipeCoords = scaledCoordinates;
4166
- if (coordinates && commanderSize) {
4167
- const followerSize = await this.getDeviceWindowSizeForScaling(follower.name, port);
4168
- if (followerSize) {
4169
- scaledSwipeCoords = {
4170
- fromX: Math.round(coordinates.fromX * (followerSize.width / commanderSize.width)),
4171
- fromY: Math.round(coordinates.fromY * (followerSize.height / commanderSize.height)),
4172
- toX: Math.round(coordinates.toX * (followerSize.width / commanderSize.width)),
4173
- toY: Math.round(coordinates.toY * (followerSize.height / commanderSize.height)),
4174
- duration: coordinates.duration || 0.5
4175
- };
4176
- }
4177
- }
4178
-
4179
- await axios.post(`${baseUrl}/session/${sessionId}/wda/dragfromtoforduration`, scaledSwipeCoords);
4180
- }
4181
- break;
4182
-
4183
- default:
4184
- this.logger.warn(`Unsupported iOS action: ${action}`);
4185
- }
4186
- } else if (follower.platform === 'android') {
4187
- // Android UIAutomator2 actions
4188
- const baseUrl = `http://localhost:${port}/wd/hub`;
4189
-
4190
- switch (action) {
4191
- case 'tap':
4192
- if (scaledCoordinates) {
4193
- this.logger.info(`🎯 Tapping Android follower ${follower.name} at scaled coordinates: (${scaledCoordinates.x}, ${scaledCoordinates.y})`);
4194
- await axios.post(`${baseUrl}/session/${sessionId}/touch/click`, {
4195
- element: { x: scaledCoordinates.x, y: scaledCoordinates.y }
4196
- });
4197
- } else if (locator) {
4198
- // Find element and tap
4199
- const elementRes = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
4200
- using: 'accessibility id',
4201
- value: locator
4202
- });
4203
- const elementId = elementRes.data.value.ELEMENT;
4204
- await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/click`);
4205
- }
4206
- break;
4207
-
4208
- case 'type':
4209
- if (input) {
4210
- await axios.post(`${baseUrl}/session/${sessionId}/keys`, {
4211
- value: [input]
4212
- });
4213
- }
4214
- break;
4215
-
4216
- default:
4217
- this.logger.warn(`Unsupported Android action: ${action}`);
4218
- }
4219
- }
4220
-
4221
- results.push({ device: follower.name, success: true });
4222
- this.logger.info(`✅ Action '${action}' broadcast to ${follower.name}`);
4223
-
4224
- } catch (error) {
4225
- this.logger.error(`❌ Failed to broadcast action to ${follower.name}: ${error.message}`);
4226
- results.push({ device: follower.name, success: false, error: error.message });
4227
- }
4228
- }
4229
-
4230
- const successCount = results.filter(r => r.success).length;
4231
- this.logger.info(`📢 Action broadcast complete: ${successCount}/${results.length} followers updated`);
4232
-
4233
- return {
4234
- success: true,
4235
- broadcastCount: successCount,
4236
- results,
4237
- action
4238
- };
4239
- }
4240
-
4241
- /**
4242
- * 🎯 NEW GENIUS METHOD: Execute Intelligent Action on Follower
4243
- * Uses the IntelligentLocatorService to perform actions with maximum accuracy
4244
- */
4245
- async executeIntelligentActionOnFollower(follower, intelligentLocator, originalX, originalY) {
4246
- // Use session lock to prevent concurrent access
4247
- return this.withSessionLock(follower.name, async () => {
4248
- return this._executeIntelligentActionOnFollower(follower, intelligentLocator, originalX, originalY);
4249
- });
4250
- }
4251
-
4252
- async _executeIntelligentActionOnFollower(follower, intelligentLocator, originalX, originalY) {
4253
- this.logger.info(`🎯 Executing intelligent action on ${follower.name} using ${intelligentLocator.method}`);
4254
-
4255
- try {
4256
- const startTime = Date.now();
4257
-
4258
- // OPTIMIZED: Only invalidate follower cache if commander cache was invalidated
4259
- // This prevents excessive cache clearing
4260
- if (!this.intelligentLocator.hasCachedLocators(follower.name)) {
4261
- this.logger.info(`🧹 Pre-invalidating locator cache for follower ${follower.name} (screen changed after commander action)`);
4262
- this.intelligentLocator.invalidateCache(follower.name);
4263
- }
4264
-
4265
- // Step 1: Execute using IntelligentLocatorService
4266
- const result = await this.intelligentLocator.executeIntelligentAction(
4267
- follower,
4268
- intelligentLocator,
4269
- 'tap'
4270
- );
4271
-
4272
- const executionTime = Date.now() - startTime;
4273
-
4274
- if (result.success) {
4275
- this.logger.info(`✅ Intelligent action succeeded on ${follower.name} in ${executionTime}ms (method: ${result.method})`);
4276
-
4277
- // Update success metrics
4278
- if (result.method === 'locator') {
4279
- this.locatorSuccessCount++;
4280
- }
4281
-
4282
- // Update per-device success rate
4283
- const deviceRate = this.locatorSuccessRate.get(follower.name) || { success: 0, total: 0 };
4284
- deviceRate.success++;
4285
- deviceRate.total++;
4286
- this.locatorSuccessRate.set(follower.name, deviceRate);
4287
-
4288
- return { success: true, method: result.method, executionTime };
4289
-
4290
- } else {
4291
- this.logger.error(`❌ Intelligent action failed on ${follower.name}: ${result.error}`);
4292
-
4293
- // Update metrics
4294
- const deviceRate = this.locatorSuccessRate.get(follower.name) || { success: 0, total: 0 };
4295
- deviceRate.total++;
4296
- this.locatorSuccessRate.set(follower.name, deviceRate);
4297
-
4298
- return { success: false, error: result.error, method: result.method };
4299
- }
4300
-
4301
- } catch (error) {
4302
- this.logger.error(`❌ Intelligent follower execution failed: ${error.message}`);
4303
-
4304
- // Fallback to traditional coordinate scaling as last resort
4305
- this.logger.warn(`🔄 Attempting traditional coordinate fallback for ${follower.name}`);
4306
-
4307
- try {
4308
- // Use the existing coordinate scaling logic as fallback
4309
- const commanderSize = this.lastWindowSize;
4310
- const followerSize = await this.getDeviceWindowSizeForScaling(follower.name, follower.port || 8100);
4311
-
4312
- if (followerSize && commanderSize) {
4313
- const scaledCoords = this.scaleCoordinatesForDevice(
4314
- { x: originalX, y: originalY },
4315
- commanderSize,
4316
- followerSize
4317
- );
4318
-
4319
- // Execute using coordinates
4320
- const coordResult = await this.intelligentLocator.executeCoordinateAction(
4321
- follower,
4322
- scaledCoords,
4323
- 'tap'
4324
- );
4325
-
4326
- if (coordResult) {
4327
- this.logger.info(`✅ Coordinate fallback succeeded on ${follower.name}`);
4328
- return { success: true, method: 'coordinates_fallback' };
4329
- }
4330
- }
4331
-
4332
- throw new Error('Both intelligent and coordinate methods failed');
4333
-
4334
- } catch (fallbackError) {
4335
- this.logger.error(`❌ All methods failed for ${follower.name}: ${fallbackError.message}`);
4336
- return { success: false, error: `All methods failed: ${error.message}` };
4337
- }
4338
- }
4339
- }
4340
-
4341
- /**
4342
- * 🎯 Get Intelligent Locator Performance Statistics
4343
- */
4344
- getIntelligentLocatorStats() {
4345
- const totalRequests = this.locatorSuccessCount + this.coordinateFallbackCount;
4346
- const locatorSuccessRate = totalRequests > 0 ?
4347
- ((this.locatorSuccessCount / totalRequests) * 100).toFixed(1) : 0;
4348
-
4349
- const deviceStats = {};
4350
- for (const [deviceName, stats] of this.locatorSuccessRate.entries()) {
4351
- const rate = stats.total > 0 ? ((stats.success / stats.total) * 100).toFixed(1) : 0;
4352
- deviceStats[deviceName] = {
4353
- successRate: `${rate}%`,
4354
- successful: stats.success,
4355
- total: stats.total
4356
- };
4357
- }
4358
-
4359
- return {
4360
- overallLocatorSuccessRate: `${locatorSuccessRate}%`,
4361
- locatorSuccesses: this.locatorSuccessCount,
4362
- coordinateFallbacks: this.coordinateFallbackCount,
4363
- totalRequests: totalRequests,
4364
- deviceStats: deviceStats,
4365
- intelligentLocatorMetrics: this.intelligentLocator?.getPerformanceMetrics()
4366
- };
4367
- }
4368
-
4369
- /**
4370
- * 🎯 Force Refresh Locator Cache for All Devices
4371
- */
4372
- async refreshIntelligentLocatorCache() {
4373
- this.logger.info('🔄 Refreshing intelligent locator cache for all devices...');
4374
-
4375
- const devices = [this.commanderDevice, ...Array.from(this.followerDevices)].filter(d => d);
4376
- const refreshResults = [];
4377
-
4378
- for (const device of devices) {
4379
- try {
4380
- const locators = await this.intelligentLocator.getCachedLocators(device.name, true);
4381
- refreshResults.push({
4382
- device: device.name,
4383
- success: true,
4384
- locatorCount: locators.length
4385
- });
4386
- this.logger.info(`✅ Refreshed ${locators.length} locators for ${device.name}`);
4387
- } catch (error) {
4388
- refreshResults.push({
4389
- device: device.name,
4390
- success: false,
4391
- error: error.message
4392
- });
4393
- this.logger.error(`❌ Failed to refresh locators for ${device.name}: ${error.message}`);
4394
- }
4395
- }
4396
-
4397
- return {
4398
- success: true,
4399
- refreshedDevices: refreshResults.filter(r => r.success).length,
4400
- totalDevices: refreshResults.length,
4401
- results: refreshResults
4402
- };
4403
- }
4404
-
4405
- getStatus() {
4406
- const baseStatus = {
4407
- isActive: this.isActive,
4408
- commander: this.commanderDevice?.name,
4409
- followers: Array.from(this.followerDevices).map(f => ({
4410
- deviceId: f.name,
4411
- name: f.name,
4412
- platform: f.platform,
4413
- port: f.port
4414
- })),
4415
- followerCount: this.followerDevices.size
4416
- };
4417
-
4418
- // Add intelligent locator stats if available
4419
- if (this.useIntelligentLocators) {
4420
- baseStatus.intelligentLocatorStats = this.getIntelligentLocatorStats();
4421
- }
4422
-
4423
- return baseStatus;
4424
- }
4425
- }
4426
-
4427
- module.exports = CommanderService;