devicely 2.2.13 → 2.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/devicely.js +1 -1
- package/lib/advanced-logger.js +1 -1
- package/lib/androidDeviceDetection.js +1 -1
- package/lib/appMappings.js +1 -1
- package/lib/commanderService.js +1 -1
- package/lib/deviceDetection.js +1 -1
- package/lib/devices.js +1 -1
- package/lib/doctor.js +1 -1
- package/lib/encryption.js +1 -1
- package/lib/executor.js +1 -1
- package/lib/hybridAI.js +1 -1
- package/lib/intelligentLocatorService.js +1 -1
- package/lib/lightweightAI.js +1 -1
- package/lib/localBuiltInAI.js +1 -1
- package/lib/localBuiltInAI_backup.js +1 -1
- package/lib/localBuiltInAI_simple.js +1 -1
- package/lib/locatorStrategy.js +1 -1
- package/lib/logger-demo.js +1 -1
- package/lib/logger.js +1 -1
- package/lib/quick-start-logger.js +1 -1
- package/lib/scriptLoader.js +1 -1
- package/lib/server.js +1 -1
- package/lib/tensorflowAI.js +1 -1
- package/lib/tinyAI.js +1 -1
- package/lib/universalSessionManager.js +1 -1
- package/package.json +1 -1
- package/lib/.logging-backup/aiProviders.js.backup +0 -654
- package/lib/.logging-backup/appMappings.js.backup +0 -337
- package/lib/.logging-backup/commanderService.js.backup +0 -4427
- package/lib/.logging-backup/devices.js.backup +0 -54
- package/lib/.logging-backup/doctor.js.backup +0 -94
- package/lib/.logging-backup/encryption.js.backup +0 -61
- package/lib/.logging-backup/executor.js.backup +0 -104
- package/lib/.logging-backup/hybridAI.js.backup +0 -154
- package/lib/.logging-backup/intelligentLocatorService.js.backup +0 -1541
- package/lib/.logging-backup/locatorStrategy.js.backup +0 -342
- package/lib/.logging-backup/scriptLoader.js.backup +0 -13
- package/lib/.logging-backup/server.js.backup +0 -6298
- package/lib/.logging-backup/tensorflowAI.js.backup +0 -714
- package/lib/.logging-backup/universalSessionManager.js.backup +0 -370
- package/lib/.logging-enhanced-backup/server.js.enhanced-backup +0 -6298
|
@@ -1,4427 +0,0 @@
|
|
|
1
|
-
const Logger = require('./logger');
|
|
2
|
-
const { spawn } = require('child_process');
|
|
3
|
-
const LocatorStrategy = require('./locatorStrategy');
|
|
4
|
-
const IntelligentLocatorService = require('./intelligentLocatorService'); // NEW: Import intelligent locator service
|
|
5
|
-
const universalSessionManager = require('./universalSessionManager'); // REVOLUTIONARY: Universal session management
|
|
6
|
-
const axios = require('axios'); // Add axios at module level
|
|
7
|
-
|
|
8
|
-
class CommanderService {
|
|
9
|
-
constructor(wss, connectedDevices = []) {
|
|
10
|
-
this.wss = wss;
|
|
11
|
-
this.connectedDevices = connectedDevices; // Reference to global connected devices
|
|
12
|
-
this.commanderDevice = null;
|
|
13
|
-
this.followerDevices = new Set();
|
|
14
|
-
this.isActive = false;
|
|
15
|
-
this.captureProcess = null;
|
|
16
|
-
this.logger = new Logger('CommanderService');
|
|
17
|
-
this.eventQueue = [];
|
|
18
|
-
this.sessionId = null;
|
|
19
|
-
this.locatorStrategy = new LocatorStrategy();
|
|
20
|
-
|
|
21
|
-
// GENIUS: Click Queue System for handling rapid clicks
|
|
22
|
-
this.clickQueue = [];
|
|
23
|
-
this.processingClick = false;
|
|
24
|
-
this.maxQueueSize = 3; // Limit queue to prevent overflow
|
|
25
|
-
|
|
26
|
-
// REVOLUTIONARY: Universal Session Management
|
|
27
|
-
this.sessionManager = universalSessionManager;
|
|
28
|
-
|
|
29
|
-
// 🎯 NEW: Initialize Intelligent Locator Service
|
|
30
|
-
this.intelligentLocator = new IntelligentLocatorService(connectedDevices);
|
|
31
|
-
|
|
32
|
-
// GENIUS: Inject universal session management into intelligent locator service
|
|
33
|
-
this.intelligentLocator.getValidWDASession = this.getValidWDASession.bind(this);
|
|
34
|
-
|
|
35
|
-
this.logger.info('🎯 Intelligent Locator Service initialized with Universal Session Management');
|
|
36
|
-
|
|
37
|
-
// Make intelligentLocatorService globally accessible for disconnect cleanup
|
|
38
|
-
global.intelligentLocatorService = this.intelligentLocator;
|
|
39
|
-
|
|
40
|
-
// GENIUS: Use global session manager instead of local wdaSessionId
|
|
41
|
-
this.cachedElements = []; // Cache all screen elements
|
|
42
|
-
this.lastCacheTime = 0; // Track when cache was last updated
|
|
43
|
-
this.isCaching = false; // Prevent duplicate caching operations
|
|
44
|
-
this.lastWindowSize = { width: 375, height: 667 };
|
|
45
|
-
this.commanderDriver = null; // WebDriver-like interface
|
|
46
|
-
this.deviceSizeCache = new Map(); // Cache device screen sizes for coordinate scaling
|
|
47
|
-
|
|
48
|
-
// Session operation locking to prevent concurrent access
|
|
49
|
-
this.sessionLocks = new Map(); // Map<deviceName, Promise>
|
|
50
|
-
|
|
51
|
-
// 🎯 NEW: Intelligent Locator Settings
|
|
52
|
-
this.useIntelligentLocators = true; // Enable intelligent locator mode
|
|
53
|
-
this.locatorSuccessRate = new Map(); // Track success rate per device
|
|
54
|
-
this.coordinateFallbackCount = 0;
|
|
55
|
-
this.locatorSuccessCount = 0;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
// GENIUS: Get WDA session ID from universal session manager
|
|
59
|
-
async getWDASessionId() {
|
|
60
|
-
if (!this.commanderDevice) return null;
|
|
61
|
-
|
|
62
|
-
try {
|
|
63
|
-
// Use universal session manager to get live session
|
|
64
|
-
const sessionId = await this.sessionManager.getLiveSessionId(this.commanderDevice.name, this.commanderDevice.port);
|
|
65
|
-
return sessionId;
|
|
66
|
-
} catch (error) {
|
|
67
|
-
this.logger.warn(`⚠️ Could not get WDA session: ${error.message}`);
|
|
68
|
-
return null;
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
// Register a WDA session using universal session manager
|
|
73
|
-
registerSessionGlobally(device, sessionId, port) {
|
|
74
|
-
if (!device || !sessionId) return;
|
|
75
|
-
|
|
76
|
-
const udid = device.udid || device.serial;
|
|
77
|
-
const platform = device.platform || 'ios';
|
|
78
|
-
const deviceName = device.name;
|
|
79
|
-
|
|
80
|
-
// Register device port in universal session manager
|
|
81
|
-
this.sessionManager.setDevicePort(deviceName, port);
|
|
82
|
-
this.logger.debug(`📌 Registered port mapping in Universal Session Manager: ${deviceName} -> ${port}`);
|
|
83
|
-
|
|
84
|
-
// Also store in connectedDevices for backward compatibility
|
|
85
|
-
const globalDevice = this.connectedDevices.find(d =>
|
|
86
|
-
d.name === deviceName || d.udid === udid
|
|
87
|
-
);
|
|
88
|
-
|
|
89
|
-
if (globalDevice) {
|
|
90
|
-
if (platform === 'ios') {
|
|
91
|
-
globalDevice.wdaSessionId = sessionId;
|
|
92
|
-
} else {
|
|
93
|
-
globalDevice.sessionId = sessionId;
|
|
94
|
-
}
|
|
95
|
-
globalDevice.port = port;
|
|
96
|
-
this.logger.debug(`📌 Registered session in connectedDevices: ${deviceName}`);
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
// REVOLUTIONARY: Universal Session Management - Always Live WDA Lookup
|
|
101
|
-
async getValidWDASession(deviceName, port) {
|
|
102
|
-
try {
|
|
103
|
-
// Register the port mapping if provided
|
|
104
|
-
if (port) {
|
|
105
|
-
this.sessionManager.setDevicePort(deviceName, port);
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
// Get live session with retry logic
|
|
109
|
-
const sessionId = await this.sessionManager.getSessionWithRetry(deviceName, port);
|
|
110
|
-
|
|
111
|
-
this.logger.debug(`✅ Live session acquired: ${sessionId.substring(0, 8)}... for ${deviceName}`);
|
|
112
|
-
return sessionId;
|
|
113
|
-
|
|
114
|
-
} catch (error) {
|
|
115
|
-
this.logger.error(`❌ Failed to get live session for ${deviceName}: ${error.message}`);
|
|
116
|
-
throw error;
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
async withSessionLock(deviceName, operation) {
|
|
123
|
-
const lockKey = `${deviceName}_wda_session`;
|
|
124
|
-
|
|
125
|
-
// If there's already an operation in progress, wait for it
|
|
126
|
-
if (this.sessionLocks.has(lockKey)) {
|
|
127
|
-
this.logger.debug(`⏳ Waiting for existing session operation on ${deviceName}...`);
|
|
128
|
-
try {
|
|
129
|
-
await this.sessionLocks.get(lockKey);
|
|
130
|
-
} catch (err) {
|
|
131
|
-
// Previous operation failed, continue with ours
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
// Create new lock
|
|
136
|
-
const operationPromise = this.executeWithLock(operation, deviceName);
|
|
137
|
-
this.sessionLocks.set(lockKey, operationPromise);
|
|
138
|
-
|
|
139
|
-
try {
|
|
140
|
-
const result = await operationPromise;
|
|
141
|
-
this.sessionLocks.delete(lockKey);
|
|
142
|
-
return result;
|
|
143
|
-
} catch (error) {
|
|
144
|
-
this.sessionLocks.delete(lockKey);
|
|
145
|
-
throw error;
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
async executeWithLock(operation, deviceName) {
|
|
150
|
-
this.logger.debug(`🔒 Acquired session lock for ${deviceName}`);
|
|
151
|
-
try {
|
|
152
|
-
const result = await operation();
|
|
153
|
-
this.logger.debug(`🔓 Released session lock for ${deviceName}`);
|
|
154
|
-
return result;
|
|
155
|
-
} catch (error) {
|
|
156
|
-
this.logger.debug(`🔓 Released session lock for ${deviceName} (error: ${error.message})`);
|
|
157
|
-
throw error;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
// Determine if an element is likely to be interactive vs layout/informational
|
|
162
|
-
isElementInteractive(elementType, rect, label) {
|
|
163
|
-
// Always include actual interactive elements
|
|
164
|
-
if (elementType.includes('Button') || elementType.includes('TextField') ||
|
|
165
|
-
elementType.includes('Switch') || elementType.includes('Link')) {
|
|
166
|
-
return true;
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
// Filter out StaticText and Cells that are likely layout elements
|
|
170
|
-
if (elementType.includes('StaticText')) {
|
|
171
|
-
// Include if it's a reasonable clickable size and has meaningful label
|
|
172
|
-
if (rect.width > 40 && rect.height > 20 && label &&
|
|
173
|
-
!label.match(/^\d+$/) && // Not just numbers
|
|
174
|
-
!label.includes('Label') && // Not generic labels
|
|
175
|
-
label.length > 1) {
|
|
176
|
-
return true;
|
|
177
|
-
}
|
|
178
|
-
return false;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
if (elementType.includes('Cell')) {
|
|
182
|
-
// Only include cells that are reasonably sized (not massive containers)
|
|
183
|
-
if (rect.width < 600 && rect.height < 200 && rect.height > 30) {
|
|
184
|
-
return true;
|
|
185
|
-
}
|
|
186
|
-
return false;
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
if (elementType.includes('Image')) {
|
|
190
|
-
// Only include images that might be clickable (reasonable size, has label)
|
|
191
|
-
if (rect.width < 200 && rect.height < 200 && rect.width > 20 && rect.height > 20 && label) {
|
|
192
|
-
return true;
|
|
193
|
-
}
|
|
194
|
-
return false;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
// Exclude Other elements (containers)
|
|
198
|
-
if (elementType.includes('Other')) {
|
|
199
|
-
return false;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
return false; // Default to exclude
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
// More permissive filtering for element caching (used for speed)
|
|
206
|
-
isElementRelevant(elementType, rect, label, enabled = true) {
|
|
207
|
-
// Always include truly interactive elements
|
|
208
|
-
if (elementType.includes('Button') || elementType.includes('TextField') ||
|
|
209
|
-
elementType.includes('Switch') || elementType.includes('Link')) {
|
|
210
|
-
return enabled;
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
// Include StaticText if it could be clickable
|
|
214
|
-
if (elementType.includes('StaticText')) {
|
|
215
|
-
// More permissive - include if decent size
|
|
216
|
-
if (rect.width > 20 && rect.height > 15 && enabled) {
|
|
217
|
-
return true;
|
|
218
|
-
}
|
|
219
|
-
return false;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
// Include reasonable cells
|
|
223
|
-
if (elementType.includes('Cell')) {
|
|
224
|
-
if (rect.width < 500 && rect.height < 150 && rect.height > 20 && enabled) {
|
|
225
|
-
return true;
|
|
226
|
-
}
|
|
227
|
-
return false;
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
// Include images that might be clickable
|
|
231
|
-
if (elementType.includes('Image')) {
|
|
232
|
-
if (rect.width < 150 && rect.height < 150 && rect.width > 15 && rect.height > 15 && enabled) {
|
|
233
|
-
return true;
|
|
234
|
-
}
|
|
235
|
-
return false;
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
// Include Other elements if they're reasonable sized (might be custom controls)
|
|
239
|
-
if (elementType.includes('Other')) {
|
|
240
|
-
if (rect.width > 20 && rect.height > 20 && rect.width < 300 && rect.height < 100 && enabled) {
|
|
241
|
-
return true;
|
|
242
|
-
}
|
|
243
|
-
return false;
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
return false;
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// Create a simple driver interface for app control
|
|
250
|
-
async createDriverInterface() {
|
|
251
|
-
if (!this.commanderDevice) {
|
|
252
|
-
return null;
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
const device = this.commanderDevice;
|
|
256
|
-
|
|
257
|
-
if (device.platform === 'ios') {
|
|
258
|
-
// iOS WebDriverAgent interface
|
|
259
|
-
const wdaSessionId = await this.getWDASessionId();
|
|
260
|
-
if (!wdaSessionId) return null;
|
|
261
|
-
|
|
262
|
-
const port = device.port || 8100;
|
|
263
|
-
const sessionId = wdaSessionId;
|
|
264
|
-
const baseUrl = `http://localhost:${port}`;
|
|
265
|
-
|
|
266
|
-
return {
|
|
267
|
-
execute: async (command, params = {}) => {
|
|
268
|
-
const axios = require('axios');
|
|
269
|
-
|
|
270
|
-
try {
|
|
271
|
-
switch (command) {
|
|
272
|
-
case 'mobile: launchApp':
|
|
273
|
-
return await axios.post(`${baseUrl}/session/${sessionId}/wda/apps/launch`, {
|
|
274
|
-
bundleId: params.bundleId
|
|
275
|
-
});
|
|
276
|
-
|
|
277
|
-
case 'mobile: terminateApp':
|
|
278
|
-
return await axios.post(`${baseUrl}/session/${sessionId}/wda/apps/terminate`, {
|
|
279
|
-
bundleId: params.bundleId
|
|
280
|
-
});
|
|
281
|
-
|
|
282
|
-
case 'mobile: activateApp':
|
|
283
|
-
return await axios.post(`${baseUrl}/session/${sessionId}/wda/apps/activate`, {
|
|
284
|
-
bundleId: params.bundleId
|
|
285
|
-
});
|
|
286
|
-
|
|
287
|
-
default:
|
|
288
|
-
throw new Error(`Unsupported iOS command: ${command}`);
|
|
289
|
-
}
|
|
290
|
-
} catch (error) {
|
|
291
|
-
this.logger.error(`iOS driver command failed: ${error.message}`);
|
|
292
|
-
throw error;
|
|
293
|
-
}
|
|
294
|
-
},
|
|
295
|
-
|
|
296
|
-
getWindowSize: async () => {
|
|
297
|
-
return this.lastWindowSize;
|
|
298
|
-
}
|
|
299
|
-
};
|
|
300
|
-
}
|
|
301
|
-
else if (device.platform === 'android') {
|
|
302
|
-
// Android - use ADB commands via shell
|
|
303
|
-
return {
|
|
304
|
-
execute: async (command, params = {}) => {
|
|
305
|
-
const { spawn } = require('child_process');
|
|
306
|
-
|
|
307
|
-
try {
|
|
308
|
-
switch (command) {
|
|
309
|
-
case 'mobile: launchApp':
|
|
310
|
-
return new Promise((resolve, reject) => {
|
|
311
|
-
const adb = spawn('adb', ['-s', device.serial || device.udid, 'shell', 'am', 'start', '-n', `${params.bundleId}/.MainActivity`]);
|
|
312
|
-
|
|
313
|
-
adb.on('close', (code) => {
|
|
314
|
-
if (code === 0) {
|
|
315
|
-
resolve({ success: true });
|
|
316
|
-
} else {
|
|
317
|
-
// Try alternative launch method
|
|
318
|
-
const adb2 = spawn('adb', ['-s', device.serial || device.udid, 'shell', 'monkey', '-p', params.bundleId, '-c', 'android.intent.category.LAUNCHER', '1']);
|
|
319
|
-
adb2.on('close', (code2) => {
|
|
320
|
-
resolve({ success: code2 === 0 });
|
|
321
|
-
});
|
|
322
|
-
adb2.on('error', () => resolve({ success: false }));
|
|
323
|
-
}
|
|
324
|
-
});
|
|
325
|
-
adb.on('error', (error) => reject(error));
|
|
326
|
-
});
|
|
327
|
-
|
|
328
|
-
case 'mobile: terminateApp':
|
|
329
|
-
return new Promise((resolve, reject) => {
|
|
330
|
-
const adb = spawn('adb', ['-s', device.serial || device.udid, 'shell', 'am', 'force-stop', params.bundleId]);
|
|
331
|
-
|
|
332
|
-
adb.on('close', (code) => {
|
|
333
|
-
resolve({ success: code === 0 });
|
|
334
|
-
});
|
|
335
|
-
adb.on('error', (error) => reject(error));
|
|
336
|
-
});
|
|
337
|
-
|
|
338
|
-
case 'mobile: activateApp':
|
|
339
|
-
// Same as launch for Android
|
|
340
|
-
return this.execute('mobile: launchApp', params);
|
|
341
|
-
|
|
342
|
-
default:
|
|
343
|
-
throw new Error(`Unsupported Android command: ${command}`);
|
|
344
|
-
}
|
|
345
|
-
} catch (error) {
|
|
346
|
-
this.logger.error(`Android driver command failed: ${error.message}`);
|
|
347
|
-
throw error;
|
|
348
|
-
}
|
|
349
|
-
},
|
|
350
|
-
|
|
351
|
-
getWindowSize: async () => {
|
|
352
|
-
return this.lastWindowSize;
|
|
353
|
-
}
|
|
354
|
-
};
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
return null;
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
/**
|
|
361
|
-
* Get UIAutomator2 session ID from universal session manager
|
|
362
|
-
* This ensures session ID is always in sync across all operations
|
|
363
|
-
*/
|
|
364
|
-
async getAndroidSession(device) {
|
|
365
|
-
if (!device || device.platform !== 'android') return null;
|
|
366
|
-
|
|
367
|
-
try {
|
|
368
|
-
// Use universal session manager to get live session for Android
|
|
369
|
-
const sessionId = await this.sessionManager.getLiveSessionId(device.name, device.port);
|
|
370
|
-
return sessionId;
|
|
371
|
-
} catch (error) {
|
|
372
|
-
this.logger.warn(`⚠️ Could not get Android session for ${device.name}: ${error.message}`);
|
|
373
|
-
|
|
374
|
-
// Fallback: Try different possible keys for Android session lookup
|
|
375
|
-
const possibleKeys = [
|
|
376
|
-
device.udid,
|
|
377
|
-
device.serial,
|
|
378
|
-
device.name
|
|
379
|
-
].filter(Boolean);
|
|
380
|
-
|
|
381
|
-
for (const key of possibleKeys) {
|
|
382
|
-
const session = global.androidSessions?.get(key);
|
|
383
|
-
if (session && session.sessionId && session.port) {
|
|
384
|
-
// Cache session info on device object for faster access
|
|
385
|
-
device.sessionId = session.sessionId;
|
|
386
|
-
device.port = session.port;
|
|
387
|
-
return session;
|
|
388
|
-
}
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
return null;
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
async setCommander(deviceName, deviceInfo) {
|
|
396
|
-
if (this.isActive && this.commanderDevice && this.commanderDevice.name === deviceName) {
|
|
397
|
-
// Same device - commander mode is already active, just return success
|
|
398
|
-
this.logger.info(`Commander mode already active on ${deviceName}`);
|
|
399
|
-
return {
|
|
400
|
-
success: true,
|
|
401
|
-
commanderDevice: this.commanderDevice,
|
|
402
|
-
sessionId: this.sessionId,
|
|
403
|
-
message: 'Commander mode already active'
|
|
404
|
-
};
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
if (this.isActive && this.commanderDevice && this.commanderDevice.name !== deviceName) {
|
|
408
|
-
// Different device - preserve sessions during commander switch
|
|
409
|
-
this.logger.info(`Switching commander from ${this.commanderDevice.name} to ${deviceName}`);
|
|
410
|
-
|
|
411
|
-
// CRITICAL: Preserve WDA session of outgoing commander for potential follower role
|
|
412
|
-
const outgoingCommander = this.commanderDevice;
|
|
413
|
-
if (outgoingCommander) {
|
|
414
|
-
this.logger.info(`📝 Preserving WDA session for outgoing commander: ${outgoingCommander.name}`);
|
|
415
|
-
|
|
416
|
-
// Add outgoing commander as follower to preserve its WDA session
|
|
417
|
-
const followerDevice = {
|
|
418
|
-
name: outgoingCommander.name,
|
|
419
|
-
udid: outgoingCommander.udid,
|
|
420
|
-
platform: outgoingCommander.platform,
|
|
421
|
-
port: outgoingCommander.port,
|
|
422
|
-
sessionId: this.sessionId, // Preserve the current session
|
|
423
|
-
wdaSessionId: this.sessionId,
|
|
424
|
-
wasCommander: true // Mark this as a former commander
|
|
425
|
-
};
|
|
426
|
-
|
|
427
|
-
this.followerDevices.add(followerDevice);
|
|
428
|
-
this.logger.info(`📝 Former commander ${outgoingCommander.name} automatically added as follower to preserve WDA session`);
|
|
429
|
-
|
|
430
|
-
// Only stop touch monitoring, keep WDA alive and preserve followers
|
|
431
|
-
this.stopTouchMonitoring();
|
|
432
|
-
this.commanderDevice = null;
|
|
433
|
-
this.sessionId = null;
|
|
434
|
-
}
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
this.commanderDevice = {
|
|
438
|
-
...deviceInfo,
|
|
439
|
-
name: deviceName // CRITICAL: Ensure deviceName overrides deviceInfo.name
|
|
440
|
-
};
|
|
441
|
-
this.isActive = true;
|
|
442
|
-
this.sessionId = `commander_${Date.now()}`;
|
|
443
|
-
|
|
444
|
-
// CRITICAL: Track if this device was recently a follower
|
|
445
|
-
this.wasRecentlyFollower = deviceInfo.wasRecentlyFollower || false;
|
|
446
|
-
|
|
447
|
-
// Debug device name handling
|
|
448
|
-
this.logger.info(`📱 Commander device set: "${deviceName}" (platform: ${deviceInfo.platform})`);
|
|
449
|
-
this.logger.info(`📊 Device info name: "${deviceInfo.name}", used name: "${this.commanderDevice.name}"`);
|
|
450
|
-
|
|
451
|
-
if (this.wasRecentlyFollower) {
|
|
452
|
-
this.logger.info(`🔄 Device ${deviceName} was recently a follower - will reuse existing WDA session`);
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
// CRITICAL: Ensure we use the passed deviceName, not deviceInfo.name
|
|
456
|
-
if (deviceInfo.name && deviceInfo.name !== deviceName) {
|
|
457
|
-
this.logger.warn(`⚠️ Name mismatch detected and corrected: deviceInfo.name="${deviceInfo.name}" → using "${deviceName}"`);
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
// For Android devices, get UIAutomator2 session info using global helper
|
|
461
|
-
if (deviceInfo.platform === 'android') {
|
|
462
|
-
const session = await this.getAndroidSession(this.commanderDevice);
|
|
463
|
-
|
|
464
|
-
if (session) {
|
|
465
|
-
this.logger.info(`✅ Android UIAutomator2 session found: port ${session.port}, sessionId ${session.sessionId.substring(0, 8)}...`);
|
|
466
|
-
this.commanderDriver = await this.createDriverInterface();
|
|
467
|
-
} else {
|
|
468
|
-
this.logger.warn(`⚠️ No UIAutomator2 session found, will look up on first screenshot`);
|
|
469
|
-
// Still create the driver interface - it will check for session when needed
|
|
470
|
-
this.commanderDriver = await this.createDriverInterface();
|
|
471
|
-
}
|
|
472
|
-
}
|
|
473
|
-
// For iOS devices, ONLY check for existing WDA session - DON'T create new one
|
|
474
|
-
else if (deviceInfo.platform === 'ios') {
|
|
475
|
-
await this.checkExistingWDASession();
|
|
476
|
-
}
|
|
477
|
-
|
|
478
|
-
// Start real-time touch event monitoring
|
|
479
|
-
this.startTouchMonitoring();
|
|
480
|
-
|
|
481
|
-
this.broadcastCommanderStatus();
|
|
482
|
-
|
|
483
|
-
return {
|
|
484
|
-
success: true,
|
|
485
|
-
commanderDevice: this.commanderDevice,
|
|
486
|
-
sessionId: this.sessionId
|
|
487
|
-
};
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
async checkExistingWDASession() {
|
|
491
|
-
const axios = require('axios');
|
|
492
|
-
const device = this.commanderDevice;
|
|
493
|
-
const port = device.port || 8100;
|
|
494
|
-
|
|
495
|
-
try {
|
|
496
|
-
// Check for existing WDA session first - ALWAYS, even for promoted followers
|
|
497
|
-
this.logger.info('Checking for existing WDA session...');
|
|
498
|
-
|
|
499
|
-
const wasRecentFollower = this.wasRecentlyFollower || false;
|
|
500
|
-
if (wasRecentFollower) {
|
|
501
|
-
this.logger.info(`🔄 Device ${device.name} was recently a follower - attempting to reuse existing WDA session`);
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
// ALWAYS try to find and reuse existing sessions first
|
|
505
|
-
try {
|
|
506
|
-
const existingSessionId = await this.sessionManager.getActiveSession(device.name, device.port || 8100);
|
|
507
|
-
|
|
508
|
-
if (existingSessionId) {
|
|
509
|
-
this.logger.info(`✅ Found existing WDA session: ${existingSessionId.substring(0, 8)}... - testing viability`);
|
|
510
|
-
|
|
511
|
-
// Test if the session actually works
|
|
512
|
-
const sessionWorks = await this.validateSessionWorking(device.port || 8100);
|
|
513
|
-
if (sessionWorks) {
|
|
514
|
-
this.commanderDriver = this.createDriverInterface();
|
|
515
|
-
this.logger.info(`✅ Reusing healthy existing WDA session for ${wasRecentFollower ? 'promoted follower' : 'commander'}`);
|
|
516
|
-
await this.getDeviceWindowSize(device.port || 8100);
|
|
517
|
-
return;
|
|
518
|
-
} else {
|
|
519
|
-
this.logger.warn(`⚠️ Existing session found but not responding - will create fresh session`);
|
|
520
|
-
}
|
|
521
|
-
} else {
|
|
522
|
-
this.logger.info('No existing session found - will create new one');
|
|
523
|
-
}
|
|
524
|
-
} catch (sessionCheckError) {
|
|
525
|
-
this.logger.warn(`Session check failed: ${sessionCheckError.message} - will create new session`);
|
|
526
|
-
}
|
|
527
|
-
|
|
528
|
-
// GENIUS: First check global session manager
|
|
529
|
-
const existingSession = global.sessionManager?.getByName(device.name);
|
|
530
|
-
if (existingSession && existingSession.platform === 'ios') {
|
|
531
|
-
this.logger.info(`✅ Found global WDA session: ${existingSession.sessionId}`);
|
|
532
|
-
|
|
533
|
-
// CRITICAL: Test if this session actually works for screenshots before using it
|
|
534
|
-
const sessionWorks = await this.validateSessionWorking(port);
|
|
535
|
-
if (sessionWorks) {
|
|
536
|
-
this.commanderDriver = this.createDriverInterface();
|
|
537
|
-
this.logger.info(`✅ Using validated global WDA session: ${existingSession.sessionId}`);
|
|
538
|
-
await this.getDeviceWindowSize(port);
|
|
539
|
-
return;
|
|
540
|
-
} else {
|
|
541
|
-
this.logger.warn(`⚠️ Global session found but screenshot test failed - creating fresh session`);
|
|
542
|
-
await this.createWDASession();
|
|
543
|
-
return;
|
|
544
|
-
}
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
// Fallback: check connectedDevices for backward compatibility
|
|
548
|
-
const globalDevice = this.connectedDevices.find(d => d.name === device.name || d.udid === device.udid);
|
|
549
|
-
if (globalDevice && globalDevice.sessionId) {
|
|
550
|
-
// FIXED: Use safe session registration (don't override for Quick actions)
|
|
551
|
-
global.sessionManager?.getOrRegisterSession(device.udid, globalDevice.sessionId, port, device.name, null, 'ios');
|
|
552
|
-
this.commanderDriver = this.createDriverInterface();
|
|
553
|
-
this.logger.info(`✅ Registered and using WDA session: ${globalDevice.sessionId}`);
|
|
554
|
-
await this.getDeviceWindowSize(port);
|
|
555
|
-
return;
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
// If no global session, check /status endpoint for existing session
|
|
559
|
-
try {
|
|
560
|
-
const statusResponse = await axios.get(`http://localhost:${port}/status`, { timeout: 5000 });
|
|
561
|
-
const sessionId = statusResponse.data?.sessionId || statusResponse.data?.value?.sessionId;
|
|
562
|
-
|
|
563
|
-
if (sessionId) {
|
|
564
|
-
// Register globally to ensure it's available everywhere
|
|
565
|
-
this.registerSessionGlobally(device, sessionId, port);
|
|
566
|
-
this.commanderDriver = this.createDriverInterface();
|
|
567
|
-
this.logger.info(`✅ Found and registered WDA session: ${sessionId.substring(0, 8)}...`);
|
|
568
|
-
await this.getDeviceWindowSize(port);
|
|
569
|
-
return;
|
|
570
|
-
}
|
|
571
|
-
} catch (statusError) {
|
|
572
|
-
this.logger.debug(`WDA /status endpoint check failed: ${statusError.message}`);
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
// Try to get existing session from /sessions endpoint
|
|
576
|
-
try {
|
|
577
|
-
const sessionsResponse = await axios.get(`http://localhost:${port}/sessions`, { timeout: 3000 });
|
|
578
|
-
const sessions = sessionsResponse.data?.value;
|
|
579
|
-
|
|
580
|
-
if (sessions && Array.isArray(sessions) && sessions.length > 0) {
|
|
581
|
-
const sessionId = sessions[0].id;
|
|
582
|
-
// Register globally
|
|
583
|
-
this.registerSessionGlobally(device, sessionId, port);
|
|
584
|
-
this.commanderDriver = this.createDriverInterface();
|
|
585
|
-
this.logger.info(`✅ Found and registered session from /sessions: ${sessionId.substring(0, 8)}...`);
|
|
586
|
-
await this.getDeviceWindowSize(port);
|
|
587
|
-
return;
|
|
588
|
-
}
|
|
589
|
-
} catch (e) {
|
|
590
|
-
this.logger.debug('No sessions found at /sessions endpoint');
|
|
591
|
-
}
|
|
592
|
-
|
|
593
|
-
// If no existing session found, create a new one (this should work since we validated WDA availability)
|
|
594
|
-
this.logger.info('⚠️ No existing WDA session found - creating new session for commander mode');
|
|
595
|
-
|
|
596
|
-
// CRITICAL DEBUG: Log the complete device state for troubleshooting
|
|
597
|
-
this.logger.info(`🔍 Device promotion debug info:`, {
|
|
598
|
-
name: device.name,
|
|
599
|
-
udid: device.udid,
|
|
600
|
-
port: device.port,
|
|
601
|
-
platform: device.platform
|
|
602
|
-
});
|
|
603
|
-
|
|
604
|
-
// Check if this device was recently a follower
|
|
605
|
-
const wasFollower = this.followerDevices && Array.from(this.followerDevices).some(f => f.name === device.name);
|
|
606
|
-
if (wasFollower) {
|
|
607
|
-
this.logger.info(`🔄 Device ${device.name} was recently a follower, ensuring clean session setup`);
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
try {
|
|
611
|
-
await this.createWDASession();
|
|
612
|
-
this.commanderDriver = this.createDriverInterface();
|
|
613
|
-
this.logger.info('✅ New WDA session created for commander mode');
|
|
614
|
-
|
|
615
|
-
// CRITICAL: Force immediate session validation after creation
|
|
616
|
-
await this.validateSessionWorking(device.port || 8100);
|
|
617
|
-
|
|
618
|
-
} catch (createError) {
|
|
619
|
-
this.logger.error(`❌ Failed to create WDA session: ${createError.message}`);
|
|
620
|
-
this.logger.warn('💡 Commander mode will work with limited functionality');
|
|
621
|
-
|
|
622
|
-
// EMERGENCY FALLBACK: Try one more time after a delay
|
|
623
|
-
setTimeout(async () => {
|
|
624
|
-
try {
|
|
625
|
-
this.logger.info(`🔧 Emergency retry: Creating WDA session for ${device.name}`);
|
|
626
|
-
await this.createWDASession();
|
|
627
|
-
this.commanderDriver = this.createDriverInterface();
|
|
628
|
-
this.logger.info('✅ Emergency retry successful');
|
|
629
|
-
} catch (retryError) {
|
|
630
|
-
this.logger.error(`❌ Emergency retry failed: ${retryError.message}`);
|
|
631
|
-
}
|
|
632
|
-
}, 2000);
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
} catch (error) {
|
|
636
|
-
this.logger.error(`❌ Failed to check WDA status: ${error.message}`);
|
|
637
|
-
this.logger.warn('💡 Commander mode will continue with limited functionality');
|
|
638
|
-
// Continue anyway - since WDA was validated before we got here, this is likely a temporary issue
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
// CRITICAL: Validate that a WDA session is actually working for screenshots
|
|
643
|
-
async validateSessionWorking(port) {
|
|
644
|
-
const axios = require('axios');
|
|
645
|
-
|
|
646
|
-
try {
|
|
647
|
-
this.logger.info(`🧪 Fast validating WDA session on port ${port}...`);
|
|
648
|
-
|
|
649
|
-
// Test screenshot endpoint with faster timeout
|
|
650
|
-
const screenshotResponse = await axios.get(`http://localhost:${port}/screenshot`, {
|
|
651
|
-
timeout: 4000, // Reduced from 8000ms to 4000ms for faster validation
|
|
652
|
-
validateStatus: () => true // Don't throw on any status
|
|
653
|
-
});
|
|
654
|
-
|
|
655
|
-
if (screenshotResponse.status === 200) {
|
|
656
|
-
const imageData = screenshotResponse.data;
|
|
657
|
-
if (imageData && (typeof imageData === 'string' && imageData.length > 100)) {
|
|
658
|
-
this.logger.info(`✅ WDA session validation successful - screenshot working`);
|
|
659
|
-
return true;
|
|
660
|
-
} else {
|
|
661
|
-
this.logger.warn(`⚠️ WDA session responds but image data invalid (length: ${imageData?.length || 0})`);
|
|
662
|
-
return false;
|
|
663
|
-
}
|
|
664
|
-
} else {
|
|
665
|
-
this.logger.warn(`⚠️ WDA session validation failed - status ${screenshotResponse.status}`);
|
|
666
|
-
return false;
|
|
667
|
-
}
|
|
668
|
-
|
|
669
|
-
} catch (error) {
|
|
670
|
-
this.logger.warn(`⚠️ WDA session validation failed: ${error.message}`);
|
|
671
|
-
return false;
|
|
672
|
-
}
|
|
673
|
-
}
|
|
674
|
-
|
|
675
|
-
async startTouchMonitoring() {
|
|
676
|
-
if (this.touchMonitorInterval) {
|
|
677
|
-
clearInterval(this.touchMonitorInterval);
|
|
678
|
-
}
|
|
679
|
-
|
|
680
|
-
this.logger.info('🎯 Starting optimized screen mirroring and touch monitoring');
|
|
681
|
-
|
|
682
|
-
// Pre-warm sessions for faster startup
|
|
683
|
-
await this.preWarmSessions();
|
|
684
|
-
|
|
685
|
-
// Start screen streaming (2 FPS for responsive interaction)
|
|
686
|
-
this.startScreenStreaming();
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
/**
|
|
690
|
-
* Pre-warm sessions to reduce initial mirroring delay
|
|
691
|
-
*/
|
|
692
|
-
async preWarmSessions() {
|
|
693
|
-
try {
|
|
694
|
-
this.logger.info('⚡ Pre-warming sessions for instant mirroring...');
|
|
695
|
-
|
|
696
|
-
// Pre-warm commander device session
|
|
697
|
-
if (this.commanderDevice) {
|
|
698
|
-
if (this.commanderDevice.platform === 'android') {
|
|
699
|
-
await this.getAndroidSession(this.commanderDevice);
|
|
700
|
-
} else if (this.commanderDevice.platform === 'ios') {
|
|
701
|
-
const port = this.commanderDevice.port || 8100;
|
|
702
|
-
try {
|
|
703
|
-
await this.sessionManager.getLiveSessionId(this.commanderDevice.name, port);
|
|
704
|
-
this.logger.info(`✅ Pre-warmed iOS session for ${this.commanderDevice.name}:${port}`);
|
|
705
|
-
} catch (error) {
|
|
706
|
-
this.logger.debug(`Pre-warm failed for ${this.commanderDevice.name}, will discover during streaming`);
|
|
707
|
-
}
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
// Pre-warm follower device sessions and cache screenshot URLs
|
|
712
|
-
if (this.followerDevices && this.followerDevices.size > 0) {
|
|
713
|
-
for (const follower of this.followerDevices) {
|
|
714
|
-
try {
|
|
715
|
-
if (follower.platform === 'android') {
|
|
716
|
-
const session = await this.getAndroidSession(follower);
|
|
717
|
-
if (session) {
|
|
718
|
-
// Pre-cache screenshot URL for faster streaming
|
|
719
|
-
follower._cachedPort = session.port;
|
|
720
|
-
follower._cachedScreenshotUrl = `http://localhost:${session.port}/wd/hub/session/${session.sessionId}/screenshot`;
|
|
721
|
-
this.logger.info(`✅ Pre-cached Android follower: ${follower.name}:${session.port}`);
|
|
722
|
-
}
|
|
723
|
-
} else if (follower.platform === 'ios') {
|
|
724
|
-
const port = follower.port || 8100;
|
|
725
|
-
try {
|
|
726
|
-
await this.sessionManager.getLiveSessionId(follower.name, port);
|
|
727
|
-
// Pre-cache screenshot URL for faster streaming
|
|
728
|
-
follower._cachedPort = port;
|
|
729
|
-
follower._cachedScreenshotUrl = `http://localhost:${port}/screenshot`;
|
|
730
|
-
this.logger.info(`✅ Pre-cached iOS follower: ${follower.name}:${port}`);
|
|
731
|
-
} catch (error) {
|
|
732
|
-
// Try alternative port for iOS
|
|
733
|
-
const altPort = port === 8100 ? 8101 : 8100;
|
|
734
|
-
try {
|
|
735
|
-
await this.sessionManager.getLiveSessionId(follower.name, altPort);
|
|
736
|
-
follower.port = altPort;
|
|
737
|
-
follower._cachedPort = altPort;
|
|
738
|
-
follower._cachedScreenshotUrl = `http://localhost:${altPort}/screenshot`;
|
|
739
|
-
this.logger.info(`✅ Pre-cached iOS follower on alt port: ${follower.name}:${altPort}`);
|
|
740
|
-
} catch (altError) {
|
|
741
|
-
this.logger.debug(`Pre-warm failed for follower ${follower.name} on both ports`);
|
|
742
|
-
}
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
} catch (error) {
|
|
746
|
-
this.logger.debug(`Pre-warm failed for follower ${follower.name}, will discover during streaming`);
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
}
|
|
750
|
-
|
|
751
|
-
this.logger.info('⚡ Session pre-warming complete');
|
|
752
|
-
} catch (error) {
|
|
753
|
-
this.logger.warn(`Pre-warming failed: ${error.message}, continuing with normal startup`);
|
|
754
|
-
}
|
|
755
|
-
}
|
|
756
|
-
|
|
757
|
-
async createWDASession() {
|
|
758
|
-
const axios = require('axios');
|
|
759
|
-
const device = this.commanderDevice;
|
|
760
|
-
const port = device.port || 8100;
|
|
761
|
-
|
|
762
|
-
try {
|
|
763
|
-
this.logger.info('No existing session found, creating new one...');
|
|
764
|
-
|
|
765
|
-
// Create new session
|
|
766
|
-
const createResponse = await axios.post(`http://localhost:${port}/session`, {
|
|
767
|
-
capabilities: {
|
|
768
|
-
alwaysMatch: {},
|
|
769
|
-
firstMatch: [{}]
|
|
770
|
-
}
|
|
771
|
-
}, { timeout: 10000 });
|
|
772
|
-
|
|
773
|
-
const newSessionId = createResponse.data?.sessionId || createResponse.data?.value?.sessionId;
|
|
774
|
-
|
|
775
|
-
if (newSessionId) {
|
|
776
|
-
// Register globally to ensure it's available everywhere
|
|
777
|
-
this.registerSessionGlobally(device, newSessionId, port);
|
|
778
|
-
this.logger.info(`✅ WDA session created and registered: ${newSessionId.substring(0, 8)}...`);
|
|
779
|
-
await this.getDeviceWindowSize(port);
|
|
780
|
-
} else {
|
|
781
|
-
throw new Error('Failed to create WDA session - no session ID returned');
|
|
782
|
-
}
|
|
783
|
-
|
|
784
|
-
} catch (error) {
|
|
785
|
-
this.logger.error(`❌ WDA session creation failed: ${error.message}`);
|
|
786
|
-
throw error;
|
|
787
|
-
}
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
async startScreenStreaming() {
|
|
791
|
-
if (this.screenStreamInterval) {
|
|
792
|
-
clearInterval(this.screenStreamInterval);
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
const axios = require('axios');
|
|
796
|
-
this.logger.info('📺 Starting optimized screen streaming at 2 FPS');
|
|
797
|
-
|
|
798
|
-
// CRITICAL DEBUG: Check device state before starting streaming
|
|
799
|
-
if (!this.commanderDevice) {
|
|
800
|
-
this.logger.error('❌ Cannot start screen streaming: no commander device set');
|
|
801
|
-
return;
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
this.logger.info(`📱 Starting screen streaming for: "${this.commanderDevice.name}" on port ${this.commanderDevice.port}`);
|
|
805
|
-
|
|
806
|
-
// Pre-validate session and cache endpoint URL for faster streaming
|
|
807
|
-
const device = this.commanderDevice;
|
|
808
|
-
let cachedScreenshotUrl = null;
|
|
809
|
-
let cachedPort = null;
|
|
810
|
-
|
|
811
|
-
try {
|
|
812
|
-
if (device.platform === 'android') {
|
|
813
|
-
const session = await this.getAndroidSession(device);
|
|
814
|
-
if (session) {
|
|
815
|
-
cachedPort = session.port;
|
|
816
|
-
cachedScreenshotUrl = `http://localhost:${cachedPort}/wd/hub/session/${session.sessionId}/screenshot`;
|
|
817
|
-
this.logger.info(`✅ Pre-cached Android screenshot URL: ${cachedScreenshotUrl.substring(0, 80)}...`);
|
|
818
|
-
}
|
|
819
|
-
} else if (device.platform === 'ios') {
|
|
820
|
-
cachedPort = device.port || 8100;
|
|
821
|
-
try {
|
|
822
|
-
const sessionId = await this.sessionManager.getLiveSessionId(device.name, cachedPort);
|
|
823
|
-
cachedScreenshotUrl = `http://localhost:${cachedPort}/screenshot`;
|
|
824
|
-
this.logger.info(`✅ Pre-validated iOS WDA session on port ${cachedPort}`);
|
|
825
|
-
} catch (error) {
|
|
826
|
-
this.logger.info(`⚡ Will try session discovery during first screenshot - continuing optimistically`);
|
|
827
|
-
cachedScreenshotUrl = `http://localhost:${cachedPort}/screenshot`;
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
} catch (error) {
|
|
831
|
-
this.logger.warn(`⚠️ Session pre-validation failed, will discover during streaming: ${error.message}`);
|
|
832
|
-
}
|
|
833
|
-
|
|
834
|
-
let lastScreenHash = null;
|
|
835
|
-
let frameCount = 0;
|
|
836
|
-
let consecutiveErrors = 0;
|
|
837
|
-
|
|
838
|
-
// Stream at 2 FPS (every 500ms) with optimized error handling
|
|
839
|
-
this.screenStreamInterval = setInterval(async () => {
|
|
840
|
-
try {
|
|
841
|
-
const device = this.commanderDevice;
|
|
842
|
-
|
|
843
|
-
if (!device) {
|
|
844
|
-
this.logger.error('❌ Commander device is null during streaming');
|
|
845
|
-
return;
|
|
846
|
-
}
|
|
847
|
-
|
|
848
|
-
// Use cached screenshot URL first, or fallback to session discovery
|
|
849
|
-
let screenshotUrl = cachedScreenshotUrl;
|
|
850
|
-
let port = cachedPort;
|
|
851
|
-
|
|
852
|
-
// If we don't have cached URL, do fast session lookup
|
|
853
|
-
if (!screenshotUrl) {
|
|
854
|
-
if (device.platform === 'android') {
|
|
855
|
-
// For Android UIAutomator2, use session-based endpoint with global session manager
|
|
856
|
-
const session = await this.getAndroidSession(device);
|
|
857
|
-
|
|
858
|
-
if (!session) {
|
|
859
|
-
consecutiveErrors++;
|
|
860
|
-
if (consecutiveErrors <= 3) { // Only log for first few errors
|
|
861
|
-
this.logger.warn(`⚠️ No Android UIAutomator2 session found for ${device.name} (attempt ${consecutiveErrors})`);
|
|
862
|
-
}
|
|
863
|
-
return;
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
port = session.port;
|
|
867
|
-
screenshotUrl = `http://localhost:${port}/wd/hub/session/${session.sessionId}/screenshot`;
|
|
868
|
-
// Cache for future use
|
|
869
|
-
cachedPort = port;
|
|
870
|
-
cachedScreenshotUrl = screenshotUrl;
|
|
871
|
-
|
|
872
|
-
} else if (device.platform === 'ios') {
|
|
873
|
-
// iOS WDA uses simple screenshot endpoint with faster port detection
|
|
874
|
-
port = device.port || cachedPort || 8100;
|
|
875
|
-
screenshotUrl = `http://localhost:${port}/screenshot`;
|
|
876
|
-
|
|
877
|
-
// Only do session validation if we haven't cached it yet
|
|
878
|
-
if (!cachedScreenshotUrl) {
|
|
879
|
-
try {
|
|
880
|
-
// Quick session check with reduced timeout
|
|
881
|
-
const sessionId = await this.sessionManager.getLiveSessionId(device.name, port);
|
|
882
|
-
// Cache successful session info
|
|
883
|
-
cachedPort = port;
|
|
884
|
-
cachedScreenshotUrl = screenshotUrl;
|
|
885
|
-
} catch (error) {
|
|
886
|
-
// Try one alternative port quickly
|
|
887
|
-
const altPort = port === 8100 ? 8101 : 8100;
|
|
888
|
-
try {
|
|
889
|
-
await this.sessionManager.getLiveSessionId(device.name, altPort);
|
|
890
|
-
port = altPort;
|
|
891
|
-
screenshotUrl = `http://localhost:${altPort}/screenshot`;
|
|
892
|
-
// Cache successful session info
|
|
893
|
-
cachedPort = port;
|
|
894
|
-
cachedScreenshotUrl = screenshotUrl;
|
|
895
|
-
device.port = altPort; // Update device port
|
|
896
|
-
} catch (altError) {
|
|
897
|
-
consecutiveErrors++;
|
|
898
|
-
if (consecutiveErrors <= 3) {
|
|
899
|
-
this.logger.warn(`⚠️ No WDA session found for ${device.name} on ports ${port} or ${altPort} (attempt ${consecutiveErrors})`);
|
|
900
|
-
}
|
|
901
|
-
return;
|
|
902
|
-
}
|
|
903
|
-
}
|
|
904
|
-
}
|
|
905
|
-
}
|
|
906
|
-
}
|
|
907
|
-
|
|
908
|
-
// Get screenshot from WDA/UIAutomator2 with optimized timeout
|
|
909
|
-
const response = await axios.get(screenshotUrl, {
|
|
910
|
-
timeout: 3000, // Reduced timeout for faster response
|
|
911
|
-
validateStatus: () => true // Don't throw on any status
|
|
912
|
-
});
|
|
913
|
-
|
|
914
|
-
if (response.status !== 200) {
|
|
915
|
-
consecutiveErrors++;
|
|
916
|
-
// Clear cache if we have errors - might be stale
|
|
917
|
-
if (consecutiveErrors >= 2) {
|
|
918
|
-
cachedScreenshotUrl = null;
|
|
919
|
-
cachedPort = null;
|
|
920
|
-
}
|
|
921
|
-
|
|
922
|
-
if (consecutiveErrors <= 3) { // Only log for first few errors
|
|
923
|
-
this.logger.warn(`⚠️ Screenshot failed with status ${response.status} for ${device.name} (attempt ${consecutiveErrors})`);
|
|
924
|
-
if (response.status === 404) {
|
|
925
|
-
this.logger.debug(`Session might be expired - will retry session discovery`);
|
|
926
|
-
} else if (response.status === 500) {
|
|
927
|
-
this.logger.debug(`Device might be locked or disconnected`);
|
|
928
|
-
}
|
|
929
|
-
}
|
|
930
|
-
return;
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
// Reset error counter on success
|
|
934
|
-
consecutiveErrors = 0;
|
|
935
|
-
|
|
936
|
-
const base64Image = response.data?.value || response.data;
|
|
937
|
-
|
|
938
|
-
if (!base64Image || typeof base64Image !== 'string' || base64Image.length < 100) {
|
|
939
|
-
this.logger.debug(`❌ Invalid image data from ${device.name}: length=${base64Image?.length}`);
|
|
940
|
-
return;
|
|
941
|
-
}
|
|
942
|
-
|
|
943
|
-
// Detect screen changes for auto-refresh
|
|
944
|
-
const currentHash = base64Image.substring(0, 100); // Simple hash using first 100 chars
|
|
945
|
-
const screenChanged = lastScreenHash && lastScreenHash !== currentHash;
|
|
946
|
-
lastScreenHash = currentHash;
|
|
947
|
-
frameCount++;
|
|
948
|
-
|
|
949
|
-
// Get device window size for coordinate scaling
|
|
950
|
-
const windowSize = await this.getDeviceWindowSize(port);
|
|
951
|
-
|
|
952
|
-
// Broadcast commander screen to all connected clients
|
|
953
|
-
this.broadcast({
|
|
954
|
-
type: 'screen_frame',
|
|
955
|
-
image: `data:image/png;base64,${base64Image}`,
|
|
956
|
-
deviceName: device.name,
|
|
957
|
-
windowSize: windowSize
|
|
958
|
-
});
|
|
959
|
-
|
|
960
|
-
// Stream follower devices screens
|
|
961
|
-
this.streamFollowerScreens();
|
|
962
|
-
|
|
963
|
-
// DISABLED: Auto-refresh locators on screen change (too aggressive)
|
|
964
|
-
// Only refresh manually or after user clicks
|
|
965
|
-
// if (screenChanged && frameCount > 5) { // Wait for 5 frames before detecting changes
|
|
966
|
-
// this.logger.info('📍 Screen changed - auto-refreshing locators...');
|
|
967
|
-
// this.broadcast({
|
|
968
|
-
// type: 'locators_reloading',
|
|
969
|
-
// message: 'Locators are reloading...'
|
|
970
|
-
// });
|
|
971
|
-
// this.cachedElements = [];
|
|
972
|
-
// this.lastCacheTime = 0;
|
|
973
|
-
// this.cacheAllElements().catch(err =>
|
|
974
|
-
// this.logger.warn(`Auto-refresh failed: ${err.message}`)
|
|
975
|
-
// );
|
|
976
|
-
// }
|
|
977
|
-
|
|
978
|
-
// DISABLED: Load locators only once after first frame (too aggressive)
|
|
979
|
-
// Only load locators manually when user requests via UI
|
|
980
|
-
// if (this.cachedElements.length === 0 && !this.isCaching && Date.now() - this.lastCacheTime > 5000) {
|
|
981
|
-
// this.broadcast({
|
|
982
|
-
// type: 'locators_loading',
|
|
983
|
-
// message: 'Loading locators...'
|
|
984
|
-
// });
|
|
985
|
-
// this.cacheAllElements().catch(err =>
|
|
986
|
-
// this.logger.warn(`Background locator loading failed: ${err.message}`)
|
|
987
|
-
// );
|
|
988
|
-
// }
|
|
989
|
-
|
|
990
|
-
} catch (error) {
|
|
991
|
-
// CRITICAL: Better error handling for commander screen streaming
|
|
992
|
-
if (error.message.includes('socket hang up')) {
|
|
993
|
-
this.logger.error(`❌ Commander WDA connection lost (socket hang up) - attempting recovery`);
|
|
994
|
-
|
|
995
|
-
// Try to recover the session
|
|
996
|
-
setTimeout(async () => {
|
|
997
|
-
try {
|
|
998
|
-
this.logger.info('🔧 Attempting commander session recovery...');
|
|
999
|
-
await this.checkExistingWDASession();
|
|
1000
|
-
} catch (recoveryError) {
|
|
1001
|
-
this.logger.error(`❌ Session recovery failed: ${recoveryError.message}`);
|
|
1002
|
-
}
|
|
1003
|
-
}, 2000);
|
|
1004
|
-
|
|
1005
|
-
} else if (error.code === 'ECONNREFUSED') {
|
|
1006
|
-
this.logger.error(`❌ Commander WDA not responding - connection refused`);
|
|
1007
|
-
} else {
|
|
1008
|
-
this.logger.error(`❌ Commander screenshot error: ${error.message}`);
|
|
1009
|
-
}
|
|
1010
|
-
}
|
|
1011
|
-
}, 500);
|
|
1012
|
-
}
|
|
1013
|
-
|
|
1014
|
-
async streamFollowerScreens() {
|
|
1015
|
-
if (!this.followerDevices || this.followerDevices.size === 0) return;
|
|
1016
|
-
|
|
1017
|
-
const axios = require('axios');
|
|
1018
|
-
|
|
1019
|
-
// Stream each follower device screen with optimized session handling
|
|
1020
|
-
for (const follower of this.followerDevices) {
|
|
1021
|
-
try {
|
|
1022
|
-
// Use cached screenshot URL if available, otherwise do fast discovery
|
|
1023
|
-
let screenshotUrl = follower._cachedScreenshotUrl;
|
|
1024
|
-
let port = follower._cachedPort;
|
|
1025
|
-
|
|
1026
|
-
if (!screenshotUrl) {
|
|
1027
|
-
// Fast session discovery for followers
|
|
1028
|
-
if (follower.platform === 'android') {
|
|
1029
|
-
const session = await this.getAndroidSession(follower);
|
|
1030
|
-
|
|
1031
|
-
if (!session || !session.port || !session.sessionId) {
|
|
1032
|
-
follower._consecutiveErrors = (follower._consecutiveErrors || 0) + 1;
|
|
1033
|
-
if (follower._consecutiveErrors <= 2) { // Reduced error logging
|
|
1034
|
-
this.logger.debug(`⚠️ No UIAutomator2 session for follower: ${follower.name} (attempt ${follower._consecutiveErrors})`);
|
|
1035
|
-
}
|
|
1036
|
-
continue;
|
|
1037
|
-
}
|
|
1038
|
-
|
|
1039
|
-
port = session.port;
|
|
1040
|
-
screenshotUrl = `http://localhost:${port}/wd/hub/session/${session.sessionId}/screenshot`;
|
|
1041
|
-
|
|
1042
|
-
// Cache for future use
|
|
1043
|
-
follower._cachedPort = port;
|
|
1044
|
-
follower._cachedScreenshotUrl = screenshotUrl;
|
|
1045
|
-
|
|
1046
|
-
} else if (follower.platform === 'ios') {
|
|
1047
|
-
port = follower.port || 8100;
|
|
1048
|
-
screenshotUrl = `http://localhost:${port}/screenshot`;
|
|
1049
|
-
|
|
1050
|
-
// Quick session validation for iOS followers (only if not cached)
|
|
1051
|
-
if (!follower._cachedScreenshotUrl) {
|
|
1052
|
-
try {
|
|
1053
|
-
await this.sessionManager.getLiveSessionId(follower.name, port);
|
|
1054
|
-
// Cache successful session info
|
|
1055
|
-
follower._cachedPort = port;
|
|
1056
|
-
follower._cachedScreenshotUrl = screenshotUrl;
|
|
1057
|
-
} catch (error) {
|
|
1058
|
-
// Try one alternative port quickly
|
|
1059
|
-
const altPort = port === 8100 ? 8101 : 8100;
|
|
1060
|
-
try {
|
|
1061
|
-
await this.sessionManager.getLiveSessionId(follower.name, altPort);
|
|
1062
|
-
port = altPort;
|
|
1063
|
-
screenshotUrl = `http://localhost:${altPort}/screenshot`;
|
|
1064
|
-
follower.port = altPort;
|
|
1065
|
-
follower._cachedPort = port;
|
|
1066
|
-
follower._cachedScreenshotUrl = screenshotUrl;
|
|
1067
|
-
} catch (altError) {
|
|
1068
|
-
follower._consecutiveErrors = (follower._consecutiveErrors || 0) + 1;
|
|
1069
|
-
if (follower._consecutiveErrors <= 2) {
|
|
1070
|
-
this.logger.debug(`⚠️ No WDA session for follower ${follower.name} on ports ${port} or ${altPort}`);
|
|
1071
|
-
}
|
|
1072
|
-
continue;
|
|
1073
|
-
}
|
|
1074
|
-
}
|
|
1075
|
-
}
|
|
1076
|
-
}
|
|
1077
|
-
}
|
|
1078
|
-
|
|
1079
|
-
// Get screenshot with optimized timeout
|
|
1080
|
-
const response = await axios.get(screenshotUrl, {
|
|
1081
|
-
timeout: 3000, // Reduced from 5000ms to 3000ms for faster follower streaming
|
|
1082
|
-
validateStatus: () => true // Don't throw on any status
|
|
1083
|
-
});
|
|
1084
|
-
|
|
1085
|
-
if (response.status !== 200) {
|
|
1086
|
-
follower._consecutiveErrors = (follower._consecutiveErrors || 0) + 1;
|
|
1087
|
-
|
|
1088
|
-
// Clear cache if we have persistent errors
|
|
1089
|
-
if (follower._consecutiveErrors >= 2) {
|
|
1090
|
-
follower._cachedScreenshotUrl = null;
|
|
1091
|
-
follower._cachedPort = null;
|
|
1092
|
-
}
|
|
1093
|
-
|
|
1094
|
-
if (follower._consecutiveErrors <= 2) {
|
|
1095
|
-
this.logger.debug(`⚠️ Follower ${follower.name} returned status ${response.status} (attempt ${follower._consecutiveErrors})`);
|
|
1096
|
-
}
|
|
1097
|
-
continue;
|
|
1098
|
-
}
|
|
1099
|
-
|
|
1100
|
-
// Reset error counter on success
|
|
1101
|
-
follower._consecutiveErrors = 0;
|
|
1102
|
-
|
|
1103
|
-
const base64Image = response.data?.value || response.data;
|
|
1104
|
-
|
|
1105
|
-
if (base64Image && typeof base64Image === 'string' && base64Image.length > 100) {
|
|
1106
|
-
this.broadcast({
|
|
1107
|
-
type: 'follower_screen',
|
|
1108
|
-
deviceId: follower.name,
|
|
1109
|
-
image: `data:image/png;base64,${base64Image}`
|
|
1110
|
-
});
|
|
1111
|
-
this.logger.debug(`📸 Streamed follower screen: ${follower.name}`);
|
|
1112
|
-
} else {
|
|
1113
|
-
this.logger.warn(`⚠️ No image data for follower: ${follower.name}`);
|
|
1114
|
-
}
|
|
1115
|
-
} catch (error) {
|
|
1116
|
-
if (error.message.includes('socket hang up')) {
|
|
1117
|
-
this.logger.warn(`⚠️ Follower ${follower.name} WDA connection lost - session may need refresh`);
|
|
1118
|
-
|
|
1119
|
-
// For followers, try to refresh their session
|
|
1120
|
-
if (follower.platform === 'ios') {
|
|
1121
|
-
setTimeout(async () => {
|
|
1122
|
-
try {
|
|
1123
|
-
this.logger.info(`🔧 Refreshing WDA session for follower ${follower.name}...`);
|
|
1124
|
-
await this.checkFollowerWDASession(follower);
|
|
1125
|
-
} catch (refreshError) {
|
|
1126
|
-
this.logger.warn(`⚠️ Failed to refresh follower session: ${refreshError.message}`);
|
|
1127
|
-
}
|
|
1128
|
-
}, 3000);
|
|
1129
|
-
}
|
|
1130
|
-
|
|
1131
|
-
} else {
|
|
1132
|
-
this.logger.warn(`⚠️ Follower ${follower.name} screenshot error: ${error.message}`);
|
|
1133
|
-
}
|
|
1134
|
-
}
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
async getDeviceWindowSize(port) {
|
|
1139
|
-
try {
|
|
1140
|
-
const axios = require('axios');
|
|
1141
|
-
let size = null;
|
|
1142
|
-
|
|
1143
|
-
// Try dedicated window size endpoint first (most reliable for logical points)
|
|
1144
|
-
const wdaSessionId = await this.getWDASessionId();
|
|
1145
|
-
if (wdaSessionId) {
|
|
1146
|
-
try {
|
|
1147
|
-
const res = await axios.get(`http://localhost:${port}/session/${wdaSessionId}/window/size`, { timeout: 1000 });
|
|
1148
|
-
if (res.data?.value) {
|
|
1149
|
-
size = res.data.value;
|
|
1150
|
-
this.logger.debug(`Window size from /window/size: ${size.width}x${size.height}`);
|
|
1151
|
-
}
|
|
1152
|
-
} catch (e) { /* ignore */ }
|
|
1153
|
-
}
|
|
1154
|
-
|
|
1155
|
-
// Fallback to /status if session endpoint failed
|
|
1156
|
-
if (!size) {
|
|
1157
|
-
const response = await axios.get(`http://localhost:${port}/status`, { timeout: 1000 });
|
|
1158
|
-
if (response.data?.value?.screenSize) {
|
|
1159
|
-
size = response.data.value.screenSize;
|
|
1160
|
-
this.logger.debug(`Window size from /status: ${size.width}x${size.height}`);
|
|
1161
|
-
}
|
|
1162
|
-
}
|
|
1163
|
-
|
|
1164
|
-
if (size && size.width > 0) {
|
|
1165
|
-
this.lastWindowSize = {
|
|
1166
|
-
width: size.width,
|
|
1167
|
-
height: size.height
|
|
1168
|
-
};
|
|
1169
|
-
}
|
|
1170
|
-
|
|
1171
|
-
return this.lastWindowSize;
|
|
1172
|
-
} catch (error) {
|
|
1173
|
-
return this.lastWindowSize;
|
|
1174
|
-
}
|
|
1175
|
-
}
|
|
1176
|
-
|
|
1177
|
-
async handleScreenClick(x, y, windowSize) {
|
|
1178
|
-
// GENIUS: Queue system for handling rapid clicks
|
|
1179
|
-
return new Promise((resolve, reject) => {
|
|
1180
|
-
const clickRequest = { x, y, windowSize, resolve, reject, timestamp: Date.now() };
|
|
1181
|
-
|
|
1182
|
-
// Limit queue size to prevent memory issues
|
|
1183
|
-
if (this.clickQueue.length >= this.maxQueueSize) {
|
|
1184
|
-
this.logger.warn(`⚠️ Click queue full (${this.maxQueueSize}), dropping oldest click`);
|
|
1185
|
-
const dropped = this.clickQueue.shift();
|
|
1186
|
-
dropped.reject(new Error('Click dropped due to queue overflow'));
|
|
1187
|
-
}
|
|
1188
|
-
|
|
1189
|
-
this.clickQueue.push(clickRequest);
|
|
1190
|
-
this.processClickQueue();
|
|
1191
|
-
});
|
|
1192
|
-
}
|
|
1193
|
-
|
|
1194
|
-
async processClickQueue() {
|
|
1195
|
-
if (this.processingClick || this.clickQueue.length === 0) {
|
|
1196
|
-
return;
|
|
1197
|
-
}
|
|
1198
|
-
|
|
1199
|
-
this.processingClick = true;
|
|
1200
|
-
|
|
1201
|
-
try {
|
|
1202
|
-
while (this.clickQueue.length > 0) {
|
|
1203
|
-
const clickRequest = this.clickQueue.shift();
|
|
1204
|
-
const age = Date.now() - clickRequest.timestamp;
|
|
1205
|
-
|
|
1206
|
-
// Skip very old clicks (>2 seconds)
|
|
1207
|
-
if (age > 2000) {
|
|
1208
|
-
this.logger.warn(`⚠️ Skipping stale click (${age}ms old)`);
|
|
1209
|
-
clickRequest.reject(new Error('Click too old'));
|
|
1210
|
-
continue;
|
|
1211
|
-
}
|
|
1212
|
-
|
|
1213
|
-
try {
|
|
1214
|
-
this.logger.info(`🔄 Processing queued click (${this.clickQueue.length} remaining)`);
|
|
1215
|
-
const result = await this._handleScreenClickInternal(clickRequest.x, clickRequest.y, clickRequest.windowSize);
|
|
1216
|
-
clickRequest.resolve(result);
|
|
1217
|
-
} catch (error) {
|
|
1218
|
-
clickRequest.reject(error);
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
// Small delay between clicks to prevent overwhelming the system
|
|
1222
|
-
if (this.clickQueue.length > 0) {
|
|
1223
|
-
await new Promise(resolve => setTimeout(resolve, 200));
|
|
1224
|
-
}
|
|
1225
|
-
}
|
|
1226
|
-
} finally {
|
|
1227
|
-
this.processingClick = false;
|
|
1228
|
-
}
|
|
1229
|
-
}
|
|
1230
|
-
|
|
1231
|
-
async _handleScreenClickInternal(x, y, windowSize) {
|
|
1232
|
-
if (!this.isActive) {
|
|
1233
|
-
throw new Error('Commander mode not active');
|
|
1234
|
-
}
|
|
1235
|
-
|
|
1236
|
-
if (!this.commanderDevice) {
|
|
1237
|
-
throw new Error('No commander device available');
|
|
1238
|
-
}
|
|
1239
|
-
|
|
1240
|
-
this.logger.info('');
|
|
1241
|
-
this.logger.info('═══════════════════════════════════════════════════════');
|
|
1242
|
-
this.logger.info(`🖱️ CLICK RECEIVED: (${x}, ${y})`);
|
|
1243
|
-
this.logger.info('═══════════════════════════════════════════════════════');
|
|
1244
|
-
|
|
1245
|
-
// Determine if incoming coordinates are physical or logical
|
|
1246
|
-
// windowSize parameter is what the frontend THINKS is the total size
|
|
1247
|
-
let logicalX = x;
|
|
1248
|
-
let logicalY = y;
|
|
1249
|
-
|
|
1250
|
-
// If the windowSize provided by frontend is physical resolution, convert to logical
|
|
1251
|
-
if (windowSize && windowSize.width > 600) { // Likely physical
|
|
1252
|
-
const scaleX = this.lastWindowSize.width / windowSize.width;
|
|
1253
|
-
const scaleY = this.lastWindowSize.height / windowSize.height;
|
|
1254
|
-
logicalX = Math.round(x * scaleX);
|
|
1255
|
-
logicalY = Math.round(y * scaleY);
|
|
1256
|
-
this.logger.info(`📐 Converting physical (${x}, ${y}) to logical (${logicalX}, ${logicalY})`);
|
|
1257
|
-
this.logger.info(`📏 Scale factors: X=${scaleX.toFixed(3)}, Y=${scaleY.toFixed(3)}`);
|
|
1258
|
-
this.logger.info(`📱 Frontend size: ${windowSize.width}x${windowSize.height}, Device size: ${this.lastWindowSize.width}x${this.lastWindowSize.height}`);
|
|
1259
|
-
} else {
|
|
1260
|
-
this.logger.info(`📍 Using logical coordinates: (${logicalX}, ${logicalY})`);
|
|
1261
|
-
this.logger.info(`📏 Frontend size: ${windowSize?.width || 'unknown'}x${windowSize?.height || 'unknown'}`);
|
|
1262
|
-
}
|
|
1263
|
-
|
|
1264
|
-
this.logger.info(`🎯 Target: ${this.commanderDevice.name}`);
|
|
1265
|
-
|
|
1266
|
-
// 🚀 SIMPLE APP LAUNCH DETECTION: Check home screen FIRST
|
|
1267
|
-
const port = this.commanderDevice.port || 8100;
|
|
1268
|
-
|
|
1269
|
-
// REVOLUTIONARY: Get live session ID for pre-checks
|
|
1270
|
-
let liveSessionId = null;
|
|
1271
|
-
let isOnHomeScreen = false;
|
|
1272
|
-
|
|
1273
|
-
try {
|
|
1274
|
-
liveSessionId = await this.sessionManager.getSessionWithRetry(this.commanderDevice.name, port);
|
|
1275
|
-
|
|
1276
|
-
try {
|
|
1277
|
-
const preAppRes = await axios.get(`http://localhost:${port}/session/${liveSessionId}/wda/activeAppInfo`, { timeout: 2000 });
|
|
1278
|
-
const preAppId = preAppRes.data?.value?.bundleId;
|
|
1279
|
-
isOnHomeScreen = preAppId === 'com.apple.springboard';
|
|
1280
|
-
this.logger.info(`🏠 SIMPLE CHECK: ${preAppId} ${isOnHomeScreen ? '(HOME SCREEN - WILL SKIP FOLLOWER COORDS)' : '(IN APP - NORMAL FLOW)'}`);
|
|
1281
|
-
} catch (preAppError) {
|
|
1282
|
-
this.logger.debug(`🏠 Pre-check failed: ${preAppError.message}`);
|
|
1283
|
-
isOnHomeScreen = false;
|
|
1284
|
-
}
|
|
1285
|
-
} catch (sessionError) {
|
|
1286
|
-
this.logger.warn(`⚠️ Failed to get live session for pre-check: ${sessionError.message}`);
|
|
1287
|
-
isOnHomeScreen = false;
|
|
1288
|
-
}
|
|
1289
|
-
|
|
1290
|
-
try {
|
|
1291
|
-
// ===============================
|
|
1292
|
-
// STEP -1: REVOLUTIONARY SESSION MANAGEMENT
|
|
1293
|
-
// ===============================
|
|
1294
|
-
|
|
1295
|
-
// ALWAYS get fresh live session ID - no more stored sessions!
|
|
1296
|
-
const liveSession = await this.sessionManager.getSessionWithRetry(this.commanderDevice.name, port);
|
|
1297
|
-
this.logger.info(`🎯 Using live WDA session: ${liveSession.substring(0, 8)}... for ${this.commanderDevice.name}`);
|
|
1298
|
-
|
|
1299
|
-
let isSessionHealthy = false;
|
|
1300
|
-
|
|
1301
|
-
// Quick health check using window size (fast and reliable)
|
|
1302
|
-
try {
|
|
1303
|
-
const windowRes = await axios.get(`http://localhost:${port}/session/${liveSession}/window/size`, { timeout: 2000 });
|
|
1304
|
-
if (windowRes.data?.value) {
|
|
1305
|
-
this.logger.info(`✅ Session healthy: ${liveSession.substring(0, 8)}...`);
|
|
1306
|
-
isSessionHealthy = true;
|
|
1307
|
-
// Store window size for coordinate scaling
|
|
1308
|
-
this.lastWindowSize = windowRes.data.value;
|
|
1309
|
-
}
|
|
1310
|
-
} catch (healthError) {
|
|
1311
|
-
this.logger.warn(`⚠️ Session health check failed: ${healthError.message}`);
|
|
1312
|
-
|
|
1313
|
-
// For 404 errors, try getting a fresh session
|
|
1314
|
-
if (healthError.response?.status === 404) {
|
|
1315
|
-
this.logger.warn(`🔄 Session expired (404), getting fresh session...`);
|
|
1316
|
-
try {
|
|
1317
|
-
const freshSession = await this.sessionManager.getSessionWithRetry(this.commanderDevice.name, port);
|
|
1318
|
-
|
|
1319
|
-
// Test the fresh session
|
|
1320
|
-
const windowRes = await axios.get(`http://localhost:${port}/session/${freshSession}/window/size`, { timeout: 2000 });
|
|
1321
|
-
if (windowRes.data?.value) {
|
|
1322
|
-
this.logger.info(`✅ Fresh session healthy: ${freshSession.substring(0, 8)}...`);
|
|
1323
|
-
isSessionHealthy = true;
|
|
1324
|
-
this.lastWindowSize = windowRes.data.value;
|
|
1325
|
-
}
|
|
1326
|
-
} catch (recoveryError) {
|
|
1327
|
-
this.logger.error(`❌ Session recovery failed: ${recoveryError.message}`);
|
|
1328
|
-
}
|
|
1329
|
-
}
|
|
1330
|
-
}
|
|
1331
|
-
|
|
1332
|
-
// If no session is healthy, use default window size
|
|
1333
|
-
if (!isSessionHealthy) {
|
|
1334
|
-
this.logger.warn(`⚠️ No healthy session found, using default window size`);
|
|
1335
|
-
this.lastWindowSize = { width: 414, height: 896 };
|
|
1336
|
-
}
|
|
1337
|
-
|
|
1338
|
-
// ===============================
|
|
1339
|
-
// STEP 0: NO AUTO LOCATOR CACHING - MANUAL ONLY
|
|
1340
|
-
// ===============================
|
|
1341
|
-
// Cache age checking disabled - only manual refresh via UI
|
|
1342
|
-
// const cacheAge = Date.now() - this.lastCacheTime;
|
|
1343
|
-
// const needsRefresh = this.cachedElements.length === 0 || cacheAge > 30000;
|
|
1344
|
-
|
|
1345
|
-
// DISABLED: Auto-refresh on clicks - use existing cache or user can manually refresh
|
|
1346
|
-
// if (needsRefresh && !this.isCaching && this.cachedElements.length < 3) {
|
|
1347
|
-
// this.logger.info(`🔄 Refreshing locators before click (${this.cachedElements.length} cached, ${cacheAge}ms old)`);
|
|
1348
|
-
// await this.fastCacheAllElements();
|
|
1349
|
-
// } else if (this.cachedElements.length > 0) {
|
|
1350
|
-
// this.logger.debug(`⚡ Skipping cache refresh for speed (${this.cachedElements.length} elements, ${cacheAge}ms old)`);
|
|
1351
|
-
// }
|
|
1352
|
-
|
|
1353
|
-
// ===============================
|
|
1354
|
-
// 🎯 STEP 1: INTELLIGENT LOCATOR DETECTION (NEW GENIUS METHOD)
|
|
1355
|
-
// ===============================
|
|
1356
|
-
let intelligentLocator = null;
|
|
1357
|
-
|
|
1358
|
-
if (isOnHomeScreen) {
|
|
1359
|
-
// 🚀 SIMPLE: Skip intelligent locator for app launches, use coordinates directly
|
|
1360
|
-
this.logger.info(`🚀 SIMPLE: Skipping intelligent locator - home screen app launch, using coordinates`);
|
|
1361
|
-
intelligentLocator = { method: 'coordinates', x: logicalX, y: logicalY };
|
|
1362
|
-
} else {
|
|
1363
|
-
this.logger.info(`🎯 INTELLIGENT LOCATOR: Finding smart locator for (${logicalX}, ${logicalY})...`);
|
|
1364
|
-
|
|
1365
|
-
if (this.useIntelligentLocators) {
|
|
1366
|
-
try {
|
|
1367
|
-
// CRITICAL FIX: Ensure cache is fresh for current screen state
|
|
1368
|
-
// Check cache age and refresh if it's older than a few seconds
|
|
1369
|
-
// PERFORMANCE: Optimize cache management for commander speed - SAFER VERSION
|
|
1370
|
-
const cacheAge = this.intelligentLocator.getCacheAge(this.commanderDevice.name);
|
|
1371
|
-
|
|
1372
|
-
// More conservative cache strategy to prevent issues
|
|
1373
|
-
const hasValidCache = cacheAge !== null && cacheAge < 15000; // REDUCED: 15 seconds (was 20)
|
|
1374
|
-
const needsRefresh = !hasValidCache;
|
|
1375
|
-
|
|
1376
|
-
if (needsRefresh) {
|
|
1377
|
-
this.logger.info(`🔄 Cache is stale (age: ${cacheAge}ms) - refreshing before locator search...`);
|
|
1378
|
-
} else {
|
|
1379
|
-
this.logger.debug(`⚡ Using existing cache (age: ${cacheAge}ms) for commander speed`);
|
|
1380
|
-
}
|
|
1381
|
-
|
|
1382
|
-
// Get intelligent locator - only force refresh if truly needed
|
|
1383
|
-
intelligentLocator = await this.intelligentLocator.getIntelligentLocator(
|
|
1384
|
-
this.commanderDevice.name,
|
|
1385
|
-
logicalX,
|
|
1386
|
-
logicalY,
|
|
1387
|
-
needsRefresh // Force refresh if cache is stale
|
|
1388
|
-
);
|
|
1389
|
-
|
|
1390
|
-
if (intelligentLocator.method === 'locator') {
|
|
1391
|
-
this.logger.info(`✅ INTELLIGENT: Found ${intelligentLocator.locator.strategy} locator with ${(intelligentLocator.confidence * 100).toFixed(1)}% confidence`);
|
|
1392
|
-
this.logger.info(` Strategy: ${intelligentLocator.locator.strategy} = "${intelligentLocator.locator.selector}"`);
|
|
1393
|
-
this.logger.info(` Element: ${intelligentLocator.element.elementType} "${intelligentLocator.element.text || intelligentLocator.element.accessibilityLabel || 'No text'}"`);
|
|
1394
|
-
} else {
|
|
1395
|
-
this.logger.warn(`⚠️ INTELLIGENT: Falling back to coordinates (${intelligentLocator.method})`);
|
|
1396
|
-
this.coordinateFallbackCount++;
|
|
1397
|
-
}
|
|
1398
|
-
|
|
1399
|
-
} catch (error) {
|
|
1400
|
-
this.logger.error(`❌ INTELLIGENT: Locator detection failed: ${error.message}`);
|
|
1401
|
-
intelligentLocator = { method: 'coordinates', x: logicalX, y: logicalY };
|
|
1402
|
-
this.coordinateFallbackCount++;
|
|
1403
|
-
}
|
|
1404
|
-
} else {
|
|
1405
|
-
// Traditional method for compatibility
|
|
1406
|
-
this.logger.info(`🔍 TRADITIONAL: Finding element at target location...`);
|
|
1407
|
-
const preClickElement = this.findClosestElement(logicalX, logicalY);
|
|
1408
|
-
|
|
1409
|
-
if (preClickElement) {
|
|
1410
|
-
intelligentLocator = {
|
|
1411
|
-
method: 'locator',
|
|
1412
|
-
locator: {
|
|
1413
|
-
strategy: 'accessibility_id',
|
|
1414
|
-
selector: preClickElement.accessibilityId || preClickElement.label || preClickElement.name
|
|
1415
|
-
},
|
|
1416
|
-
fallbackCoords: { x: logicalX, y: logicalY },
|
|
1417
|
-
element: preClickElement
|
|
1418
|
-
};
|
|
1419
|
-
this.logger.info(`✅ TRADITIONAL: Found element: ${preClickElement.label}`);
|
|
1420
|
-
} else {
|
|
1421
|
-
intelligentLocator = { method: 'coordinates', x: logicalX, y: logicalY };
|
|
1422
|
-
this.logger.warn(`⚠️ TRADITIONAL: No element found, using coordinates`);
|
|
1423
|
-
}
|
|
1424
|
-
}
|
|
1425
|
-
} // Close the else block for non-home screen processing
|
|
1426
|
-
|
|
1427
|
-
// ===============================
|
|
1428
|
-
// STEP 2: EXECUTE CLICK ON COMMANDER
|
|
1429
|
-
// ===============================
|
|
1430
|
-
// Get fresh live session using universal session manager
|
|
1431
|
-
const device = this.commanderDevice;
|
|
1432
|
-
|
|
1433
|
-
// REVOLUTIONARY: Always get current live session
|
|
1434
|
-
let currentSession;
|
|
1435
|
-
try {
|
|
1436
|
-
currentSession = await this.sessionManager.getSessionWithRetry(device.name, port);
|
|
1437
|
-
this.logger.info(`📱 Using WDA session: ${currentSession.substring(0, 8)}... on port ${port}`);
|
|
1438
|
-
} catch (sessionError) {
|
|
1439
|
-
throw new Error(`No valid WDA session found for ${device.name}`);
|
|
1440
|
-
}
|
|
1441
|
-
|
|
1442
|
-
let retryAttempted = false;
|
|
1443
|
-
|
|
1444
|
-
// Execute tap on commander using LOGICAL coordinates - AWAIT to ensure it completes
|
|
1445
|
-
try {
|
|
1446
|
-
this.logger.info(`⏳ Executing tap on commander device...`);
|
|
1447
|
-
this.logger.info(`📍 Final tap coordinates: (${logicalX}, ${logicalY})`);
|
|
1448
|
-
|
|
1449
|
-
// Simple coordinate validation - only prevent obviously dangerous coordinates
|
|
1450
|
-
let safeX = Math.round(logicalX);
|
|
1451
|
-
let safeY = Math.round(logicalY);
|
|
1452
|
-
|
|
1453
|
-
const deviceHeight = this.lastWindowSize?.height || 896;
|
|
1454
|
-
const deviceWidth = this.lastWindowSize?.width || 414;
|
|
1455
|
-
|
|
1456
|
-
// Basic bounds checking only (don't modify unless clearly outside device bounds)
|
|
1457
|
-
if (safeX < 0 || safeX > deviceWidth || safeY < 0 || safeY > deviceHeight) {
|
|
1458
|
-
this.logger.warn(`⚠️ Coordinates outside device bounds, clamping: (${safeX}, ${safeY})`);
|
|
1459
|
-
safeX = Math.max(10, Math.min(safeX, deviceWidth - 10));
|
|
1460
|
-
safeY = Math.max(50, Math.min(safeY, deviceHeight - 100)); // Avoid control center
|
|
1461
|
-
}
|
|
1462
|
-
|
|
1463
|
-
// CRITICAL FIX: Use current session from above
|
|
1464
|
-
await axios.post(
|
|
1465
|
-
`http://localhost:${port}/session/${currentSession}/actions`,
|
|
1466
|
-
{
|
|
1467
|
-
actions: [{
|
|
1468
|
-
type: 'pointer',
|
|
1469
|
-
id: 'finger1',
|
|
1470
|
-
parameters: { pointerType: 'touch' },
|
|
1471
|
-
actions: [
|
|
1472
|
-
{ type: 'pointerMove', duration: 0, x: safeX, y: safeY },
|
|
1473
|
-
{ type: 'pointerDown', button: 0 },
|
|
1474
|
-
{ type: 'pause', duration: 100 },
|
|
1475
|
-
{ type: 'pointerUp', button: 0 }
|
|
1476
|
-
]
|
|
1477
|
-
}]
|
|
1478
|
-
},
|
|
1479
|
-
{ timeout: 15000 }
|
|
1480
|
-
);
|
|
1481
|
-
this.logger.info(`✅ Tap executed on commander: ${this.commanderDevice.name}`);
|
|
1482
|
-
|
|
1483
|
-
// 🚀 SIMPLE APP LAUNCH DETECTION: Check if app changed after tap
|
|
1484
|
-
setTimeout(async () => {
|
|
1485
|
-
// Only check for app launch if we were originally on home screen
|
|
1486
|
-
if (!isOnHomeScreen) {
|
|
1487
|
-
this.logger.debug('📱 Skipping app detection - was not on home screen');
|
|
1488
|
-
return;
|
|
1489
|
-
}
|
|
1490
|
-
|
|
1491
|
-
try {
|
|
1492
|
-
const currentAppRes = await axios.get(`http://localhost:${port}/session/${currentSession}/wda/activeAppInfo`, { timeout: 3000 });
|
|
1493
|
-
const currentAppId = currentAppRes.data?.value?.bundleId;
|
|
1494
|
-
|
|
1495
|
-
if (currentAppId && currentAppId !== 'com.apple.springboard') {
|
|
1496
|
-
this.logger.info(`🚀 SIMPLE: App launched on commander: ${currentAppId} - launching on followers...`);
|
|
1497
|
-
|
|
1498
|
-
// Launch same app on all followers immediately
|
|
1499
|
-
for (const follower of this.followerDevices) {
|
|
1500
|
-
if (follower.platform === 'ios') {
|
|
1501
|
-
try {
|
|
1502
|
-
const followerPort = follower.port || 8100;
|
|
1503
|
-
|
|
1504
|
-
// Get fresh session ID using universal session manager
|
|
1505
|
-
let followerSessionId = null;
|
|
1506
|
-
try {
|
|
1507
|
-
followerSessionId = await this.sessionManager.getSessionWithRetry(follower.name, followerPort);
|
|
1508
|
-
} catch (sessionError) {
|
|
1509
|
-
this.logger.warn(`⚠️ Could not get session for follower ${follower.name}: ${sessionError.message}`);
|
|
1510
|
-
continue; // Skip this follower
|
|
1511
|
-
}
|
|
1512
|
-
|
|
1513
|
-
if (followerSessionId) {
|
|
1514
|
-
this.logger.info(`🚀 Auto-launching ${currentAppId} on follower: ${follower.name}`);
|
|
1515
|
-
await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/wda/apps/launch`, {
|
|
1516
|
-
bundleId: currentAppId
|
|
1517
|
-
}, { timeout: 15000 });
|
|
1518
|
-
this.logger.info(`✅ App ${currentAppId} launched on ${follower.name}`);
|
|
1519
|
-
} else {
|
|
1520
|
-
this.logger.warn(`⚠️ No valid session found for ${follower.name}`);
|
|
1521
|
-
}
|
|
1522
|
-
} catch (launchError) {
|
|
1523
|
-
this.logger.warn(`⚠️ Failed to launch ${currentAppId} on ${follower.name}: ${launchError.message}`);
|
|
1524
|
-
}
|
|
1525
|
-
}
|
|
1526
|
-
}
|
|
1527
|
-
|
|
1528
|
-
// 🚀 REFRESH CACHE AFTER APP LAUNCH: Now that app is launched, refresh locators for new app
|
|
1529
|
-
setTimeout(() => {
|
|
1530
|
-
this.logger.info('🔄 Refreshing locators after successful app launch...');
|
|
1531
|
-
this.intelligentLocator.invalidateCache(this.commanderDevice.name);
|
|
1532
|
-
// Refresh in background for next interactions in the new app
|
|
1533
|
-
this.intelligentLocator.getCachedLocators(this.commanderDevice.name, true).catch(() => {});
|
|
1534
|
-
for (const follower of this.followerDevices) {
|
|
1535
|
-
this.intelligentLocator.getCachedLocators(follower.name, true).catch(() => {});
|
|
1536
|
-
}
|
|
1537
|
-
}, 2000); // Wait 2s for app to fully load
|
|
1538
|
-
|
|
1539
|
-
}
|
|
1540
|
-
} catch (appCheckError) {
|
|
1541
|
-
// Ignore errors in app detection - not critical
|
|
1542
|
-
this.logger.debug(`📱 App check failed: ${appCheckError.message}`);
|
|
1543
|
-
}
|
|
1544
|
-
}, 1000); // Wait 1 second for app to launch (reduced from 2s)
|
|
1545
|
-
|
|
1546
|
-
// 🎯 OPTIMIZED CACHE INVALIDATION: Less aggressive, smarter cache management
|
|
1547
|
-
if (!isOnHomeScreen) {
|
|
1548
|
-
this.logger.info('🔄 Invalidating locator cache after successful tap to ensure fresh screen state');
|
|
1549
|
-
this.intelligentLocator.invalidateCache(this.commanderDevice.name);
|
|
1550
|
-
|
|
1551
|
-
// PERFORMANCE: Make background refresh safer with better error handling
|
|
1552
|
-
setImmediate(() => {
|
|
1553
|
-
// Safer async background refresh
|
|
1554
|
-
(async () => {
|
|
1555
|
-
try {
|
|
1556
|
-
this.logger.debug('🔄 Background refresh of locators after regular tap...');
|
|
1557
|
-
|
|
1558
|
-
// Refresh commander cache with timeout protection
|
|
1559
|
-
const refreshCommander = this.intelligentLocator.getCachedLocators(this.commanderDevice.name, true)
|
|
1560
|
-
.catch(err => {
|
|
1561
|
-
this.logger.debug(`Commander refresh failed: ${err.message}`);
|
|
1562
|
-
return null;
|
|
1563
|
-
});
|
|
1564
|
-
|
|
1565
|
-
await refreshCommander;
|
|
1566
|
-
|
|
1567
|
-
// Also refresh follower caches in background with individual error handling
|
|
1568
|
-
if (this.followerDevices && this.followerDevices.size > 0) {
|
|
1569
|
-
const refreshPromises = Array.from(this.followerDevices).map(async (follower) => {
|
|
1570
|
-
try {
|
|
1571
|
-
await this.intelligentLocator.getCachedLocators(follower.name, true);
|
|
1572
|
-
} catch (err) {
|
|
1573
|
-
this.logger.debug(`Background refresh failed for ${follower.name}: ${err.message}`);
|
|
1574
|
-
}
|
|
1575
|
-
});
|
|
1576
|
-
|
|
1577
|
-
await Promise.allSettled(refreshPromises);
|
|
1578
|
-
}
|
|
1579
|
-
|
|
1580
|
-
this.logger.debug('✅ Background locator refresh completed');
|
|
1581
|
-
} catch (refreshErr) {
|
|
1582
|
-
this.logger.debug(`Background refresh error: ${refreshErr.message}`);
|
|
1583
|
-
}
|
|
1584
|
-
})().catch(err => {
|
|
1585
|
-
// Final safety net to prevent unhandled promise rejections
|
|
1586
|
-
this.logger.debug(`Background refresh async error: ${err.message}`);
|
|
1587
|
-
});
|
|
1588
|
-
});
|
|
1589
|
-
} else {
|
|
1590
|
-
this.logger.info('🚀 Skipping cache refresh - app launch detected, will refresh after app stabilizes');
|
|
1591
|
-
}
|
|
1592
|
-
} catch (err) {
|
|
1593
|
-
// GENIUS: Smarter session handling - avoid recreating sessions that just need a status sync
|
|
1594
|
-
if (err.response && err.response.status === 404 && !retryAttempted) {
|
|
1595
|
-
this.logger.warn(`⚠️ Session might be expired (404), attempting smart recovery...`);
|
|
1596
|
-
|
|
1597
|
-
// First, try to get a valid session from WDA status
|
|
1598
|
-
let sessionRecovered = false;
|
|
1599
|
-
try {
|
|
1600
|
-
const port = this.commanderDevice.port || 8100;
|
|
1601
|
-
const statusRes = await axios.get(`http://localhost:${port}/status`, { timeout: 5000 });
|
|
1602
|
-
const activeSession = statusRes.data?.sessionId;
|
|
1603
|
-
|
|
1604
|
-
if (activeSession && activeSession !== currentSession) {
|
|
1605
|
-
// WDA has an active session, just update our reference
|
|
1606
|
-
this.logger.info(`🔄 Found active WDA session: ${activeSession.substring(0, 8)}...`);
|
|
1607
|
-
this.sessionManager.updateSession(this.commanderDevice.name, activeSession);
|
|
1608
|
-
currentSession = activeSession;
|
|
1609
|
-
sessionRecovered = true;
|
|
1610
|
-
this.logger.info(`✅ Session recovered without recreation`);
|
|
1611
|
-
}
|
|
1612
|
-
} catch (statusError) {
|
|
1613
|
-
this.logger.debug(`Status check failed: ${statusError.message}`);
|
|
1614
|
-
}
|
|
1615
|
-
|
|
1616
|
-
// Only recreate if we couldn't recover an existing session
|
|
1617
|
-
if (!sessionRecovered) {
|
|
1618
|
-
// Before recreating, try one more recovery approach - check if device is still responsive
|
|
1619
|
-
try {
|
|
1620
|
-
const port = this.commanderDevice.port || 8100;
|
|
1621
|
-
const healthRes = await axios.get(`http://localhost:${port}/wda/healthcheck`, { timeout: 2000 });
|
|
1622
|
-
if (healthRes.status === 200) {
|
|
1623
|
-
// WDA is healthy, just create a new session without full recreation
|
|
1624
|
-
this.logger.info(`🔄 WDA is healthy, creating new session without full restart...`);
|
|
1625
|
-
const newSessionRes = await axios.post(`http://localhost:${port}/session`, {
|
|
1626
|
-
capabilities: {
|
|
1627
|
-
alwaysMatch: {},
|
|
1628
|
-
firstMatch: [{}]
|
|
1629
|
-
}
|
|
1630
|
-
}, { timeout: 10000 });
|
|
1631
|
-
|
|
1632
|
-
const newSessionId = newSessionRes.data?.sessionId || newSessionRes.data?.value?.sessionId;
|
|
1633
|
-
if (newSessionId) {
|
|
1634
|
-
this.logger.info(`✅ New session created: ${newSessionId.substring(0, 8)}...`);
|
|
1635
|
-
this.commanderDevice.sessionId = newSessionId;
|
|
1636
|
-
if (global.sessionManager) {
|
|
1637
|
-
global.sessionManager.updateSession(this.commanderDevice.udid, newSessionId);
|
|
1638
|
-
}
|
|
1639
|
-
currentSession = newSessionId;
|
|
1640
|
-
sessionRecovered = true;
|
|
1641
|
-
}
|
|
1642
|
-
}
|
|
1643
|
-
} catch (healthError) {
|
|
1644
|
-
this.logger.debug(`Health check failed: ${healthError.message}`);
|
|
1645
|
-
}
|
|
1646
|
-
}
|
|
1647
|
-
|
|
1648
|
-
// If session recovery failed, use universal session manager as last resort
|
|
1649
|
-
if (!sessionRecovered) {
|
|
1650
|
-
this.logger.warn(`⚠️ No active session found, getting fresh session...`);
|
|
1651
|
-
try {
|
|
1652
|
-
currentSession = await this.sessionManager.getSessionWithRetry(device.name, port);
|
|
1653
|
-
sessionRecovered = true;
|
|
1654
|
-
} catch (recreateError) {
|
|
1655
|
-
this.logger.error(`❌ Session recovery failed: ${recreateError.message}`);
|
|
1656
|
-
throw new Error(`Commander tap failed: ${err.message}`);
|
|
1657
|
-
}
|
|
1658
|
-
}
|
|
1659
|
-
|
|
1660
|
-
// Retry the tap with recovered/new session
|
|
1661
|
-
if (sessionRecovered) {
|
|
1662
|
-
// Retry the tap with new session - use simple coordinate validation
|
|
1663
|
-
try {
|
|
1664
|
-
await axios.post(
|
|
1665
|
-
`http://localhost:${port}/session/${currentSession}/actions`,
|
|
1666
|
-
{
|
|
1667
|
-
actions: [{
|
|
1668
|
-
type: 'pointer',
|
|
1669
|
-
id: 'finger1',
|
|
1670
|
-
parameters: { pointerType: 'touch' },
|
|
1671
|
-
actions: [
|
|
1672
|
-
{ type: 'pointerMove', duration: 0, x: safeX, y: safeY },
|
|
1673
|
-
{ type: 'pointerDown', button: 0 },
|
|
1674
|
-
{ type: 'pause', duration: 100 },
|
|
1675
|
-
{ type: 'pointerUp', button: 0 }
|
|
1676
|
-
]
|
|
1677
|
-
}]
|
|
1678
|
-
},
|
|
1679
|
-
{ timeout: 15000 }
|
|
1680
|
-
);
|
|
1681
|
-
this.logger.info(`✅ Tap executed on commander: ${this.commanderDevice.name} (after session recovery)`);
|
|
1682
|
-
|
|
1683
|
-
// 🎯 CACHE INVALIDATION: Force fresh locators after successful recovery tap
|
|
1684
|
-
this.logger.info('🔄 Invalidating locator cache after successful recovery tap');
|
|
1685
|
-
this.intelligentLocator.invalidateCache(this.commanderDevice.name);
|
|
1686
|
-
|
|
1687
|
-
// 🎯 OPTIMIZED CACHE REFRESH: Schedule refresh but don't block execution
|
|
1688
|
-
setTimeout(async () => {
|
|
1689
|
-
try {
|
|
1690
|
-
this.logger.info('🔄 Background refresh after recovery tap...');
|
|
1691
|
-
await this.intelligentLocator.getCachedLocators(this.commanderDevice.name, true);
|
|
1692
|
-
|
|
1693
|
-
// Background refresh for followers
|
|
1694
|
-
// OPTIMIZATION: Skip duplicate follower refresh - already handled in intelligent actions
|
|
1695
|
-
this.logger.info('ℹ️ Follower caches managed in intelligent actions - skipping duplicate refresh');
|
|
1696
|
-
|
|
1697
|
-
this.logger.info('✅ Background refresh completed (recovery)');
|
|
1698
|
-
} catch (refreshErr) {
|
|
1699
|
-
// Ignore background refresh errors
|
|
1700
|
-
}
|
|
1701
|
-
}, 500);
|
|
1702
|
-
} catch (retryErr) {
|
|
1703
|
-
this.logger.error(`❌ Commander tap failed even after session recovery: ${retryErr.message}`);
|
|
1704
|
-
throw new Error(`Commander tap failed: ${retryErr.message}`);
|
|
1705
|
-
}
|
|
1706
|
-
} else {
|
|
1707
|
-
this.logger.error(`❌ Could not recover WDA session for commander`);
|
|
1708
|
-
throw new Error(`Commander tap failed: ${err.message}`);
|
|
1709
|
-
}
|
|
1710
|
-
} else {
|
|
1711
|
-
this.logger.error(`❌ Commander tap failed: ${err.message}`);
|
|
1712
|
-
throw new Error(`Commander tap failed: ${err.message}`);
|
|
1713
|
-
}
|
|
1714
|
-
}
|
|
1715
|
-
|
|
1716
|
-
// ===============================
|
|
1717
|
-
// 🎯 STEP 4: BROADCAST INTELLIGENT LOCATORS TO FOLLOWERS (NEW GENIUS METHOD)
|
|
1718
|
-
// ===============================
|
|
1719
|
-
const followerDevices = Array.from(this.followerDevices);
|
|
1720
|
-
if (followerDevices.length > 0) {
|
|
1721
|
-
|
|
1722
|
-
if (isOnHomeScreen) {
|
|
1723
|
-
this.logger.info('🏠 SIMPLE: On home screen - skipping follower broadcast, will detect app launch instead...');
|
|
1724
|
-
// Skip immediate coordinate broadcast, let app detection handle it
|
|
1725
|
-
} else {
|
|
1726
|
-
this.logger.info('');
|
|
1727
|
-
this.logger.info(`📡 Broadcasting INTELLIGENT locators to ${followerDevices.length} follower(s):`);
|
|
1728
|
-
for (const follower of followerDevices) {
|
|
1729
|
-
this.logger.info(` • ${follower.name} (${follower.platform})`);
|
|
1730
|
-
this.executeIntelligentActionOnFollower(follower, intelligentLocator, logicalX, logicalY)
|
|
1731
|
-
.catch(e => this.logger.error(`Follower ${follower.name} intelligent action failed: ${e.message}`));
|
|
1732
|
-
}
|
|
1733
|
-
}
|
|
1734
|
-
} else {
|
|
1735
|
-
this.logger.info('ℹ️ No followers connected');
|
|
1736
|
-
}
|
|
1737
|
-
|
|
1738
|
-
// Log performance metrics
|
|
1739
|
-
const metrics = this.intelligentLocator.getPerformanceMetrics();
|
|
1740
|
-
this.logger.info(`📊 Intelligent Locator Metrics: ${metrics.accuracyRate} accuracy, ${metrics.cacheHitRate} cache hit rate`);
|
|
1741
|
-
|
|
1742
|
-
this.logger.info('═══════════════════════════════════════════════════════');
|
|
1743
|
-
this.logger.info('');
|
|
1744
|
-
return {
|
|
1745
|
-
success: true,
|
|
1746
|
-
devicesCount: 1 + followerDevices.length,
|
|
1747
|
-
intelligentLocator: intelligentLocator,
|
|
1748
|
-
method: intelligentLocator.method,
|
|
1749
|
-
metrics: metrics
|
|
1750
|
-
};
|
|
1751
|
-
} catch (error) {
|
|
1752
|
-
this.logger.error('Screen click handling error:', error.message);
|
|
1753
|
-
throw error;
|
|
1754
|
-
}
|
|
1755
|
-
}
|
|
1756
|
-
|
|
1757
|
-
// Faster caching method that avoids heavy API calls
|
|
1758
|
-
async fastCacheAllElements() {
|
|
1759
|
-
const wdaSessionId = await this.getWDASessionId();
|
|
1760
|
-
if (!wdaSessionId || !this.commanderDevice) return;
|
|
1761
|
-
if (this.isCaching) return;
|
|
1762
|
-
|
|
1763
|
-
this.isCaching = true;
|
|
1764
|
-
const startTime = Date.now();
|
|
1765
|
-
|
|
1766
|
-
try {
|
|
1767
|
-
const device = this.commanderDevice;
|
|
1768
|
-
const port = device.port || 8100;
|
|
1769
|
-
const sessionId = wdaSessionId;
|
|
1770
|
-
|
|
1771
|
-
this.logger.info('🔍 Starting fast locator detection...');
|
|
1772
|
-
|
|
1773
|
-
// Try page source first (fastest method)
|
|
1774
|
-
try {
|
|
1775
|
-
const sourceRes = await axios.get(
|
|
1776
|
-
`http://localhost:${port}/session/${sessionId}/source`,
|
|
1777
|
-
{ timeout: 3000 } // Reduced timeout for speed
|
|
1778
|
-
);
|
|
1779
|
-
|
|
1780
|
-
if (sourceRes.data?.value) {
|
|
1781
|
-
const pageSource = sourceRes.data.value;
|
|
1782
|
-
const allElements = this.parsePageSourceFast(pageSource);
|
|
1783
|
-
|
|
1784
|
-
if (allElements.length > 0) {
|
|
1785
|
-
this.cachedElements = allElements;
|
|
1786
|
-
this.lastCacheTime = Date.now();
|
|
1787
|
-
|
|
1788
|
-
const elapsed = Date.now() - startTime;
|
|
1789
|
-
this.logger.info(`✅ Fast cached ${allElements.length} elements in ${elapsed}ms`);
|
|
1790
|
-
return;
|
|
1791
|
-
}
|
|
1792
|
-
}
|
|
1793
|
-
} catch (pageSourceErr) {
|
|
1794
|
-
this.logger.warn(`Page source method failed, falling back to element query`);
|
|
1795
|
-
|
|
1796
|
-
// Fallback: Quick element query for essential elements only - SPEED OPTIMIZED
|
|
1797
|
-
const elementTypes = ['XCUIElementTypeButton']; // Only buttons for speed
|
|
1798
|
-
const allElements = [];
|
|
1799
|
-
|
|
1800
|
-
for (const elementType of elementTypes) {
|
|
1801
|
-
try {
|
|
1802
|
-
const query = { using: 'class name', value: elementType };
|
|
1803
|
-
const elementsRes = await axios.post(
|
|
1804
|
-
`http://localhost:${port}/session/${sessionId}/elements`,
|
|
1805
|
-
query,
|
|
1806
|
-
{ timeout: 2000 } // Quick timeout
|
|
1807
|
-
);
|
|
1808
|
-
|
|
1809
|
-
if (elementsRes.data?.value && Array.isArray(elementsRes.data.value)) {
|
|
1810
|
-
const elements = elementsRes.data.value.slice(0, 10); // Further reduced for speed
|
|
1811
|
-
|
|
1812
|
-
for (const elem of elements) {
|
|
1813
|
-
const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
|
|
1814
|
-
if (!elementId) continue;
|
|
1815
|
-
|
|
1816
|
-
try {
|
|
1817
|
-
const rectRes = await axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/rect`, { timeout: 500 });
|
|
1818
|
-
|
|
1819
|
-
if (rectRes.data?.value) {
|
|
1820
|
-
const rect = rectRes.data.value;
|
|
1821
|
-
if (rect.width > 5 && rect.height > 5) { // Skip tiny elements
|
|
1822
|
-
allElements.push({
|
|
1823
|
-
type: elementType,
|
|
1824
|
-
label: 'Unlabeled', // Skip label for speed
|
|
1825
|
-
name: 'Unlabeled',
|
|
1826
|
-
accessibilityId: 'Unlabeled',
|
|
1827
|
-
x: Math.round(rect.x),
|
|
1828
|
-
y: Math.round(rect.y),
|
|
1829
|
-
width: Math.round(rect.width),
|
|
1830
|
-
height: Math.round(rect.height),
|
|
1831
|
-
centerX: Math.round(rect.x + rect.width / 2),
|
|
1832
|
-
centerY: Math.round(rect.y + rect.height / 2)
|
|
1833
|
-
});
|
|
1834
|
-
}
|
|
1835
|
-
}
|
|
1836
|
-
} catch (err) {
|
|
1837
|
-
// Skip this element, continue with others
|
|
1838
|
-
continue;
|
|
1839
|
-
}
|
|
1840
|
-
}
|
|
1841
|
-
}
|
|
1842
|
-
} catch (queryErr) {
|
|
1843
|
-
this.logger.warn(`Query ${elementType} failed: ${queryErr.message}`);
|
|
1844
|
-
continue;
|
|
1845
|
-
}
|
|
1846
|
-
}
|
|
1847
|
-
|
|
1848
|
-
this.cachedElements = allElements;
|
|
1849
|
-
this.lastCacheTime = Date.now();
|
|
1850
|
-
|
|
1851
|
-
const elapsed = Date.now() - startTime;
|
|
1852
|
-
this.logger.info(`✅ Cached ${allElements.length} elements in ${elapsed}ms`);
|
|
1853
|
-
}
|
|
1854
|
-
} catch (error) {
|
|
1855
|
-
this.logger.error(`Fast cache failed: ${error.message}`);
|
|
1856
|
-
} finally {
|
|
1857
|
-
this.isCaching = false;
|
|
1858
|
-
}
|
|
1859
|
-
}
|
|
1860
|
-
|
|
1861
|
-
async cacheAllElements() {
|
|
1862
|
-
const wdaSessionId = await this.getWDASessionId();
|
|
1863
|
-
if (!wdaSessionId || !this.commanderDevice) return;
|
|
1864
|
-
if (this.isCaching) return;
|
|
1865
|
-
|
|
1866
|
-
this.isCaching = true;
|
|
1867
|
-
const startTime = Date.now();
|
|
1868
|
-
|
|
1869
|
-
try {
|
|
1870
|
-
const axios = require('axios');
|
|
1871
|
-
const device = this.commanderDevice;
|
|
1872
|
-
const port = device.port || 8100;
|
|
1873
|
-
const sessionId = wdaSessionId;
|
|
1874
|
-
|
|
1875
|
-
this.logger.info('🔍 Starting fast locator detection...');
|
|
1876
|
-
|
|
1877
|
-
// Use page source for MUCH faster element detection
|
|
1878
|
-
try {
|
|
1879
|
-
const sourceRes = await axios.get(
|
|
1880
|
-
`http://localhost:${port}/session/${sessionId}/source`,
|
|
1881
|
-
{ timeout: 8000 }
|
|
1882
|
-
);
|
|
1883
|
-
|
|
1884
|
-
if (sourceRes.data?.value) {
|
|
1885
|
-
const pageSource = sourceRes.data.value;
|
|
1886
|
-
const allElements = this.parsePageSourceFast(pageSource);
|
|
1887
|
-
|
|
1888
|
-
if (allElements.length > 0) {
|
|
1889
|
-
this.cachedElements = allElements;
|
|
1890
|
-
this.lastCacheTime = Date.now();
|
|
1891
|
-
|
|
1892
|
-
const elapsed = Date.now() - startTime;
|
|
1893
|
-
this.logger.info(`✅ Fast cached ${allElements.length} elements in ${elapsed}ms`);
|
|
1894
|
-
|
|
1895
|
-
// Send all at once from page source (fastest method)
|
|
1896
|
-
this.broadcast({
|
|
1897
|
-
type: 'cached_elements',
|
|
1898
|
-
elements: allElements.map(e => ({
|
|
1899
|
-
label: e.label || e.name,
|
|
1900
|
-
type: e.type.replace('XCUIElementType', ''),
|
|
1901
|
-
x: e.x,
|
|
1902
|
-
y: e.y,
|
|
1903
|
-
width: e.width,
|
|
1904
|
-
height: e.height
|
|
1905
|
-
})),
|
|
1906
|
-
total: allElements.length,
|
|
1907
|
-
timestamp: Date.now()
|
|
1908
|
-
});
|
|
1909
|
-
|
|
1910
|
-
// Broadcast completion
|
|
1911
|
-
this.broadcast({
|
|
1912
|
-
type: 'locators_loaded',
|
|
1913
|
-
total: allElements.length
|
|
1914
|
-
});
|
|
1915
|
-
|
|
1916
|
-
this.isCaching = false;
|
|
1917
|
-
return;
|
|
1918
|
-
}
|
|
1919
|
-
}
|
|
1920
|
-
} catch (err) {
|
|
1921
|
-
this.logger.warn('Page source method failed, falling back to element query');
|
|
1922
|
-
}
|
|
1923
|
-
|
|
1924
|
-
// Optimized queries - focus on most important element types only
|
|
1925
|
-
const queries = [
|
|
1926
|
-
{ using: 'class name', value: 'XCUIElementTypeButton' },
|
|
1927
|
-
{ using: 'class name', value: 'XCUIElementTypeStaticText' }
|
|
1928
|
-
];
|
|
1929
|
-
|
|
1930
|
-
const allElements = [];
|
|
1931
|
-
const MAX_ELEMENTS = 20; // Further reduced for speed
|
|
1932
|
-
|
|
1933
|
-
// Process queries sequentially with timeout protection
|
|
1934
|
-
for (const query of queries) {
|
|
1935
|
-
if (allElements.length >= MAX_ELEMENTS) break;
|
|
1936
|
-
|
|
1937
|
-
try {
|
|
1938
|
-
const elementsRes = await axios.post(
|
|
1939
|
-
`http://localhost:${port}/session/${sessionId}/elements`,
|
|
1940
|
-
query,
|
|
1941
|
-
{ timeout: 3000 } // Reduced timeout
|
|
1942
|
-
);
|
|
1943
|
-
|
|
1944
|
-
if (elementsRes.data?.value && Array.isArray(elementsRes.data.value)) {
|
|
1945
|
-
const elements = elementsRes.data.value.slice(0, MAX_ELEMENTS - allElements.length);
|
|
1946
|
-
|
|
1947
|
-
// Process elements in smaller batches to reduce session load
|
|
1948
|
-
const chunkSize = 5; // Smaller chunks for better performance
|
|
1949
|
-
for (let i = 0; i < elements.length && allElements.length < MAX_ELEMENTS; i += chunkSize) {
|
|
1950
|
-
const chunk = elements.slice(i, i + chunkSize);
|
|
1951
|
-
const chunkPromises = chunk.map(async (elem) => {
|
|
1952
|
-
const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
|
|
1953
|
-
if (!elementId) return null;
|
|
1954
|
-
|
|
1955
|
-
try {
|
|
1956
|
-
// Only get rect - skip label to reduce API calls
|
|
1957
|
-
const rectRes = await axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/rect`, { timeout: 800 });
|
|
1958
|
-
|
|
1959
|
-
if (rectRes.data?.value) {
|
|
1960
|
-
const rect = rectRes.data.value;
|
|
1961
|
-
|
|
1962
|
-
// Skip very small elements that are not useful
|
|
1963
|
-
if (rect.width < 10 || rect.height < 10) return null;
|
|
1964
|
-
|
|
1965
|
-
// Get label only for buttons (most important)
|
|
1966
|
-
let label = 'Unlabeled';
|
|
1967
|
-
if (query.value === 'XCUIElementTypeButton') {
|
|
1968
|
-
try {
|
|
1969
|
-
const labelRes = await axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/attribute/label`, { timeout: 500 });
|
|
1970
|
-
label = labelRes.data?.value || 'Unlabeled';
|
|
1971
|
-
} catch (labelErr) {
|
|
1972
|
-
// Skip this element if we can't get its label for buttons
|
|
1973
|
-
return null;
|
|
1974
|
-
}
|
|
1975
|
-
}
|
|
1976
|
-
|
|
1977
|
-
return {
|
|
1978
|
-
type: query.value,
|
|
1979
|
-
label: label,
|
|
1980
|
-
name: label || 'Unlabeled',
|
|
1981
|
-
accessibilityId: label,
|
|
1982
|
-
x: Math.round(rect.x),
|
|
1983
|
-
y: Math.round(rect.y),
|
|
1984
|
-
width: Math.round(rect.width),
|
|
1985
|
-
height: Math.round(rect.height),
|
|
1986
|
-
centerX: Math.round(rect.x + rect.width / 2),
|
|
1987
|
-
centerY: Math.round(rect.y + rect.height / 2)
|
|
1988
|
-
};
|
|
1989
|
-
}
|
|
1990
|
-
} catch (err) { return null; }
|
|
1991
|
-
return null;
|
|
1992
|
-
});
|
|
1993
|
-
|
|
1994
|
-
const chunkResults = await Promise.all(chunkPromises);
|
|
1995
|
-
const validElements = chunkResults.filter(e => e !== null);
|
|
1996
|
-
|
|
1997
|
-
if (validElements.length > 0) {
|
|
1998
|
-
typeElements.push(...validElements);
|
|
1999
|
-
allElements.push(...validElements);
|
|
2000
|
-
|
|
2001
|
-
// Broadcast incrementally for instant UI feedback
|
|
2002
|
-
this.broadcast({
|
|
2003
|
-
type: 'cached_elements',
|
|
2004
|
-
elements: validElements.map(e => ({
|
|
2005
|
-
label: e.label || e.name,
|
|
2006
|
-
type: e.type.replace('XCUIElementType', ''),
|
|
2007
|
-
x: e.x,
|
|
2008
|
-
y: e.y,
|
|
2009
|
-
width: e.width,
|
|
2010
|
-
height: e.height
|
|
2011
|
-
})),
|
|
2012
|
-
total: allElements.length,
|
|
2013
|
-
timestamp: Date.now()
|
|
2014
|
-
});
|
|
2015
|
-
}
|
|
2016
|
-
}
|
|
2017
|
-
}
|
|
2018
|
-
} catch (err) {
|
|
2019
|
-
this.logger.warn(`Query ${query.value} failed: ${err.message}`);
|
|
2020
|
-
}
|
|
2021
|
-
}
|
|
2022
|
-
|
|
2023
|
-
this.cachedElements = allElements;
|
|
2024
|
-
this.lastCacheTime = Date.now();
|
|
2025
|
-
|
|
2026
|
-
const elapsed = Date.now() - startTime;
|
|
2027
|
-
this.logger.info(`✅ Cached ${allElements.length} elements in ${elapsed}ms`);
|
|
2028
|
-
|
|
2029
|
-
// Broadcast completion
|
|
2030
|
-
this.broadcast({
|
|
2031
|
-
type: 'locators_loaded',
|
|
2032
|
-
total: allElements.length
|
|
2033
|
-
});
|
|
2034
|
-
|
|
2035
|
-
} finally {
|
|
2036
|
-
this.isCaching = false;
|
|
2037
|
-
}
|
|
2038
|
-
}
|
|
2039
|
-
|
|
2040
|
-
// GENIUS: Enhanced session recovery with minimal disruption
|
|
2041
|
-
async recoverOrCreateSession(device, originalSessionId = null) {
|
|
2042
|
-
const port = device.port || 8100;
|
|
2043
|
-
|
|
2044
|
-
this.logger.info(`🔍 Smart session recovery for ${device.name}...`);
|
|
2045
|
-
|
|
2046
|
-
// STEP 1: Check if WDA server itself is responsive
|
|
2047
|
-
try {
|
|
2048
|
-
const statusRes = await axios.get(`http://localhost:${port}/status`, { timeout: 2000 });
|
|
2049
|
-
const serverSessionId = statusRes.data?.sessionId;
|
|
2050
|
-
|
|
2051
|
-
if (serverSessionId && serverSessionId !== originalSessionId) {
|
|
2052
|
-
// WDA has a VALID session that's different from ours - use it!
|
|
2053
|
-
this.logger.info(`✅ Found active WDA session: ${serverSessionId.substring(0, 8)}...`);
|
|
2054
|
-
|
|
2055
|
-
// Test the session works
|
|
2056
|
-
try {
|
|
2057
|
-
await axios.get(`http://localhost:${port}/session/${serverSessionId}/window/size`, { timeout: 2000 });
|
|
2058
|
-
|
|
2059
|
-
// Update our references
|
|
2060
|
-
device.sessionId = serverSessionId;
|
|
2061
|
-
if (global.sessionManager) {
|
|
2062
|
-
global.sessionManager.updateSession(device.udid, serverSessionId);
|
|
2063
|
-
}
|
|
2064
|
-
|
|
2065
|
-
this.logger.info(`✅ Session recovered without recreation`);
|
|
2066
|
-
return serverSessionId;
|
|
2067
|
-
} catch (testErr) {
|
|
2068
|
-
this.logger.debug(`Found session ${serverSessionId} doesn't work: ${testErr.message}`);
|
|
2069
|
-
}
|
|
2070
|
-
}
|
|
2071
|
-
} catch (statusErr) {
|
|
2072
|
-
this.logger.debug(`WDA status check failed: ${statusErr.message}`);
|
|
2073
|
-
}
|
|
2074
|
-
|
|
2075
|
-
// STEP 2: Try creating a NEW session without killing existing ones
|
|
2076
|
-
try {
|
|
2077
|
-
this.logger.info(`🔄 Creating fresh WDA session (keeping WDA running)...`);
|
|
2078
|
-
const sessionRes = await axios.post(`http://localhost:${port}/session`, {
|
|
2079
|
-
capabilities: {
|
|
2080
|
-
alwaysMatch: {},
|
|
2081
|
-
firstMatch: [{}]
|
|
2082
|
-
}
|
|
2083
|
-
}, { timeout: 15000 });
|
|
2084
|
-
|
|
2085
|
-
const newSessionId = sessionRes.data?.sessionId || sessionRes.data?.value?.sessionId;
|
|
2086
|
-
if (newSessionId) {
|
|
2087
|
-
this.logger.info(`✅ Fresh session created: ${newSessionId.substring(0, 8)}...`);
|
|
2088
|
-
|
|
2089
|
-
// Update references
|
|
2090
|
-
device.sessionId = newSessionId;
|
|
2091
|
-
if (global.sessionManager) {
|
|
2092
|
-
global.sessionManager.updateSession(device.udid, newSessionId);
|
|
2093
|
-
}
|
|
2094
|
-
|
|
2095
|
-
return newSessionId;
|
|
2096
|
-
}
|
|
2097
|
-
} catch (createErr) {
|
|
2098
|
-
this.logger.warn(`⚠️ Fresh session creation failed: ${createErr.message}`);
|
|
2099
|
-
}
|
|
2100
|
-
|
|
2101
|
-
// STEP 3: Last resort - full WDA restart (only if nothing else worked)
|
|
2102
|
-
this.logger.warn(`🔄 All gentle approaches failed, doing full session recreation...`);
|
|
2103
|
-
await this.recreateWDASession();
|
|
2104
|
-
return this.getWDASessionId();
|
|
2105
|
-
}
|
|
2106
|
-
|
|
2107
|
-
async recreateWDASession() {
|
|
2108
|
-
if (!this.commanderDevice) {
|
|
2109
|
-
throw new Error('No commander device set');
|
|
2110
|
-
}
|
|
2111
|
-
|
|
2112
|
-
this.logger.info(`🔄 Recreating WDA session for ${this.commanderDevice.name}...`);
|
|
2113
|
-
|
|
2114
|
-
try {
|
|
2115
|
-
const device = this.commanderDevice;
|
|
2116
|
-
const port = device.port || 8100;
|
|
2117
|
-
|
|
2118
|
-
// Store current app before session recreation
|
|
2119
|
-
let currentAppId = null;
|
|
2120
|
-
try {
|
|
2121
|
-
const oldSessionId = this.getWDASessionId();
|
|
2122
|
-
if (oldSessionId) {
|
|
2123
|
-
const appRes = await axios.get(`http://localhost:${port}/session/${oldSessionId}/wda/activeAppInfo`, { timeout: 2000 });
|
|
2124
|
-
currentAppId = appRes.data?.value?.bundleId;
|
|
2125
|
-
this.logger.info(`📱 Current app before session recreation: ${currentAppId || 'unknown'}`);
|
|
2126
|
-
}
|
|
2127
|
-
} catch (appCheckError) {
|
|
2128
|
-
this.logger.warn(`⚠️ Could not detect current app: ${appCheckError.message}`);
|
|
2129
|
-
}
|
|
2130
|
-
|
|
2131
|
-
// Remove the old session from global manager
|
|
2132
|
-
if (global.sessionManager) {
|
|
2133
|
-
global.sessionManager.deleteSession(device.udid);
|
|
2134
|
-
this.logger.info('🗑️ Removed stale session from global manager');
|
|
2135
|
-
}
|
|
2136
|
-
|
|
2137
|
-
// Create a completely new WDA session
|
|
2138
|
-
await this.createWDASession();
|
|
2139
|
-
|
|
2140
|
-
// Verify the new session works
|
|
2141
|
-
const newSessionId = this.getWDASessionId();
|
|
2142
|
-
if (newSessionId) {
|
|
2143
|
-
// Test the session with a simple status call
|
|
2144
|
-
await axios.get(`http://localhost:${port}/session/${newSessionId}/window/size`, { timeout: 3000 });
|
|
2145
|
-
|
|
2146
|
-
// Smart app restoration - restore the ACTUAL app that was running
|
|
2147
|
-
if (currentAppId && currentAppId !== 'com.apple.springboard') {
|
|
2148
|
-
try {
|
|
2149
|
-
this.logger.info(`🔄 Restoring original app: ${currentAppId}`);
|
|
2150
|
-
await axios.post(`http://localhost:${port}/session/${newSessionId}/wda/apps/launch`, {
|
|
2151
|
-
bundleId: currentAppId
|
|
2152
|
-
}, { timeout: 10000 }); // Increased from 5s to 10s for app restoration
|
|
2153
|
-
|
|
2154
|
-
// Wait longer for app to fully stabilize after session recreation
|
|
2155
|
-
await new Promise(resolve => setTimeout(resolve, 4000)); // Increased from 2.5s to 4s
|
|
2156
|
-
this.logger.info(`✅ Original app restored: ${currentAppId}`);
|
|
2157
|
-
|
|
2158
|
-
// Verify app actually launched
|
|
2159
|
-
const verifyRes = await axios.get(`http://localhost:${port}/session/${newSessionId}/wda/activeAppInfo`, { timeout: 3000 });
|
|
2160
|
-
const actualApp = verifyRes.data?.value?.bundleId;
|
|
2161
|
-
if (actualApp === currentAppId) {
|
|
2162
|
-
this.logger.info(`✅ App restoration verified: ${actualApp}`);
|
|
2163
|
-
} else {
|
|
2164
|
-
this.logger.warn(`⚠️ App restoration failed: expected ${currentAppId}, got ${actualApp}`);
|
|
2165
|
-
}
|
|
2166
|
-
} catch (appRestoreError) {
|
|
2167
|
-
this.logger.warn(`⚠️ Could not restore original app ${currentAppId}: ${appRestoreError.message}`);
|
|
2168
|
-
// Don't try to restore anything else - leave in home screen
|
|
2169
|
-
}
|
|
2170
|
-
} else {
|
|
2171
|
-
this.logger.info(`ℹ️ No specific app to restore (was in: ${currentAppId || 'home screen'})`);
|
|
2172
|
-
}
|
|
2173
|
-
|
|
2174
|
-
this.logger.info(`✅ WDA session recreated successfully: ${newSessionId.substring(0, 8)}...`);
|
|
2175
|
-
return true;
|
|
2176
|
-
}
|
|
2177
|
-
} catch (error) {
|
|
2178
|
-
this.logger.error(`❌ Failed to recreate WDA session: ${error.message}`);
|
|
2179
|
-
throw error;
|
|
2180
|
-
}
|
|
2181
|
-
|
|
2182
|
-
return false;
|
|
2183
|
-
}
|
|
2184
|
-
|
|
2185
|
-
parsePageSourceFast(pageSource) {
|
|
2186
|
-
// Parse XML page source quickly - focus on truly interactive elements
|
|
2187
|
-
const elements = [];
|
|
2188
|
-
// Focus on most reliable interactive elements for speed and accuracy
|
|
2189
|
-
const xmlRegex = /<XCUIElementType(Button|TextField|Switch|Link|StaticText)[^>]*>/g;
|
|
2190
|
-
|
|
2191
|
-
let match;
|
|
2192
|
-
while ((match = xmlRegex.exec(pageSource)) !== null) {
|
|
2193
|
-
const elementTag = match[0];
|
|
2194
|
-
|
|
2195
|
-
// Extract attributes
|
|
2196
|
-
const xMatch = elementTag.match(/x="([^"]+)"/);
|
|
2197
|
-
const yMatch = elementTag.match(/y="([^"]+)"/);
|
|
2198
|
-
const widthMatch = elementTag.match(/width="([^"]+)"/);
|
|
2199
|
-
const heightMatch = elementTag.match(/height="([^"]+)"/);
|
|
2200
|
-
const labelMatch = elementTag.match(/label="([^"]+)"/);
|
|
2201
|
-
const nameMatch = elementTag.match(/name="([^"]+)"/);
|
|
2202
|
-
const enabledMatch = elementTag.match(/enabled="([^"]+)"/);
|
|
2203
|
-
|
|
2204
|
-
if (xMatch && yMatch && widthMatch && heightMatch) {
|
|
2205
|
-
const x = parseFloat(xMatch[1]);
|
|
2206
|
-
const y = parseFloat(yMatch[1]);
|
|
2207
|
-
const width = parseFloat(widthMatch[1]);
|
|
2208
|
-
const height = parseFloat(heightMatch[1]);
|
|
2209
|
-
const label = labelMatch?.[1] || nameMatch?.[1] || `${match[1]}_${Math.round(x)}_${Math.round(y)}`;
|
|
2210
|
-
const enabled = enabledMatch?.[1] !== 'false';
|
|
2211
|
-
const elementType = `XCUIElementType${match[1]}`;
|
|
2212
|
-
|
|
2213
|
-
// Fast filtering - keep it simple for speed
|
|
2214
|
-
if (width > 8 && height > 8 && enabled) {
|
|
2215
|
-
// Quick relevance check for speed
|
|
2216
|
-
if (elementType.includes('Button') || elementType.includes('TextField') ||
|
|
2217
|
-
elementType.includes('Switch') || elementType.includes('Link')) {
|
|
2218
|
-
// Always include truly interactive elements
|
|
2219
|
-
elements.push({
|
|
2220
|
-
type: elementType,
|
|
2221
|
-
label: label,
|
|
2222
|
-
name: label,
|
|
2223
|
-
accessibilityId: label,
|
|
2224
|
-
x: Math.round(x),
|
|
2225
|
-
y: Math.round(y),
|
|
2226
|
-
width: Math.round(width),
|
|
2227
|
-
height: Math.round(height),
|
|
2228
|
-
centerX: Math.round(x + width / 2),
|
|
2229
|
-
centerY: Math.round(y + height / 2)
|
|
2230
|
-
});
|
|
2231
|
-
} else if (elementType.includes('StaticText') && width > 30 && height > 15 && label && label.length > 2) {
|
|
2232
|
-
// Only include meaningful StaticText elements
|
|
2233
|
-
elements.push({
|
|
2234
|
-
type: elementType,
|
|
2235
|
-
label: label,
|
|
2236
|
-
name: label,
|
|
2237
|
-
accessibilityId: label,
|
|
2238
|
-
x: Math.round(x),
|
|
2239
|
-
y: Math.round(y),
|
|
2240
|
-
width: Math.round(width),
|
|
2241
|
-
height: Math.round(height),
|
|
2242
|
-
centerX: Math.round(x + width / 2),
|
|
2243
|
-
centerY: Math.round(y + height / 2)
|
|
2244
|
-
});
|
|
2245
|
-
}
|
|
2246
|
-
}
|
|
2247
|
-
}
|
|
2248
|
-
}
|
|
2249
|
-
|
|
2250
|
-
return elements;
|
|
2251
|
-
}
|
|
2252
|
-
|
|
2253
|
-
findClosestElement(x, y) {
|
|
2254
|
-
if (this.cachedElements.length === 0) return null;
|
|
2255
|
-
|
|
2256
|
-
// FAST STRATEGY: Find elements containing the click point
|
|
2257
|
-
const containingElements = this.cachedElements.filter(elem => {
|
|
2258
|
-
return x >= elem.x && x <= elem.x + elem.width &&
|
|
2259
|
-
y >= elem.y && y <= elem.y + elem.height;
|
|
2260
|
-
});
|
|
2261
|
-
|
|
2262
|
-
// If we found elements containing the point, return the smallest one (most specific)
|
|
2263
|
-
if (containingElements.length > 0) {
|
|
2264
|
-
// Sort by area (smallest first = most specific element)
|
|
2265
|
-
containingElements.sort((a, b) => (a.width * a.height) - (b.width * b.height));
|
|
2266
|
-
const bestElement = containingElements[0];
|
|
2267
|
-
|
|
2268
|
-
// Build strategies quickly - defer sibling detection
|
|
2269
|
-
bestElement.locatorStrategies = [];
|
|
2270
|
-
this.buildFastLocatorStrategies(bestElement);
|
|
2271
|
-
|
|
2272
|
-
return bestElement;
|
|
2273
|
-
}
|
|
2274
|
-
|
|
2275
|
-
// ENHANCED STRATEGY: Search for nearby elements with expanded tolerance
|
|
2276
|
-
const nearbyElements = this.cachedElements.filter(elem => {
|
|
2277
|
-
const dx = elem.centerX - x;
|
|
2278
|
-
const dy = elem.centerY - y;
|
|
2279
|
-
const distance = Math.sqrt(dx * dx + dy * dy);
|
|
2280
|
-
return distance <= 80; // Increased from 60px for better coverage
|
|
2281
|
-
});
|
|
2282
|
-
|
|
2283
|
-
if (nearbyElements.length > 0) {
|
|
2284
|
-
// Enhanced sorting with sibling text detection
|
|
2285
|
-
nearbyElements.forEach(elem => {
|
|
2286
|
-
const dx = elem.centerX - x;
|
|
2287
|
-
const dy = elem.centerY - y;
|
|
2288
|
-
elem.distance = Math.sqrt(dx * dx + dy * dy);
|
|
2289
|
-
|
|
2290
|
-
// Smart priority boosts
|
|
2291
|
-
if (elem.label && elem.label !== 'Unlabeled' && elem.label.length > 2) {
|
|
2292
|
-
elem.distance *= 0.7; // Strong preference for labeled elements
|
|
2293
|
-
}
|
|
2294
|
-
if (elem.type.includes('Button')) {
|
|
2295
|
-
elem.distance *= 0.8; // Prefer buttons
|
|
2296
|
-
}
|
|
2297
|
-
if (elem.type.includes('Cell')) {
|
|
2298
|
-
elem.distance *= 0.85; // Table cells are often clickable
|
|
2299
|
-
}
|
|
2300
|
-
|
|
2301
|
-
// GENIUS: Look for nearby text elements that could describe this element
|
|
2302
|
-
const nearbyText = this.cachedElements.find(textElem => {
|
|
2303
|
-
if (textElem.type === 'XCUIElementTypeStaticText' && textElem.label && textElem.label !== 'Unlabeled') {
|
|
2304
|
-
const textDistance = Math.sqrt(
|
|
2305
|
-
Math.pow(textElem.centerX - elem.centerX, 2) +
|
|
2306
|
-
Math.pow(textElem.centerY - elem.centerY, 2)
|
|
2307
|
-
);
|
|
2308
|
-
return textDistance < 60; // Text within 60px could be related
|
|
2309
|
-
}
|
|
2310
|
-
return false;
|
|
2311
|
-
});
|
|
2312
|
-
|
|
2313
|
-
if (nearbyText) {
|
|
2314
|
-
elem.associatedText = nearbyText.label;
|
|
2315
|
-
elem.distance *= 0.6; // Big boost for elements with associated text
|
|
2316
|
-
}
|
|
2317
|
-
});
|
|
2318
|
-
|
|
2319
|
-
nearbyElements.sort((a, b) => a.distance - b.distance);
|
|
2320
|
-
const bestNearby = nearbyElements[0];
|
|
2321
|
-
|
|
2322
|
-
// Build enhanced strategies with sibling info
|
|
2323
|
-
bestNearby.locatorStrategies = [];
|
|
2324
|
-
this.buildEnhancedLocatorStrategies(bestNearby);
|
|
2325
|
-
|
|
2326
|
-
return bestNearby;
|
|
2327
|
-
}
|
|
2328
|
-
|
|
2329
|
-
return null;
|
|
2330
|
-
}
|
|
2331
|
-
|
|
2332
|
-
// Enhanced strategy building with sibling detection for better accuracy
|
|
2333
|
-
buildEnhancedLocatorStrategies(element) {
|
|
2334
|
-
const strategies = [];
|
|
2335
|
-
|
|
2336
|
-
// Strategy 1: Direct element locators (highest priority)
|
|
2337
|
-
if (element.accessibilityId && element.accessibilityId !== 'Unlabeled') {
|
|
2338
|
-
strategies.push({
|
|
2339
|
-
strategy: 'accessibility id',
|
|
2340
|
-
value: element.accessibilityId,
|
|
2341
|
-
priority: 1
|
|
2342
|
-
});
|
|
2343
|
-
}
|
|
2344
|
-
|
|
2345
|
-
if (element.label && element.label !== 'Unlabeled' && element.label !== element.accessibilityId) {
|
|
2346
|
-
strategies.push({
|
|
2347
|
-
strategy: 'name',
|
|
2348
|
-
value: element.label,
|
|
2349
|
-
priority: 2
|
|
2350
|
-
});
|
|
2351
|
-
}
|
|
2352
|
-
|
|
2353
|
-
// Strategy 2: Use associated text if available (very good accuracy)
|
|
2354
|
-
if (element.associatedText) {
|
|
2355
|
-
strategies.push({
|
|
2356
|
-
strategy: 'name',
|
|
2357
|
-
value: element.associatedText,
|
|
2358
|
-
priority: 1.5 // Between accessibility id and direct label
|
|
2359
|
-
});
|
|
2360
|
-
|
|
2361
|
-
// Also try accessibility id with associated text
|
|
2362
|
-
strategies.push({
|
|
2363
|
-
strategy: 'accessibility id',
|
|
2364
|
-
value: element.associatedText,
|
|
2365
|
-
priority: 1.5
|
|
2366
|
-
});
|
|
2367
|
-
}
|
|
2368
|
-
|
|
2369
|
-
// Strategy 3: Class name (lower priority)
|
|
2370
|
-
if (element.type) {
|
|
2371
|
-
strategies.push({
|
|
2372
|
-
strategy: 'class name',
|
|
2373
|
-
value: element.type,
|
|
2374
|
-
priority: 5
|
|
2375
|
-
});
|
|
2376
|
-
}
|
|
2377
|
-
|
|
2378
|
-
// Sort by priority (lower number = higher priority)
|
|
2379
|
-
strategies.sort((a, b) => a.priority - b.priority);
|
|
2380
|
-
|
|
2381
|
-
// Store the locator strategies on the element
|
|
2382
|
-
element.locatorStrategies = strategies;
|
|
2383
|
-
|
|
2384
|
-
return strategies;
|
|
2385
|
-
}
|
|
2386
|
-
|
|
2387
|
-
// Enhanced strategy building with sibling detection for better accuracy
|
|
2388
|
-
buildFastLocatorStrategies(element) {
|
|
2389
|
-
const strategies = [];
|
|
2390
|
-
|
|
2391
|
-
// Strategy 1: Direct element locators
|
|
2392
|
-
|
|
2393
|
-
// Strategy 1: Direct element locators
|
|
2394
|
-
if (element.accessibilityId && element.accessibilityId !== 'Unlabeled') {
|
|
2395
|
-
strategies.push({
|
|
2396
|
-
strategy: 'accessibility id',
|
|
2397
|
-
value: element.accessibilityId,
|
|
2398
|
-
priority: 1
|
|
2399
|
-
});
|
|
2400
|
-
}
|
|
2401
|
-
|
|
2402
|
-
if (element.label && element.label !== 'Unlabeled' && element.label !== element.accessibilityId) {
|
|
2403
|
-
strategies.push({
|
|
2404
|
-
strategy: 'name',
|
|
2405
|
-
value: element.label,
|
|
2406
|
-
priority: 2
|
|
2407
|
-
});
|
|
2408
|
-
}
|
|
2409
|
-
|
|
2410
|
-
// Strategy 2: Class name (lower priority)
|
|
2411
|
-
if (element.type) {
|
|
2412
|
-
strategies.push({
|
|
2413
|
-
strategy: 'class name',
|
|
2414
|
-
value: element.type,
|
|
2415
|
-
priority: 5
|
|
2416
|
-
});
|
|
2417
|
-
}
|
|
2418
|
-
|
|
2419
|
-
// Strategy 3: Sibling-based strategies for better accuracy
|
|
2420
|
-
const siblingStrategies = this.buildSiblingStrategies(element);
|
|
2421
|
-
strategies.push(...siblingStrategies);
|
|
2422
|
-
|
|
2423
|
-
element.locatorStrategies = strategies;
|
|
2424
|
-
}
|
|
2425
|
-
|
|
2426
|
-
// Build sibling-based locator strategies
|
|
2427
|
-
buildSiblingStrategies(element) {
|
|
2428
|
-
const strategies = [];
|
|
2429
|
-
|
|
2430
|
-
// Find nearby labeled elements that can help identify this element
|
|
2431
|
-
const nearbyElements = this.cachedElements.filter(elem => {
|
|
2432
|
-
if (elem === element) return false;
|
|
2433
|
-
|
|
2434
|
-
const dx = Math.abs(elem.centerX - element.centerX);
|
|
2435
|
-
const dy = Math.abs(elem.centerY - element.centerY);
|
|
2436
|
-
const distance = Math.sqrt(dx * dx + dy * dy);
|
|
2437
|
-
|
|
2438
|
-
// Look for nearby elements with good labels
|
|
2439
|
-
return distance <= 100 && elem.label && elem.label !== 'Unlabeled' && elem.label.length > 2;
|
|
2440
|
-
});
|
|
2441
|
-
|
|
2442
|
-
// Create sibling-based strategies
|
|
2443
|
-
for (const sibling of nearbyElements.slice(0, 3)) { // Limit to 3 for performance
|
|
2444
|
-
if (sibling.accessibilityId && sibling.accessibilityId !== 'Unlabeled') {
|
|
2445
|
-
// Create XPath strategy using sibling
|
|
2446
|
-
const xpath = this.buildSiblingXPath(element, sibling);
|
|
2447
|
-
if (xpath) {
|
|
2448
|
-
strategies.push({
|
|
2449
|
-
strategy: 'xpath',
|
|
2450
|
-
value: xpath,
|
|
2451
|
-
priority: 3,
|
|
2452
|
-
siblingBased: true,
|
|
2453
|
-
siblingLabel: sibling.label
|
|
2454
|
-
});
|
|
2455
|
-
}
|
|
2456
|
-
}
|
|
2457
|
-
}
|
|
2458
|
-
|
|
2459
|
-
return strategies;
|
|
2460
|
-
}
|
|
2461
|
-
|
|
2462
|
-
// Build XPath using sibling elements
|
|
2463
|
-
buildSiblingXPath(targetElement, siblingElement) {
|
|
2464
|
-
try {
|
|
2465
|
-
// Simple sibling-based XPath strategies
|
|
2466
|
-
if (siblingElement.accessibilityId) {
|
|
2467
|
-
// Target element following sibling
|
|
2468
|
-
const siblingClass = siblingElement.type.replace('XCUIElementType', '');
|
|
2469
|
-
const targetClass = targetElement.type.replace('XCUIElementType', '');
|
|
2470
|
-
|
|
2471
|
-
// Try "following" axis
|
|
2472
|
-
return `//XCUIElementType${siblingClass}[@name="${siblingElement.accessibilityId}"]/following::XCUIElementType${targetClass}[1]`;
|
|
2473
|
-
}
|
|
2474
|
-
} catch (error) {
|
|
2475
|
-
this.logger.debug(`Sibling XPath creation failed: ${error.message}`);
|
|
2476
|
-
}
|
|
2477
|
-
|
|
2478
|
-
return null;
|
|
2479
|
-
}
|
|
2480
|
-
|
|
2481
|
-
// Enhanced strategy building with siblings - called only when broadcasting to followers
|
|
2482
|
-
buildEnhancedLocatorStrategies(element, clickX, clickY) {
|
|
2483
|
-
// Start with fast strategies
|
|
2484
|
-
this.buildFastLocatorStrategies(element);
|
|
2485
|
-
|
|
2486
|
-
// Add sibling strategies only if we don't have good direct strategies
|
|
2487
|
-
const hasGoodDirectStrategy = element.locatorStrategies.some(s =>
|
|
2488
|
-
s.strategy === 'accessibility id' || (s.strategy === 'name' && s.value.length > 3)
|
|
2489
|
-
);
|
|
2490
|
-
|
|
2491
|
-
if (!hasGoodDirectStrategy) {
|
|
2492
|
-
// Only do expensive sibling detection when needed
|
|
2493
|
-
const siblings = this.findSiblingElements(element, clickX, clickY);
|
|
2494
|
-
|
|
2495
|
-
siblings.forEach((sibling, index) => {
|
|
2496
|
-
const siblingLabel = sibling.element.label;
|
|
2497
|
-
if (siblingLabel && siblingLabel !== 'Unlabeled' && siblingLabel.length > 2) {
|
|
2498
|
-
|
|
2499
|
-
// Simple sibling strategies (avoid complex XPath)
|
|
2500
|
-
element.locatorStrategies.push({
|
|
2501
|
-
strategy: 'accessibility id',
|
|
2502
|
-
value: siblingLabel,
|
|
2503
|
-
priority: 4 + index,
|
|
2504
|
-
siblingBased: true,
|
|
2505
|
-
siblingLabel: siblingLabel
|
|
2506
|
-
});
|
|
2507
|
-
|
|
2508
|
-
// Only add XPath for very short, common labels
|
|
2509
|
-
if (siblingLabel.length < 15) {
|
|
2510
|
-
element.locatorStrategies.push({
|
|
2511
|
-
strategy: 'xpath',
|
|
2512
|
-
value: `//*[contains(@label, '${siblingLabel}')]`,
|
|
2513
|
-
priority: 6 + index,
|
|
2514
|
-
siblingBased: true,
|
|
2515
|
-
siblingLabel: siblingLabel
|
|
2516
|
-
});
|
|
2517
|
-
}
|
|
2518
|
-
}
|
|
2519
|
-
});
|
|
2520
|
-
|
|
2521
|
-
// Log sibling enhancement
|
|
2522
|
-
const siblingStrategies = element.locatorStrategies.filter(s => s.siblingBased);
|
|
2523
|
-
if (siblingStrategies.length > 0) {
|
|
2524
|
-
this.logger.info(`🔗 Enhanced with ${siblingStrategies.length} sibling strategies: ${siblingStrategies.map(s => s.siblingLabel).join(', ')}`);
|
|
2525
|
-
}
|
|
2526
|
-
}
|
|
2527
|
-
|
|
2528
|
-
// Limit to top 4 strategies for speed
|
|
2529
|
-
element.locatorStrategies = element.locatorStrategies
|
|
2530
|
-
.sort((a, b) => a.priority - b.priority)
|
|
2531
|
-
.slice(0, 4);
|
|
2532
|
-
}
|
|
2533
|
-
|
|
2534
|
-
// Find sibling elements that might provide better locators
|
|
2535
|
-
findSiblingElements(element, clickX, clickY) {
|
|
2536
|
-
const siblings = [];
|
|
2537
|
-
const searchRadius = 100; // Search area around the element
|
|
2538
|
-
|
|
2539
|
-
// Look for text elements near the clicked element
|
|
2540
|
-
this.cachedElements.forEach(candidate => {
|
|
2541
|
-
if (candidate === element) return; // Skip self
|
|
2542
|
-
|
|
2543
|
-
// Calculate distance between elements (not click point)
|
|
2544
|
-
const dx = Math.abs(candidate.centerX - element.centerX);
|
|
2545
|
-
const dy = Math.abs(candidate.centerY - element.centerY);
|
|
2546
|
-
const distance = Math.sqrt(dx * dx + dy * dy);
|
|
2547
|
-
|
|
2548
|
-
if (distance <= searchRadius) {
|
|
2549
|
-
// Prefer StaticText elements with meaningful labels
|
|
2550
|
-
if (candidate.type.includes('StaticText') &&
|
|
2551
|
-
candidate.label &&
|
|
2552
|
-
candidate.label !== 'Unlabeled' &&
|
|
2553
|
-
candidate.label.length > 2) {
|
|
2554
|
-
|
|
2555
|
-
// Check if it's likely a label for this element (vertical or horizontal alignment)
|
|
2556
|
-
const isVerticallyAligned = Math.abs(candidate.centerX - element.centerX) < 30;
|
|
2557
|
-
const isHorizontallyAligned = Math.abs(candidate.centerY - element.centerY) < 20;
|
|
2558
|
-
const isNearby = distance < 60;
|
|
2559
|
-
|
|
2560
|
-
if ((isVerticallyAligned || isHorizontallyAligned) && isNearby) {
|
|
2561
|
-
siblings.push({
|
|
2562
|
-
element: candidate,
|
|
2563
|
-
distance: distance,
|
|
2564
|
-
relationship: isVerticallyAligned ? 'vertical' : 'horizontal'
|
|
2565
|
-
});
|
|
2566
|
-
}
|
|
2567
|
-
}
|
|
2568
|
-
}
|
|
2569
|
-
});
|
|
2570
|
-
|
|
2571
|
-
// Sort siblings by distance and return best ones
|
|
2572
|
-
return siblings.sort((a, b) => a.distance - b.distance).slice(0, 3);
|
|
2573
|
-
}
|
|
2574
|
-
|
|
2575
|
-
async executeOnFollowerWithLocators(follower, locators, fallbackX, fallbackY) {
|
|
2576
|
-
// Use session lock to prevent concurrent WDA access
|
|
2577
|
-
return this.withSessionLock(follower.name, async () => {
|
|
2578
|
-
return this._executeOnFollowerWithLocators(follower, locators, fallbackX, fallbackY);
|
|
2579
|
-
});
|
|
2580
|
-
}
|
|
2581
|
-
|
|
2582
|
-
async _executeOnFollowerWithLocators(follower, locators, fallbackX, fallbackY) {
|
|
2583
|
-
const axios = require('axios');
|
|
2584
|
-
const followerPort = follower.port || 8100;
|
|
2585
|
-
|
|
2586
|
-
// OPTIMIZATION: Build enhanced strategies with siblings only when needed for followers
|
|
2587
|
-
if (locators && locators.element) {
|
|
2588
|
-
// Start with fast strategies only
|
|
2589
|
-
this.buildFastLocatorStrategies(locators.element);
|
|
2590
|
-
// Update locators with basic strategies first
|
|
2591
|
-
locators.locatorStrategies = locators.element.locatorStrategies;
|
|
2592
|
-
}
|
|
2593
|
-
|
|
2594
|
-
this.logger.info('');
|
|
2595
|
-
this.logger.info(` 📲 Processing follower: ${follower.name}`);
|
|
2596
|
-
|
|
2597
|
-
// GENIUS: Use global session approach - get session from connectedDevices
|
|
2598
|
-
let followerSessionId = null;
|
|
2599
|
-
|
|
2600
|
-
// First, try to get sessionId from the follower object itself (may be wdaSessionId for iOS)
|
|
2601
|
-
if (follower.wdaSessionId) {
|
|
2602
|
-
followerSessionId = follower.wdaSessionId;
|
|
2603
|
-
this.logger.info(` ✓ Session found on follower object: ${followerSessionId.substring(0, 8)}...`);
|
|
2604
|
-
} else if (follower.sessionId) {
|
|
2605
|
-
followerSessionId = follower.sessionId;
|
|
2606
|
-
this.logger.info(` ✓ Session found on follower object: ${followerSessionId.substring(0, 8)}...`);
|
|
2607
|
-
}
|
|
2608
|
-
|
|
2609
|
-
// If not found on follower object, find the follower in global connectedDevices array
|
|
2610
|
-
if (!followerSessionId) {
|
|
2611
|
-
this.logger.info(` - Checking global connectedDevices...`);
|
|
2612
|
-
const globalDevice = this.connectedDevices.find(d => d.name === follower.name || d.udid === follower.udid);
|
|
2613
|
-
if (globalDevice) {
|
|
2614
|
-
// Check both sessionId (Android) and wdaSessionId (iOS)
|
|
2615
|
-
followerSessionId = globalDevice.wdaSessionId || globalDevice.sessionId;
|
|
2616
|
-
if (followerSessionId) {
|
|
2617
|
-
this.logger.info(` ✓ Session found in global devices: ${followerSessionId.substring(0, 8)}...`);
|
|
2618
|
-
}
|
|
2619
|
-
}
|
|
2620
|
-
}
|
|
2621
|
-
|
|
2622
|
-
// If no global session, try to get from WDA status endpoint
|
|
2623
|
-
if (!followerSessionId) {
|
|
2624
|
-
this.logger.info(` - Querying WDA /status endpoint on port ${followerPort}...`);
|
|
2625
|
-
try {
|
|
2626
|
-
const followerStatusRes = await axios.get(`http://localhost:${followerPort}/status`, { timeout: 2000 });
|
|
2627
|
-
followerSessionId = followerStatusRes.data?.sessionId || followerStatusRes.data?.value?.sessionId;
|
|
2628
|
-
|
|
2629
|
-
// If we found a session, store it globally for reuse
|
|
2630
|
-
if (followerSessionId) {
|
|
2631
|
-
this.logger.info(` ✓ Session obtained from /status: ${followerSessionId.substring(0, 8)}...`);
|
|
2632
|
-
const globalDevice = this.connectedDevices.find(d => d.name === follower.name);
|
|
2633
|
-
if (globalDevice) {
|
|
2634
|
-
if (follower.platform === 'ios') {
|
|
2635
|
-
globalDevice.wdaSessionId = followerSessionId;
|
|
2636
|
-
} else {
|
|
2637
|
-
globalDevice.sessionId = followerSessionId;
|
|
2638
|
-
}
|
|
2639
|
-
}
|
|
2640
|
-
}
|
|
2641
|
-
} catch (error) {
|
|
2642
|
-
this.logger.error(` ✗ Failed to get session from /status: ${error.message}`);
|
|
2643
|
-
return;
|
|
2644
|
-
}
|
|
2645
|
-
}
|
|
2646
|
-
|
|
2647
|
-
if (!followerSessionId) {
|
|
2648
|
-
this.logger.warn(` ⚠️ No active session for follower ${follower.name}, attempting to create one...`);
|
|
2649
|
-
|
|
2650
|
-
// Last resort: Try to create a new session for the follower
|
|
2651
|
-
try {
|
|
2652
|
-
this.logger.info(` 🔧 Creating new WDA session for follower ${follower.name}...`);
|
|
2653
|
-
const sessionResponse = await axios.post(`http://localhost:${followerPort}/session`, {
|
|
2654
|
-
capabilities: {
|
|
2655
|
-
alwaysMatch: {},
|
|
2656
|
-
firstMatch: [{}]
|
|
2657
|
-
}
|
|
2658
|
-
}, { timeout: 10000 });
|
|
2659
|
-
|
|
2660
|
-
followerSessionId = sessionResponse.data?.sessionId || sessionResponse.data?.value?.sessionId;
|
|
2661
|
-
if (followerSessionId) {
|
|
2662
|
-
this.logger.info(` ✅ Created new session for follower: ${followerSessionId.substring(0, 8)}...`);
|
|
2663
|
-
|
|
2664
|
-
// Store the session
|
|
2665
|
-
if (follower.platform === 'ios') {
|
|
2666
|
-
follower.wdaSessionId = followerSessionId;
|
|
2667
|
-
} else {
|
|
2668
|
-
follower.sessionId = followerSessionId;
|
|
2669
|
-
}
|
|
2670
|
-
|
|
2671
|
-
// Store in global connectedDevices and session manager
|
|
2672
|
-
const globalDevice = this.connectedDevices.find(d => d.name === follower.name);
|
|
2673
|
-
if (globalDevice) {
|
|
2674
|
-
if (follower.platform === 'ios') {
|
|
2675
|
-
globalDevice.wdaSessionId = followerSessionId;
|
|
2676
|
-
} else {
|
|
2677
|
-
globalDevice.sessionId = followerSessionId;
|
|
2678
|
-
}
|
|
2679
|
-
}
|
|
2680
|
-
|
|
2681
|
-
// Register with global session manager if available
|
|
2682
|
-
if (global.sessionManager && follower.udid) {
|
|
2683
|
-
global.sessionManager.registerSession(follower.udid, followerSessionId, followerPort, follower.name, null, follower.platform);
|
|
2684
|
-
}
|
|
2685
|
-
} else {
|
|
2686
|
-
this.logger.error(` ✗ Failed to create session - no sessionId in response`);
|
|
2687
|
-
return;
|
|
2688
|
-
}
|
|
2689
|
-
} catch (createError) {
|
|
2690
|
-
this.logger.error(` ✗ Failed to create session for follower: ${createError.message}`);
|
|
2691
|
-
return;
|
|
2692
|
-
}
|
|
2693
|
-
}
|
|
2694
|
-
|
|
2695
|
-
this.logger.info(` 📱 Using WDA session: ${followerSessionId.substring(0, 8)}... on port ${followerPort}`);
|
|
2696
|
-
|
|
2697
|
-
// Validate session is still active before trying locators
|
|
2698
|
-
try {
|
|
2699
|
-
this.logger.info(` ✓ Validating WDA session is active...`);
|
|
2700
|
-
const statusCheck = await axios.get(`http://localhost:${followerPort}/status`, { timeout: 2000 });
|
|
2701
|
-
if (statusCheck.data?.sessionId !== followerSessionId) {
|
|
2702
|
-
this.logger.warn(` ⚠️ Session ID mismatch - WDA has different session now`);
|
|
2703
|
-
this.logger.warn(` Expected: ${followerSessionId.substring(0, 8)}...`);
|
|
2704
|
-
this.logger.warn(` Current: ${statusCheck.data?.sessionId?.substring(0, 8)}...`);
|
|
2705
|
-
|
|
2706
|
-
// Update to the current active session
|
|
2707
|
-
followerSessionId = statusCheck.data?.sessionId;
|
|
2708
|
-
if (follower.platform === 'ios') {
|
|
2709
|
-
follower.wdaSessionId = followerSessionId;
|
|
2710
|
-
} else {
|
|
2711
|
-
follower.sessionId = followerSessionId;
|
|
2712
|
-
}
|
|
2713
|
-
this.logger.info(` ✓ Updated to current session: ${followerSessionId.substring(0, 8)}...`);
|
|
2714
|
-
|
|
2715
|
-
// Try to sync follower to same app as commander
|
|
2716
|
-
try {
|
|
2717
|
-
// Get commander's current app
|
|
2718
|
-
const commanderPort = this.commanderDevice.port || 8100;
|
|
2719
|
-
const commanderSessionId = this.getWDASessionId();
|
|
2720
|
-
|
|
2721
|
-
if (commanderSessionId) {
|
|
2722
|
-
const commanderAppRes = await axios.get(`http://localhost:${commanderPort}/session/${commanderSessionId}/wda/activeAppInfo`, { timeout: 3000 });
|
|
2723
|
-
const commanderAppId = commanderAppRes.data?.value?.bundleId;
|
|
2724
|
-
|
|
2725
|
-
if (commanderAppId && commanderAppId !== 'com.apple.springboard') {
|
|
2726
|
-
this.logger.info(` 🔄 Syncing follower to commander app: ${commanderAppId}`);
|
|
2727
|
-
await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/wda/apps/launch`, {
|
|
2728
|
-
bundleId: commanderAppId
|
|
2729
|
-
}, { timeout: 20000 }); // Increased from 15s to 20s for app launch
|
|
2730
|
-
|
|
2731
|
-
// Wait longer for app to load and stabilize
|
|
2732
|
-
await new Promise(resolve => setTimeout(resolve, 3000)); // Increased from 1.5s to 3s
|
|
2733
|
-
this.logger.info(` ✅ Follower synced to ${commanderAppId}`);
|
|
2734
|
-
} else {
|
|
2735
|
-
this.logger.info(` ℹ️ Commander in home screen, not syncing follower app`);
|
|
2736
|
-
}
|
|
2737
|
-
}
|
|
2738
|
-
} catch (syncError) {
|
|
2739
|
-
this.logger.warn(` ⚠️ Could not sync follower app: ${syncError.message}`);
|
|
2740
|
-
}
|
|
2741
|
-
}
|
|
2742
|
-
} catch (statusErr) {
|
|
2743
|
-
this.logger.warn(` ⚠️ Could not validate session: ${statusErr.message}`);
|
|
2744
|
-
// Continue anyway - maybe the /status endpoint is not available
|
|
2745
|
-
}
|
|
2746
|
-
|
|
2747
|
-
let tapExecuted = false;
|
|
2748
|
-
let sessionInvalid = false;
|
|
2749
|
-
|
|
2750
|
-
// Try locator strategies in priority order
|
|
2751
|
-
if (locators?.locatorStrategies && locators.locatorStrategies.length > 0) {
|
|
2752
|
-
// Sort by priority
|
|
2753
|
-
const strategies = [...locators.locatorStrategies].sort((a, b) => a.priority - b.priority);
|
|
2754
|
-
|
|
2755
|
-
this.logger.info(` 🔎 Trying ${strategies.length} locator strategies:`);
|
|
2756
|
-
|
|
2757
|
-
for (const strategy of strategies) {
|
|
2758
|
-
if (tapExecuted) break;
|
|
2759
|
-
|
|
2760
|
-
try {
|
|
2761
|
-
const strategyDesc = strategy.siblingBased ?
|
|
2762
|
-
`${strategy.strategy}="${strategy.value}" (sibling: "${strategy.siblingLabel}")` :
|
|
2763
|
-
`${strategy.strategy}="${strategy.value}"`;
|
|
2764
|
-
this.logger.info(` [${strategy.priority}] Trying: ${strategyDesc}`);
|
|
2765
|
-
|
|
2766
|
-
const elementRes = await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element`, {
|
|
2767
|
-
using: strategy.strategy,
|
|
2768
|
-
value: strategy.value
|
|
2769
|
-
}, { timeout: 4000 }); // Slightly increased for XPath queries
|
|
2770
|
-
|
|
2771
|
-
if (elementRes.data?.value) {
|
|
2772
|
-
const elementId = elementRes.data.value.ELEMENT || elementRes.data.value['element-6066-11e4-a52e-4f735466cecf'];
|
|
2773
|
-
const successMsg = strategy.siblingBased ?
|
|
2774
|
-
`Element found via sibling "${strategy.siblingLabel}": ${elementId.substring(0, 16)}...` :
|
|
2775
|
-
`Element found: ${elementId.substring(0, 16)}...`;
|
|
2776
|
-
this.logger.info(` ✓ ${successMsg}`);
|
|
2777
|
-
|
|
2778
|
-
this.logger.info(` ⏳ Executing click...`);
|
|
2779
|
-
await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element/${elementId}/click`, {}, { timeout: 3000 });
|
|
2780
|
-
|
|
2781
|
-
const clickSuccessMsg = strategy.siblingBased ?
|
|
2782
|
-
`SUCCESS: Follower ${follower.name} tapped using sibling strategy (${strategy.siblingLabel})` :
|
|
2783
|
-
`SUCCESS: Follower ${follower.name} tapped using ${strategy.strategy}`;
|
|
2784
|
-
this.logger.info(` ✅ ${clickSuccessMsg}`);
|
|
2785
|
-
tapExecuted = true;
|
|
2786
|
-
}
|
|
2787
|
-
} catch (error) {
|
|
2788
|
-
// Check if session is invalid (404)
|
|
2789
|
-
if (error.response?.status === 404) {
|
|
2790
|
-
sessionInvalid = true;
|
|
2791
|
-
this.logger.warn(` ✗ Session invalid (404 error)`);
|
|
2792
|
-
this.logger.warn(` ✗ WDA endpoint returned 404 - session may have expired`);
|
|
2793
|
-
this.logger.debug(` Response: ${JSON.stringify(error.response?.data)?.substring(0, 200)}`);
|
|
2794
|
-
break; // Stop trying strategies, move to fallback
|
|
2795
|
-
} else if (error.response?.status === 400) {
|
|
2796
|
-
// Bad request - strategy/value might be invalid
|
|
2797
|
-
this.logger.warn(` ✗ Bad request (400) - strategy or value format invalid`);
|
|
2798
|
-
this.logger.debug(` Response: ${JSON.stringify(error.response?.data)?.substring(0, 200)}`);
|
|
2799
|
-
} else if (error.code === 'ECONNREFUSED') {
|
|
2800
|
-
// Can't reach WDA server
|
|
2801
|
-
this.logger.error(` ✗ ECONNREFUSED - WDA server not responding on port ${followerPort}`);
|
|
2802
|
-
sessionInvalid = true;
|
|
2803
|
-
break;
|
|
2804
|
-
} else {
|
|
2805
|
-
this.logger.debug(` ✗ ${strategy.strategy} strategy failed: ${error.message}`);
|
|
2806
|
-
}
|
|
2807
|
-
}
|
|
2808
|
-
}
|
|
2809
|
-
} else if (locators && (locators.accessibilityId || locators.label)) {
|
|
2810
|
-
// Fallback for old format
|
|
2811
|
-
const strategy = locators.accessibilityId ? 'accessibility id' : 'name';
|
|
2812
|
-
const value = locators.accessibilityId || locators.label;
|
|
2813
|
-
|
|
2814
|
-
this.logger.info(` 🔎 Using fallback locator: ${strategy}="${value}"`);
|
|
2815
|
-
|
|
2816
|
-
try {
|
|
2817
|
-
const elementRes = await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element`, {
|
|
2818
|
-
using: strategy,
|
|
2819
|
-
value: value
|
|
2820
|
-
}, { timeout: 3000 });
|
|
2821
|
-
|
|
2822
|
-
if (elementRes.data?.value) {
|
|
2823
|
-
const elementId = elementRes.data.value.ELEMENT || elementRes.data.value['element-6066-11e4-a52e-4f735466cecf'];
|
|
2824
|
-
this.logger.info(` ✓ Element found: ${elementId.substring(0, 16)}...`);
|
|
2825
|
-
|
|
2826
|
-
this.logger.info(` ⏳ Executing click...`);
|
|
2827
|
-
await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element/${elementId}/click`, {}, { timeout: 3000 });
|
|
2828
|
-
this.logger.info(` ✅ Tap executed!`);
|
|
2829
|
-
this.logger.info(` ✅ SUCCESS: Follower ${follower.name} tapped using ${strategy}`);
|
|
2830
|
-
tapExecuted = true;
|
|
2831
|
-
}
|
|
2832
|
-
} catch (error) {
|
|
2833
|
-
// Check if session is invalid (404)
|
|
2834
|
-
if (error.response?.status === 404) {
|
|
2835
|
-
sessionInvalid = true;
|
|
2836
|
-
this.logger.warn(` ✗ Session invalid (404), will use coordinate fallback`);
|
|
2837
|
-
} else {
|
|
2838
|
-
this.logger.debug(` ✗ Locator-based tap failed: ${error.message}`);
|
|
2839
|
-
}
|
|
2840
|
-
}
|
|
2841
|
-
} else {
|
|
2842
|
-
this.logger.info(` ℹ️ No locator strategies available, will use coordinate fallback`);
|
|
2843
|
-
}
|
|
2844
|
-
|
|
2845
|
-
// FALLBACK: If no locator worked and we have element context, try enhanced strategies
|
|
2846
|
-
if (!tapExecuted && !sessionInvalid && locators?.element && locators.locatorStrategies && locators.locatorStrategies.length > 0) {
|
|
2847
|
-
this.logger.info(` 🔄 Primary locators failed, trying enhanced sibling strategies...`);
|
|
2848
|
-
|
|
2849
|
-
// Build enhanced strategies with siblings
|
|
2850
|
-
this.buildEnhancedLocatorStrategies(locators.element, fallbackX, fallbackY);
|
|
2851
|
-
const enhancedStrategies = locators.element.locatorStrategies.filter(s => s.siblingBased);
|
|
2852
|
-
|
|
2853
|
-
if (enhancedStrategies.length > 0) {
|
|
2854
|
-
this.logger.info(` 🔗 Trying ${enhancedStrategies.length} sibling-based strategies...`);
|
|
2855
|
-
|
|
2856
|
-
for (const strategy of enhancedStrategies.slice(0, 2)) { // Limit to 2 for speed
|
|
2857
|
-
if (tapExecuted) break;
|
|
2858
|
-
|
|
2859
|
-
try {
|
|
2860
|
-
const elementRes = await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element`, {
|
|
2861
|
-
using: strategy.strategy,
|
|
2862
|
-
value: strategy.value
|
|
2863
|
-
}, { timeout: 3000 });
|
|
2864
|
-
|
|
2865
|
-
if (elementRes.data?.value) {
|
|
2866
|
-
const elementId = elementRes.data.value.ELEMENT || elementRes.data.value['element-6066-11e4-a52e-4f735466cecf'];
|
|
2867
|
-
this.logger.info(` ✅ Sibling strategy worked: ${strategy.siblingLabel}`);
|
|
2868
|
-
|
|
2869
|
-
await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/element/${elementId}/click`, {}, { timeout: 3000 });
|
|
2870
|
-
this.logger.info(` ✅ SUCCESS: Enhanced locator tap executed on ${follower.name}`);
|
|
2871
|
-
tapExecuted = true;
|
|
2872
|
-
}
|
|
2873
|
-
} catch (enhancedErr) {
|
|
2874
|
-
this.logger.debug(` ✗ Enhanced strategy failed: ${enhancedErr.message}`);
|
|
2875
|
-
}
|
|
2876
|
-
}
|
|
2877
|
-
}
|
|
2878
|
-
}
|
|
2879
|
-
|
|
2880
|
-
if (!tapExecuted) {
|
|
2881
|
-
try {
|
|
2882
|
-
// GENIUS: Smart session recovery - avoid recreating sessions unless absolutely necessary
|
|
2883
|
-
if (sessionInvalid) {
|
|
2884
|
-
this.logger.info(` 🔄 Session was invalid (404) - MUST recover before coordinate tap`);
|
|
2885
|
-
|
|
2886
|
-
// Step 1: Check if WDA has any active session we can use
|
|
2887
|
-
try {
|
|
2888
|
-
const statusRes = await axios.get(`http://localhost:${followerPort}/status`, { timeout: 2000 });
|
|
2889
|
-
const activeSession = statusRes.data?.sessionId;
|
|
2890
|
-
|
|
2891
|
-
if (activeSession && activeSession !== followerSessionId) {
|
|
2892
|
-
// WDA has a different session - update our reference
|
|
2893
|
-
this.logger.info(` ✅ Found active WDA session: ${activeSession.substring(0, 8)}...`);
|
|
2894
|
-
followerSessionId = activeSession;
|
|
2895
|
-
|
|
2896
|
-
// Update follower object
|
|
2897
|
-
if (follower.platform === 'ios') {
|
|
2898
|
-
follower.wdaSessionId = activeSession;
|
|
2899
|
-
} else {
|
|
2900
|
-
follower.sessionId = activeSession;
|
|
2901
|
-
}
|
|
2902
|
-
|
|
2903
|
-
// Update global session manager
|
|
2904
|
-
if (global.sessionManager) {
|
|
2905
|
-
global.sessionManager.updateSession(follower.udid, activeSession);
|
|
2906
|
-
}
|
|
2907
|
-
|
|
2908
|
-
this.logger.info(` ✅ Session recovered without recreation: ${activeSession.substring(0, 8)}...`);
|
|
2909
|
-
|
|
2910
|
-
// Quick test to make sure it works
|
|
2911
|
-
await axios.get(`http://localhost:${followerPort}/session/${activeSession}/window/size`, { timeout: 2000 });
|
|
2912
|
-
this.logger.debug(` ✓ Session validated and ready`);
|
|
2913
|
-
|
|
2914
|
-
} else if (!activeSession) {
|
|
2915
|
-
this.logger.error(` ✗ Cannot proceed - WDA has no active session and creation would close apps`);
|
|
2916
|
-
this.logger.warn(` 🚫 Skipping ${follower.name} to prevent app disruption`);
|
|
2917
|
-
return;
|
|
2918
|
-
} else {
|
|
2919
|
-
this.logger.debug(` ℹ️ Session ID matches, session should be valid now`);
|
|
2920
|
-
}
|
|
2921
|
-
} catch (statusError) {
|
|
2922
|
-
this.logger.error(` ✗ Cannot proceed - session is still invalid and WDA status failed`);
|
|
2923
|
-
this.logger.warn(` 🚫 Skipping ${follower.name} to prevent further issues`);
|
|
2924
|
-
return;
|
|
2925
|
-
}
|
|
2926
|
-
}
|
|
2927
|
-
|
|
2928
|
-
// Coordinate fallback after session recovery
|
|
2929
|
-
if (!tapExecuted && followerSessionId) {
|
|
2930
|
-
// Simple coordinate validation for follower
|
|
2931
|
-
let safeFallbackX = Math.round(fallbackX);
|
|
2932
|
-
let safeFallbackY = Math.round(fallbackY);
|
|
2933
|
-
|
|
2934
|
-
const deviceHeight = 896; // iPhone default
|
|
2935
|
-
const deviceWidth = 414;
|
|
2936
|
-
|
|
2937
|
-
// Basic bounds checking only
|
|
2938
|
-
if (safeFallbackX < 0 || safeFallbackX > deviceWidth || safeFallbackY < 0 || safeFallbackY > deviceHeight) {
|
|
2939
|
-
this.logger.warn(`⚠️ FOLLOWER: Coordinates outside device bounds, clamping: (${safeFallbackX}, ${safeFallbackY})`);
|
|
2940
|
-
safeFallbackX = Math.max(10, Math.min(safeFallbackX, deviceWidth - 10));
|
|
2941
|
-
safeFallbackY = Math.max(50, Math.min(safeFallbackY, deviceHeight - 100));
|
|
2942
|
-
}
|
|
2943
|
-
|
|
2944
|
-
this.logger.info(` 🎯 Fallback: Using coordinate-based tap at (${safeFallbackX}, ${safeFallbackY})`);
|
|
2945
|
-
this.logger.info(` Session: ${followerSessionId.substring(0, 8)}...`);
|
|
2946
|
-
|
|
2947
|
-
try {
|
|
2948
|
-
await axios.post(`http://localhost:${followerPort}/session/${followerSessionId}/actions`, {
|
|
2949
|
-
actions: [{
|
|
2950
|
-
type: 'pointer',
|
|
2951
|
-
id: 'finger1',
|
|
2952
|
-
parameters: { pointerType: 'touch' },
|
|
2953
|
-
actions: [
|
|
2954
|
-
{ type: 'pointerMove', duration: 0, x: safeFallbackX, y: safeFallbackY },
|
|
2955
|
-
{ type: 'pointerDown', button: 0 },
|
|
2956
|
-
{ type: 'pause', duration: 100 },
|
|
2957
|
-
{ type: 'pointerUp', button: 0 }
|
|
2958
|
-
]
|
|
2959
|
-
}]
|
|
2960
|
-
}, { timeout: 15000 });
|
|
2961
|
-
this.logger.info(` ✅ SUCCESS: Fallback tap executed on ${follower.name}`);
|
|
2962
|
-
} catch (fallbackErr) {
|
|
2963
|
-
if (fallbackErr.response?.status === 404) {
|
|
2964
|
-
this.logger.error(` ✗ FAILED: Coordinate tap got 404 - session is still invalid`);
|
|
2965
|
-
this.logger.error(` Port: ${followerPort}, Session: ${followerSessionId.substring(0, 8)}...`);
|
|
2966
|
-
} else {
|
|
2967
|
-
this.logger.error(` ✗ FAILED: Coordinate tap failed: ${fallbackErr.message}`);
|
|
2968
|
-
}
|
|
2969
|
-
}
|
|
2970
|
-
}
|
|
2971
|
-
} catch (error) {
|
|
2972
|
-
this.logger.error(` ✗ FAILED: Follower ${follower.name} tap execution failed: ${error.message}`);
|
|
2973
|
-
}
|
|
2974
|
-
}
|
|
2975
|
-
}
|
|
2976
|
-
|
|
2977
|
-
async addFollower(deviceName, deviceInfo) {
|
|
2978
|
-
if (!this.isActive) {
|
|
2979
|
-
throw new Error('Commander mode is not active');
|
|
2980
|
-
}
|
|
2981
|
-
|
|
2982
|
-
// FIXED: Prevent duplicate followers - check if follower already exists
|
|
2983
|
-
const existingFollower = Array.from(this.followerDevices).find(f => f.name === deviceName);
|
|
2984
|
-
if (existingFollower) {
|
|
2985
|
-
this.logger.info(`✓ Follower ${deviceName} already exists, skipping duplicate addition`);
|
|
2986
|
-
return {
|
|
2987
|
-
success: true,
|
|
2988
|
-
follower: deviceName,
|
|
2989
|
-
totalFollowers: this.followerDevices.size,
|
|
2990
|
-
isDuplicate: true
|
|
2991
|
-
};
|
|
2992
|
-
}
|
|
2993
|
-
|
|
2994
|
-
const followerDevice = {
|
|
2995
|
-
name: deviceName,
|
|
2996
|
-
...deviceInfo
|
|
2997
|
-
};
|
|
2998
|
-
|
|
2999
|
-
// For Android devices, ensure session info is loaded and cache screenshot URL
|
|
3000
|
-
if (deviceInfo.platform === 'android') {
|
|
3001
|
-
// GENIUS: Use sessionManager for reliable session lookup
|
|
3002
|
-
const session = global.sessionManager?.getSession(followerDevice) || global.androidSessions?.get(deviceInfo.udid);
|
|
3003
|
-
|
|
3004
|
-
if (session && session.port && session.sessionId) {
|
|
3005
|
-
followerDevice.port = session.port;
|
|
3006
|
-
followerDevice.sessionId = session.sessionId;
|
|
3007
|
-
|
|
3008
|
-
// Pre-cache screenshot URL for instant mirroring
|
|
3009
|
-
followerDevice._cachedPort = session.port;
|
|
3010
|
-
followerDevice._cachedScreenshotUrl = `http://localhost:${session.port}/wd/hub/session/${session.sessionId}/screenshot`;
|
|
3011
|
-
|
|
3012
|
-
this.logger.info(`✅ Android follower session found and cached: port ${session.port}, sessionId ${session.sessionId.substring(0, 8)}...`);
|
|
3013
|
-
} else {
|
|
3014
|
-
this.logger.warn(`⚠️ No UIAutomator2 session found for Android follower ${deviceName}`);
|
|
3015
|
-
}
|
|
3016
|
-
}
|
|
3017
|
-
// For iOS devices, check for existing WDA session and cache screenshot URL
|
|
3018
|
-
else if (deviceInfo.platform === 'ios') {
|
|
3019
|
-
await this.checkFollowerWDASession(followerDevice);
|
|
3020
|
-
|
|
3021
|
-
// Pre-cache iOS screenshot URL if port is available
|
|
3022
|
-
if (followerDevice.port) {
|
|
3023
|
-
followerDevice._cachedPort = followerDevice.port;
|
|
3024
|
-
followerDevice._cachedScreenshotUrl = `http://localhost:${followerDevice.port}/screenshot`;
|
|
3025
|
-
this.logger.info(`✅ iOS follower screenshot URL cached: ${followerDevice.name}:${followerDevice.port}`);
|
|
3026
|
-
}
|
|
3027
|
-
}
|
|
3028
|
-
|
|
3029
|
-
this.followerDevices.add(followerDevice);
|
|
3030
|
-
this.logger.info(`Follower added: ${deviceName} (${deviceInfo.platform})`);
|
|
3031
|
-
this.broadcastCommanderStatus();
|
|
3032
|
-
|
|
3033
|
-
return {
|
|
3034
|
-
success: true,
|
|
3035
|
-
follower: deviceName,
|
|
3036
|
-
totalFollowers: this.followerDevices.size
|
|
3037
|
-
};
|
|
3038
|
-
}
|
|
3039
|
-
|
|
3040
|
-
async checkFollowerWDASession(followerDevice) {
|
|
3041
|
-
const axios = require('axios');
|
|
3042
|
-
const port = followerDevice.port || 8100;
|
|
3043
|
-
|
|
3044
|
-
try {
|
|
3045
|
-
this.logger.info(`Fast checking WDA session on follower ${followerDevice.name}...`);
|
|
3046
|
-
|
|
3047
|
-
// Check /status for existing session with reduced timeout
|
|
3048
|
-
const statusResponse = await axios.get(`http://localhost:${port}/status`, { timeout: 3000 }); // Reduced from 5000ms
|
|
3049
|
-
const sessionId = statusResponse.data?.sessionId || statusResponse.data?.value?.sessionId;
|
|
3050
|
-
|
|
3051
|
-
if (sessionId) {
|
|
3052
|
-
followerDevice.wdaSessionId = sessionId;
|
|
3053
|
-
followerDevice.port = port; // Ensure port is set
|
|
3054
|
-
this.logger.info(`✅ Follower using existing session: ${sessionId}`);
|
|
3055
|
-
return;
|
|
3056
|
-
}
|
|
3057
|
-
|
|
3058
|
-
// Try to get existing session from /sessions endpoint with reduced timeout
|
|
3059
|
-
try {
|
|
3060
|
-
const sessionsResponse = await axios.get(`http://localhost:${port}/sessions`, { timeout: 2000 }); // Reduced from 3000ms
|
|
3061
|
-
const sessions = sessionsResponse.data?.value;
|
|
3062
|
-
|
|
3063
|
-
if (sessions && Array.isArray(sessions) && sessions.length > 0) {
|
|
3064
|
-
followerDevice.wdaSessionId = sessions[0].id;
|
|
3065
|
-
followerDevice.port = port; // Ensure port is set
|
|
3066
|
-
this.logger.info(`✅ Follower found session from /sessions: ${followerDevice.wdaSessionId}`);
|
|
3067
|
-
return;
|
|
3068
|
-
}
|
|
3069
|
-
} catch (e) {
|
|
3070
|
-
this.logger.debug('No sessions found at /sessions endpoint for follower');
|
|
3071
|
-
}
|
|
3072
|
-
|
|
3073
|
-
// Try alternative port quickly if first port failed
|
|
3074
|
-
if (port === 8100) {
|
|
3075
|
-
const altPort = 8101;
|
|
3076
|
-
try {
|
|
3077
|
-
this.logger.debug(`Trying alternative port ${altPort} for follower ${followerDevice.name}...`);
|
|
3078
|
-
const altStatusResponse = await axios.get(`http://localhost:${altPort}/status`, { timeout: 2000 });
|
|
3079
|
-
const altSessionId = altStatusResponse.data?.sessionId || altStatusResponse.data?.value?.sessionId;
|
|
3080
|
-
|
|
3081
|
-
if (altSessionId) {
|
|
3082
|
-
followerDevice.wdaSessionId = altSessionId;
|
|
3083
|
-
followerDevice.port = altPort;
|
|
3084
|
-
this.logger.info(`✅ Follower found session on alternative port ${altPort}: ${altSessionId}`);
|
|
3085
|
-
return;
|
|
3086
|
-
}
|
|
3087
|
-
} catch (altError) {
|
|
3088
|
-
this.logger.debug(`Alternative port ${altPort} also failed for follower`);
|
|
3089
|
-
}
|
|
3090
|
-
}
|
|
3091
|
-
|
|
3092
|
-
this.logger.warn(`⚠️ No existing WDA session found for follower ${followerDevice.name}`);
|
|
3093
|
-
|
|
3094
|
-
// OPTIMIZATION: Create session in background to avoid blocking UI
|
|
3095
|
-
this.createFollowerSessionInBackground(followerDevice, port);
|
|
3096
|
-
|
|
3097
|
-
this.logger.warn(`💡 Follower will work for screen mirroring when WDA becomes available`);
|
|
3098
|
-
|
|
3099
|
-
} catch (error) {
|
|
3100
|
-
this.logger.warn(`⚠️ Error checking follower WDA session: ${error.message}`);
|
|
3101
|
-
this.logger.warn(`💡 Follower will work for screen mirroring when session becomes available`);
|
|
3102
|
-
}
|
|
3103
|
-
}
|
|
3104
|
-
|
|
3105
|
-
/**
|
|
3106
|
-
* Create follower WDA session in background without blocking startup
|
|
3107
|
-
*/
|
|
3108
|
-
async createFollowerSessionInBackground(followerDevice, port) {
|
|
3109
|
-
// Don't block the main thread - create session asynchronously
|
|
3110
|
-
setTimeout(async () => {
|
|
3111
|
-
try {
|
|
3112
|
-
this.logger.info(`🔧 Creating WDA session for follower ${followerDevice.name} in background...`);
|
|
3113
|
-
|
|
3114
|
-
const axios = require('axios');
|
|
3115
|
-
const device = this.connectedDevices.find(d => d.name === followerDevice.name);
|
|
3116
|
-
if (!device) {
|
|
3117
|
-
throw new Error('Device not found');
|
|
3118
|
-
}
|
|
3119
|
-
|
|
3120
|
-
const sessionResponse = await axios.post(`http://localhost:${port}/session`, {
|
|
3121
|
-
capabilities: {
|
|
3122
|
-
alwaysMatch: {},
|
|
3123
|
-
firstMatch: [{}]
|
|
3124
|
-
}
|
|
3125
|
-
}, { timeout: 10000 });
|
|
3126
|
-
|
|
3127
|
-
const sessionId = sessionResponse.data?.sessionId || sessionResponse.data?.value?.sessionId;
|
|
3128
|
-
if (!sessionId) {
|
|
3129
|
-
throw new Error('No sessionId in response');
|
|
3130
|
-
}
|
|
3131
|
-
|
|
3132
|
-
// Update follower device with new session
|
|
3133
|
-
followerDevice.wdaSessionId = sessionId;
|
|
3134
|
-
followerDevice.port = port;
|
|
3135
|
-
|
|
3136
|
-
// Pre-cache screenshot URL for instant mirroring
|
|
3137
|
-
followerDevice._cachedPort = port;
|
|
3138
|
-
followerDevice._cachedScreenshotUrl = `http://localhost:${port}/screenshot`;
|
|
3139
|
-
|
|
3140
|
-
// Register with global session manager if available
|
|
3141
|
-
if (global.sessionManager) {
|
|
3142
|
-
global.sessionManager.registerSession(device.udid, sessionId, port, followerDevice.name, null, 'ios');
|
|
3143
|
-
}
|
|
3144
|
-
|
|
3145
|
-
// Store in global connectedDevices for persistence
|
|
3146
|
-
const globalDevice = this.connectedDevices.find(d => d.name === followerDevice.name);
|
|
3147
|
-
if (globalDevice) {
|
|
3148
|
-
globalDevice.wdaSessionId = sessionId;
|
|
3149
|
-
}
|
|
3150
|
-
|
|
3151
|
-
this.logger.info(`✅ Background session created for follower: ${followerDevice.name} (${sessionId.substring(0, 8)}...)`);
|
|
3152
|
-
|
|
3153
|
-
// Notify UI that follower is now fully ready
|
|
3154
|
-
this.broadcast({
|
|
3155
|
-
type: 'follower_session_ready',
|
|
3156
|
-
deviceName: followerDevice.name,
|
|
3157
|
-
sessionId: sessionId.substring(0, 8) + '...'
|
|
3158
|
-
});
|
|
3159
|
-
|
|
3160
|
-
} catch (sessionError) {
|
|
3161
|
-
this.logger.warn(`⚠️ Background session creation failed for follower ${followerDevice.name}: ${sessionError.message}`);
|
|
3162
|
-
// Retry once after 5 seconds
|
|
3163
|
-
setTimeout(() => {
|
|
3164
|
-
this.createFollowerSessionInBackground(followerDevice, port);
|
|
3165
|
-
}, 5000);
|
|
3166
|
-
}
|
|
3167
|
-
}, 1000); // Start after 1 second delay to not block initial setup
|
|
3168
|
-
}
|
|
3169
|
-
|
|
3170
|
-
removeFollower(deviceName, preserveSession = true) {
|
|
3171
|
-
const follower = Array.from(this.followerDevices).find(f => f.name === deviceName);
|
|
3172
|
-
if (follower) {
|
|
3173
|
-
this.followerDevices.delete(follower);
|
|
3174
|
-
|
|
3175
|
-
// CRITICAL: Clear any follower-specific state that might conflict with commander role
|
|
3176
|
-
this.logger.info(`🧹 Removing follower: ${deviceName} (clearing follower state)`);
|
|
3177
|
-
|
|
3178
|
-
// Only preserve session info if explicitly requested (for role changes, not disconnects)
|
|
3179
|
-
if (preserveSession && (follower.sessionId || follower.wdaSessionId)) {
|
|
3180
|
-
this.logger.info(`📝 Preserving session info from follower ${deviceName} for potential commander use`);
|
|
3181
|
-
|
|
3182
|
-
// Update the device in connectedDevices with session info
|
|
3183
|
-
const globalDevice = this.connectedDevices.find(d => d.name === deviceName);
|
|
3184
|
-
if (globalDevice) {
|
|
3185
|
-
if (follower.sessionId) globalDevice.sessionId = follower.sessionId;
|
|
3186
|
-
if (follower.wdaSessionId) globalDevice.wdaSessionId = follower.wdaSessionId;
|
|
3187
|
-
if (follower.port) globalDevice.port = follower.port;
|
|
3188
|
-
|
|
3189
|
-
this.logger.info(`📝 Session preserved: ${globalDevice.sessionId || globalDevice.wdaSessionId}`);
|
|
3190
|
-
}
|
|
3191
|
-
} else if (!preserveSession) {
|
|
3192
|
-
this.logger.info(`🧹 Device ${deviceName} disconnecting - not preserving session`);
|
|
3193
|
-
}
|
|
3194
|
-
|
|
3195
|
-
this.broadcastCommanderStatus();
|
|
3196
|
-
return { success: true };
|
|
3197
|
-
}
|
|
3198
|
-
return { success: false, error: 'Follower not found' };
|
|
3199
|
-
}
|
|
3200
|
-
|
|
3201
|
-
stopTouchMonitoring() {
|
|
3202
|
-
this.logger.info('🛑 Stopping touch monitoring and screen streaming...');
|
|
3203
|
-
|
|
3204
|
-
// Stop screen streaming
|
|
3205
|
-
if (this.screenStreamInterval) {
|
|
3206
|
-
clearInterval(this.screenStreamInterval);
|
|
3207
|
-
this.screenStreamInterval = null;
|
|
3208
|
-
}
|
|
3209
|
-
|
|
3210
|
-
// Stop touch monitoring
|
|
3211
|
-
if (this.touchMonitorInterval) {
|
|
3212
|
-
clearInterval(this.touchMonitorInterval);
|
|
3213
|
-
this.touchMonitorInterval = null;
|
|
3214
|
-
}
|
|
3215
|
-
|
|
3216
|
-
this.logger.info('✅ Touch monitoring stopped');
|
|
3217
|
-
}
|
|
3218
|
-
|
|
3219
|
-
stopCommanderMode() {
|
|
3220
|
-
if (!this.isActive) return { success: false };
|
|
3221
|
-
|
|
3222
|
-
this.logger.info('🛑 Stopping commander mode and cleaning up...');
|
|
3223
|
-
|
|
3224
|
-
this.stopTouchMonitoring();
|
|
3225
|
-
|
|
3226
|
-
// Clear commander device
|
|
3227
|
-
const commanderName = this.commanderDevice?.name;
|
|
3228
|
-
this.commanderDevice = null;
|
|
3229
|
-
|
|
3230
|
-
// Clear all followers
|
|
3231
|
-
this.followerDevices.clear();
|
|
3232
|
-
|
|
3233
|
-
// Reset state
|
|
3234
|
-
this.isActive = false;
|
|
3235
|
-
// Note: We don't reset wdaSessionId anymore - it's managed globally
|
|
3236
|
-
this.cachedElements = [];
|
|
3237
|
-
this.lastCacheTime = 0;
|
|
3238
|
-
this.sessionId = null;
|
|
3239
|
-
|
|
3240
|
-
this.broadcastCommanderStatus();
|
|
3241
|
-
|
|
3242
|
-
this.logger.info(`✅ Commander mode stopped. Removed device: ${commanderName}`);
|
|
3243
|
-
|
|
3244
|
-
return {
|
|
3245
|
-
success: true,
|
|
3246
|
-
removedCommander: commanderName,
|
|
3247
|
-
removedFollowers: 0
|
|
3248
|
-
};
|
|
3249
|
-
}
|
|
3250
|
-
|
|
3251
|
-
async refreshLocators(fullScan = false) {
|
|
3252
|
-
if (!this.isActive) {
|
|
3253
|
-
throw new Error('Commander mode not active');
|
|
3254
|
-
}
|
|
3255
|
-
|
|
3256
|
-
this.logger.info(`🔄 ${fullScan ? 'Full' : 'Fast'} locator refresh requested`);
|
|
3257
|
-
this.cachedElements = [];
|
|
3258
|
-
this.lastCacheTime = 0;
|
|
3259
|
-
|
|
3260
|
-
if (fullScan) {
|
|
3261
|
-
await this.cacheAllElementsFullScan();
|
|
3262
|
-
} else {
|
|
3263
|
-
await this.cacheAllElements();
|
|
3264
|
-
}
|
|
3265
|
-
|
|
3266
|
-
return {
|
|
3267
|
-
success: true,
|
|
3268
|
-
elementsCount: this.cachedElements.length
|
|
3269
|
-
};
|
|
3270
|
-
}
|
|
3271
|
-
|
|
3272
|
-
/**
|
|
3273
|
-
* Background locator refresh - runs in a separate thread after locator is found
|
|
3274
|
-
* This ensures UI elements stay current without blocking the current click operation
|
|
3275
|
-
*/
|
|
3276
|
-
async refreshLocatorsInBackground() {
|
|
3277
|
-
if (!this.isActive || !this.commanderDevice) {
|
|
3278
|
-
this.logger.debug('⚠️ Commander not active, skipping background refresh');
|
|
3279
|
-
return;
|
|
3280
|
-
}
|
|
3281
|
-
|
|
3282
|
-
try {
|
|
3283
|
-
this.logger.info('🔄 Background: Starting automatic locator refresh...');
|
|
3284
|
-
|
|
3285
|
-
// Add a small delay to avoid interfering with the current operation
|
|
3286
|
-
await new Promise(resolve => setTimeout(resolve, 250));
|
|
3287
|
-
|
|
3288
|
-
// Check if we're not in the middle of another caching operation
|
|
3289
|
-
if (this.isCaching) {
|
|
3290
|
-
this.logger.debug('⚠️ Background: Caching already in progress, skipping');
|
|
3291
|
-
return;
|
|
3292
|
-
}
|
|
3293
|
-
|
|
3294
|
-
// Perform a fast refresh (not full scan) to keep it lightweight
|
|
3295
|
-
const previousCount = this.cachedElements.length;
|
|
3296
|
-
await this.fastCacheAllElements();
|
|
3297
|
-
const newCount = this.cachedElements.length;
|
|
3298
|
-
|
|
3299
|
-
this.logger.info(`✅ Background: Locators refreshed (${previousCount} → ${newCount} elements)`);
|
|
3300
|
-
|
|
3301
|
-
// Broadcast refresh completion to UI if there are significant changes
|
|
3302
|
-
const elementCountDiff = Math.abs(newCount - previousCount);
|
|
3303
|
-
if (elementCountDiff > 5) {
|
|
3304
|
-
this.broadcast({
|
|
3305
|
-
type: 'locators_auto_refreshed',
|
|
3306
|
-
message: `Locators auto-refreshed: ${newCount} elements available`,
|
|
3307
|
-
elementsCount: newCount,
|
|
3308
|
-
changeCount: elementCountDiff
|
|
3309
|
-
});
|
|
3310
|
-
this.logger.info(`📡 Background: Notified UI of significant changes (+/- ${elementCountDiff} elements)`);
|
|
3311
|
-
}
|
|
3312
|
-
|
|
3313
|
-
} catch (error) {
|
|
3314
|
-
this.logger.warn(`⚠️ Background: Auto-refresh failed: ${error.message}`);
|
|
3315
|
-
// Don't throw error since this is background operation
|
|
3316
|
-
}
|
|
3317
|
-
}
|
|
3318
|
-
|
|
3319
|
-
/**
|
|
3320
|
-
* Schedule a background refresh - safer approach than setImmediate
|
|
3321
|
-
*/
|
|
3322
|
-
scheduleBackgroundRefresh() {
|
|
3323
|
-
if (this._refreshTimeout) {
|
|
3324
|
-
clearTimeout(this._refreshTimeout);
|
|
3325
|
-
}
|
|
3326
|
-
|
|
3327
|
-
this.logger.info('🔄 Near locator found - scheduling auto-refresh...');
|
|
3328
|
-
|
|
3329
|
-
this._refreshTimeout = setTimeout(() => {
|
|
3330
|
-
this.refreshLocatorsInBackground()
|
|
3331
|
-
.catch(error => {
|
|
3332
|
-
this.logger.warn(`Background locator refresh failed: ${error.message}`);
|
|
3333
|
-
});
|
|
3334
|
-
}, 500); // Wait 500ms to avoid interfering with current click
|
|
3335
|
-
}
|
|
3336
|
-
|
|
3337
|
-
async getAllLocatorsForInspector() {
|
|
3338
|
-
// Use session lock to prevent concurrent access with click operations
|
|
3339
|
-
return this.withSessionLock(this.commanderDevice?.name || 'commander', async () => {
|
|
3340
|
-
return this._getAllLocatorsForInspector();
|
|
3341
|
-
});
|
|
3342
|
-
}
|
|
3343
|
-
|
|
3344
|
-
async _getAllLocatorsForInspector() {
|
|
3345
|
-
// Reset session recovery flag for this call
|
|
3346
|
-
this._sessionRecoveryAttempted = false;
|
|
3347
|
-
|
|
3348
|
-
// Independent locator fetch for inspector - doesn't use cache
|
|
3349
|
-
if (!this.isActive || !this.commanderDevice) {
|
|
3350
|
-
throw new Error('Commander mode not active');
|
|
3351
|
-
}
|
|
3352
|
-
|
|
3353
|
-
let wdaSessionId = this.getWDASessionId();
|
|
3354
|
-
|
|
3355
|
-
// If session ID is not available, try to get it from the status endpoint
|
|
3356
|
-
if (!wdaSessionId && this.commanderDevice.platform === 'ios') {
|
|
3357
|
-
try {
|
|
3358
|
-
const axios = require('axios');
|
|
3359
|
-
const port = this.commanderDevice.port || 8100;
|
|
3360
|
-
const statusRes = await axios.get(`http://localhost:${port}/status`, { timeout: 3000 });
|
|
3361
|
-
|
|
3362
|
-
if (statusRes.data?.sessionId) {
|
|
3363
|
-
wdaSessionId = statusRes.data.sessionId;
|
|
3364
|
-
this.logger.debug(`🔍 Inspector: Retrieved WDA session from /status: ${wdaSessionId}`);
|
|
3365
|
-
|
|
3366
|
-
// Register this session for future use
|
|
3367
|
-
global.sessionManager?.getOrRegisterSession(
|
|
3368
|
-
this.commanderDevice.udid,
|
|
3369
|
-
wdaSessionId,
|
|
3370
|
-
port,
|
|
3371
|
-
this.commanderDevice.name,
|
|
3372
|
-
null,
|
|
3373
|
-
'ios'
|
|
3374
|
-
);
|
|
3375
|
-
}
|
|
3376
|
-
} catch (err) {
|
|
3377
|
-
this.logger.debug(`Could not get WDA session from /status: ${err.message}`);
|
|
3378
|
-
}
|
|
3379
|
-
}
|
|
3380
|
-
|
|
3381
|
-
// If still no session ID, we'll try to proceed anyway - the endpoint may not require it
|
|
3382
|
-
if (!wdaSessionId && this.commanderDevice.platform === 'ios') {
|
|
3383
|
-
this.logger.warn('⚠️ No WDA session ID available, will attempt without it');
|
|
3384
|
-
}
|
|
3385
|
-
|
|
3386
|
-
const startTime = Date.now();
|
|
3387
|
-
this.logger.info('🔍 Inspector: Starting independent locator fetch...');
|
|
3388
|
-
|
|
3389
|
-
try {
|
|
3390
|
-
const axios = require('axios');
|
|
3391
|
-
const device = this.commanderDevice;
|
|
3392
|
-
const port = device.port || 8100;
|
|
3393
|
-
const sessionId = wdaSessionId;
|
|
3394
|
-
|
|
3395
|
-
// Try page source method first (fastest)
|
|
3396
|
-
try {
|
|
3397
|
-
// Skip page source method if no session ID - it won't work
|
|
3398
|
-
if (!sessionId) {
|
|
3399
|
-
this.logger.debug('Skipping page source method (no session ID), will try element finding');
|
|
3400
|
-
throw new Error('No session ID available');
|
|
3401
|
-
}
|
|
3402
|
-
|
|
3403
|
-
const sourceRes = await axios.get(
|
|
3404
|
-
`http://localhost:${port}/session/${sessionId}/source`,
|
|
3405
|
-
{ timeout: 8000 }
|
|
3406
|
-
);
|
|
3407
|
-
|
|
3408
|
-
if (sourceRes.data?.value) {
|
|
3409
|
-
const pageSource = sourceRes.data.value;
|
|
3410
|
-
const allElements = this.parsePageSourceFast(pageSource);
|
|
3411
|
-
|
|
3412
|
-
if (allElements.length > 0) {
|
|
3413
|
-
const elapsed = Date.now() - startTime;
|
|
3414
|
-
this.logger.info(`✅ Inspector: Loaded ${allElements.length} elements in ${elapsed}ms (page source)`);
|
|
3415
|
-
|
|
3416
|
-
return {
|
|
3417
|
-
success: true,
|
|
3418
|
-
elements: allElements
|
|
3419
|
-
.filter(e => {
|
|
3420
|
-
// Filter out overly large background elements that cover entire screen
|
|
3421
|
-
const isEntireScreen = (e.width >= 350 && e.height >= 700); // Most of screen
|
|
3422
|
-
const isContainer = e.type?.includes('View') || e.type?.includes('ScrollView') || e.type?.includes('CollectionView') || e.type?.includes('TableView');
|
|
3423
|
-
const hasNoText = !e.label && !e.name && !e.value;
|
|
3424
|
-
|
|
3425
|
-
// Exclude large containers without useful labels
|
|
3426
|
-
if (isEntireScreen && isContainer && hasNoText) {
|
|
3427
|
-
return false;
|
|
3428
|
-
}
|
|
3429
|
-
|
|
3430
|
-
// Exclude elements smaller than 10x10 pixels (too small to click)
|
|
3431
|
-
if (e.width < 10 || e.height < 10) {
|
|
3432
|
-
return false;
|
|
3433
|
-
}
|
|
3434
|
-
|
|
3435
|
-
return true;
|
|
3436
|
-
})
|
|
3437
|
-
.map(e => ({
|
|
3438
|
-
label: e.label || e.name,
|
|
3439
|
-
type: e.type.replace('XCUIElementType', ''),
|
|
3440
|
-
x: e.x,
|
|
3441
|
-
y: e.y,
|
|
3442
|
-
width: e.width,
|
|
3443
|
-
height: e.height,
|
|
3444
|
-
centerX: e.centerX,
|
|
3445
|
-
centerY: e.centerY
|
|
3446
|
-
})),
|
|
3447
|
-
method: 'page_source'
|
|
3448
|
-
};
|
|
3449
|
-
}
|
|
3450
|
-
}
|
|
3451
|
-
} catch (err) {
|
|
3452
|
-
// If no session ID or 404 error, session is invalid - try to recover once
|
|
3453
|
-
if ((err.message === 'No session ID available' || (err.response && err.response.status === 404)) && !this._sessionRecoveryAttempted) {
|
|
3454
|
-
this.logger.warn(`⚠️ Inspector: Page source failed (${err.message || 'status ' + err.response.status}), attempting recovery...`);
|
|
3455
|
-
this._sessionRecoveryAttempted = true;
|
|
3456
|
-
|
|
3457
|
-
try {
|
|
3458
|
-
// Use smart recovery instead of full recreation
|
|
3459
|
-
const newSessionId = await this.recoverOrCreateSession(this.commanderDevice, wdaSessionId);
|
|
3460
|
-
|
|
3461
|
-
if (newSessionId && newSessionId !== sessionId) {
|
|
3462
|
-
this.logger.info(`✅ Inspector: Session recovered, retrying with ${newSessionId.substring(0, 8)}...`);
|
|
3463
|
-
|
|
3464
|
-
// Retry page source with new session
|
|
3465
|
-
const retryRes = await axios.get(
|
|
3466
|
-
`http://localhost:${port}/session/${newSessionId}/source`,
|
|
3467
|
-
{ timeout: 8000 }
|
|
3468
|
-
);
|
|
3469
|
-
|
|
3470
|
-
if (retryRes.data?.value) {
|
|
3471
|
-
const pageSource = retryRes.data.value;
|
|
3472
|
-
const allElements = this.parsePageSourceFast(pageSource);
|
|
3473
|
-
|
|
3474
|
-
if (allElements.length > 0) {
|
|
3475
|
-
const elapsed = Date.now() - startTime;
|
|
3476
|
-
this.logger.info(`✅ Inspector: Recovered ${allElements.length} elements in ${elapsed}ms (page source after recovery)`);
|
|
3477
|
-
|
|
3478
|
-
return {
|
|
3479
|
-
success: true,
|
|
3480
|
-
elements: allElements.map(e => ({
|
|
3481
|
-
label: e.label || e.name,
|
|
3482
|
-
type: e.type.replace('XCUIElementType', ''),
|
|
3483
|
-
x: e.x,
|
|
3484
|
-
y: e.y,
|
|
3485
|
-
width: e.width,
|
|
3486
|
-
height: e.height,
|
|
3487
|
-
centerX: e.centerX,
|
|
3488
|
-
centerY: e.centerY
|
|
3489
|
-
})),
|
|
3490
|
-
method: 'page_source_recovered'
|
|
3491
|
-
};
|
|
3492
|
-
}
|
|
3493
|
-
}
|
|
3494
|
-
}
|
|
3495
|
-
} catch (recoveryErr) {
|
|
3496
|
-
this.logger.error(`❌ Inspector: Page source session recovery failed: ${recoveryErr.message}`);
|
|
3497
|
-
}
|
|
3498
|
-
}
|
|
3499
|
-
|
|
3500
|
-
this.logger.warn('Inspector: Page source failed, using element query');
|
|
3501
|
-
}
|
|
3502
|
-
|
|
3503
|
-
// Fallback: Full element query (comprehensive)
|
|
3504
|
-
const queries = [
|
|
3505
|
-
{ using: 'class name', value: 'XCUIElementTypeButton' },
|
|
3506
|
-
{ using: 'class name', value: 'XCUIElementTypeStaticText' },
|
|
3507
|
-
{ using: 'class name', value: 'XCUIElementTypeTextField' },
|
|
3508
|
-
{ using: 'class name', value: 'XCUIElementTypeCell' },
|
|
3509
|
-
{ using: 'class name', value: 'XCUIElementTypeImage' },
|
|
3510
|
-
{ using: 'class name', value: 'XCUIElementTypeSwitch' },
|
|
3511
|
-
{ using: 'class name', value: 'XCUIElementTypeLink' }
|
|
3512
|
-
];
|
|
3513
|
-
|
|
3514
|
-
const allElements = [];
|
|
3515
|
-
const MAX_ELEMENTS = 50;
|
|
3516
|
-
|
|
3517
|
-
for (const query of queries) {
|
|
3518
|
-
try {
|
|
3519
|
-
const elementsRes = await axios.post(
|
|
3520
|
-
`http://localhost:${port}/session/${sessionId}/elements`,
|
|
3521
|
-
query,
|
|
3522
|
-
{ timeout: 5000 }
|
|
3523
|
-
);
|
|
3524
|
-
|
|
3525
|
-
if (elementsRes.data?.value && Array.isArray(elementsRes.data.value)) {
|
|
3526
|
-
const elements = elementsRes.data.value.slice(0, MAX_ELEMENTS);
|
|
3527
|
-
|
|
3528
|
-
const chunkSize = 15;
|
|
3529
|
-
for (let i = 0; i < elements.length; i += chunkSize) {
|
|
3530
|
-
const chunk = elements.slice(i, i + chunkSize);
|
|
3531
|
-
const chunkPromises = chunk.map(async (elem) => {
|
|
3532
|
-
const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
|
|
3533
|
-
if (!elementId) return null;
|
|
3534
|
-
|
|
3535
|
-
try {
|
|
3536
|
-
const [rectRes, labelRes] = await Promise.all([
|
|
3537
|
-
axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/rect`, { timeout: 1000 }),
|
|
3538
|
-
axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/attribute/label`, { timeout: 1000 }).catch(() => ({ data: { value: null } }))
|
|
3539
|
-
]);
|
|
3540
|
-
|
|
3541
|
-
if (rectRes.data?.value) {
|
|
3542
|
-
const rect = rectRes.data.value;
|
|
3543
|
-
const label = labelRes.data?.value;
|
|
3544
|
-
|
|
3545
|
-
if (rect.width > 3 && rect.height > 3) {
|
|
3546
|
-
return {
|
|
3547
|
-
type: query.value,
|
|
3548
|
-
label: label || 'Unlabeled',
|
|
3549
|
-
name: label || 'Unlabeled',
|
|
3550
|
-
x: Math.round(rect.x),
|
|
3551
|
-
y: Math.round(rect.y),
|
|
3552
|
-
width: Math.round(rect.width),
|
|
3553
|
-
height: Math.round(rect.height),
|
|
3554
|
-
centerX: Math.round(rect.x + rect.width / 2),
|
|
3555
|
-
centerY: Math.round(rect.y + rect.height / 2)
|
|
3556
|
-
};
|
|
3557
|
-
}
|
|
3558
|
-
}
|
|
3559
|
-
} catch (err) { return null; }
|
|
3560
|
-
return null;
|
|
3561
|
-
});
|
|
3562
|
-
|
|
3563
|
-
const chunkResults = await Promise.all(chunkPromises);
|
|
3564
|
-
const validElements = chunkResults.filter(e => e !== null);
|
|
3565
|
-
allElements.push(...validElements);
|
|
3566
|
-
}
|
|
3567
|
-
}
|
|
3568
|
-
} catch (err) {
|
|
3569
|
-
// If 404 error, session is invalid - try to recover once
|
|
3570
|
-
if (err.response && err.response.status === 404 && !this._sessionRecoveryAttempted) {
|
|
3571
|
-
this.logger.warn(`⚠️ Inspector: Session expired (404), attempting recovery...`);
|
|
3572
|
-
this._sessionRecoveryAttempted = true;
|
|
3573
|
-
|
|
3574
|
-
try {
|
|
3575
|
-
// Use smart recovery instead of full recreation
|
|
3576
|
-
const newSessionId = await this.recoverOrCreateSession(this.commanderDevice, sessionId);
|
|
3577
|
-
|
|
3578
|
-
if (newSessionId && newSessionId !== sessionId) {
|
|
3579
|
-
this.logger.info(`✅ Inspector: Session recovered, retrying with ${newSessionId.substring(0, 8)}...`);
|
|
3580
|
-
|
|
3581
|
-
// Retry the query with new session
|
|
3582
|
-
const retryRes = await axios.post(
|
|
3583
|
-
`http://localhost:${port}/session/${newSessionId}/elements`,
|
|
3584
|
-
query,
|
|
3585
|
-
{ timeout: 5000 }
|
|
3586
|
-
);
|
|
3587
|
-
|
|
3588
|
-
if (retryRes.data?.value && Array.isArray(retryRes.data.value)) {
|
|
3589
|
-
// Process the retry results - simplified version
|
|
3590
|
-
const elements = retryRes.data.value.slice(0, MAX_ELEMENTS);
|
|
3591
|
-
for (const elem of elements.slice(0, 10)) { // Limit for quick recovery
|
|
3592
|
-
const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
|
|
3593
|
-
if (elementId) {
|
|
3594
|
-
try {
|
|
3595
|
-
const rectRes = await axios.get(`http://localhost:${port}/session/${newSessionId}/element/${elementId}/rect`, { timeout: 1000 });
|
|
3596
|
-
if (rectRes.data?.value && rectRes.data.value.width > 3) {
|
|
3597
|
-
const rect = rectRes.data.value;
|
|
3598
|
-
allElements.push({
|
|
3599
|
-
type: query.value,
|
|
3600
|
-
label: 'Recovered Element',
|
|
3601
|
-
name: 'Recovered Element',
|
|
3602
|
-
x: Math.round(rect.x),
|
|
3603
|
-
y: Math.round(rect.y),
|
|
3604
|
-
width: Math.round(rect.width),
|
|
3605
|
-
height: Math.round(rect.height),
|
|
3606
|
-
centerX: Math.round(rect.x + rect.width / 2),
|
|
3607
|
-
centerY: Math.round(rect.y + rect.height / 2)
|
|
3608
|
-
});
|
|
3609
|
-
}
|
|
3610
|
-
} catch (e) { /* ignore individual element errors during recovery */ }
|
|
3611
|
-
}
|
|
3612
|
-
}
|
|
3613
|
-
this.logger.info(`✅ Inspector: Recovered ${allElements.length} elements after session recovery`);
|
|
3614
|
-
continue; // Skip the normal error logging since we recovered
|
|
3615
|
-
}
|
|
3616
|
-
}
|
|
3617
|
-
} catch (recoveryErr) {
|
|
3618
|
-
this.logger.error(`❌ Inspector: Session recovery failed: ${recoveryErr.message}`);
|
|
3619
|
-
}
|
|
3620
|
-
}
|
|
3621
|
-
|
|
3622
|
-
this.logger.warn(`Inspector query ${query.value} failed: ${err.message}`);
|
|
3623
|
-
}
|
|
3624
|
-
}
|
|
3625
|
-
|
|
3626
|
-
const elapsed = Date.now() - startTime;
|
|
3627
|
-
this.logger.info(`✅ Inspector: Loaded ${allElements.length} elements in ${elapsed}ms (element query)`);
|
|
3628
|
-
|
|
3629
|
-
return {
|
|
3630
|
-
success: true,
|
|
3631
|
-
elements: allElements.map(e => ({
|
|
3632
|
-
label: e.label || e.name,
|
|
3633
|
-
type: e.type.replace('XCUIElementType', ''),
|
|
3634
|
-
x: e.x,
|
|
3635
|
-
y: e.y,
|
|
3636
|
-
width: e.width,
|
|
3637
|
-
height: e.height,
|
|
3638
|
-
centerX: e.centerX,
|
|
3639
|
-
centerY: e.centerY
|
|
3640
|
-
})),
|
|
3641
|
-
method: 'element_query'
|
|
3642
|
-
};
|
|
3643
|
-
|
|
3644
|
-
} catch (error) {
|
|
3645
|
-
this.logger.error(`Inspector locator fetch failed: ${error.message}`);
|
|
3646
|
-
throw error;
|
|
3647
|
-
}
|
|
3648
|
-
}
|
|
3649
|
-
|
|
3650
|
-
async cacheAllElementsFullScan() {
|
|
3651
|
-
// Full comprehensive scan for manual refresh
|
|
3652
|
-
const wdaSessionId = await this.getWDASessionId();
|
|
3653
|
-
if (!wdaSessionId || !this.commanderDevice) return;
|
|
3654
|
-
if (this.isCaching) return;
|
|
3655
|
-
|
|
3656
|
-
this.isCaching = true;
|
|
3657
|
-
const startTime = Date.now();
|
|
3658
|
-
|
|
3659
|
-
try {
|
|
3660
|
-
const axios = require('axios');
|
|
3661
|
-
const device = this.commanderDevice;
|
|
3662
|
-
const port = device.port || 8100;
|
|
3663
|
-
const sessionId = wdaSessionId;
|
|
3664
|
-
|
|
3665
|
-
this.logger.info('🔍 Starting FULL locator scan...');
|
|
3666
|
-
|
|
3667
|
-
// Fast and accurate queries - back to original working set
|
|
3668
|
-
const queries = [
|
|
3669
|
-
{ using: 'class name', value: 'XCUIElementTypeButton' },
|
|
3670
|
-
{ using: 'class name', value: 'XCUIElementTypeTextField' },
|
|
3671
|
-
{ using: 'class name', value: 'XCUIElementTypeStaticText' }
|
|
3672
|
-
];
|
|
3673
|
-
|
|
3674
|
-
const allElements = [];
|
|
3675
|
-
const MAX_ELEMENTS = 100; // Full scan - more elements
|
|
3676
|
-
|
|
3677
|
-
for (const query of queries) {
|
|
3678
|
-
try {
|
|
3679
|
-
const elementsRes = await axios.post(
|
|
3680
|
-
`http://localhost:${port}/session/${sessionId}/elements`,
|
|
3681
|
-
query,
|
|
3682
|
-
{ timeout: 3000 }
|
|
3683
|
-
);
|
|
3684
|
-
|
|
3685
|
-
if (elementsRes.data?.value && Array.isArray(elementsRes.data.value)) {
|
|
3686
|
-
const elements = elementsRes.data.value.slice(0, MAX_ELEMENTS);
|
|
3687
|
-
|
|
3688
|
-
const chunkSize = 15;
|
|
3689
|
-
for (let i = 0; i < elements.length; i += chunkSize) {
|
|
3690
|
-
const chunk = elements.slice(i, i + chunkSize);
|
|
3691
|
-
const chunkPromises = chunk.map(async (elem) => {
|
|
3692
|
-
const elementId = elem.ELEMENT || elem['element-6066-11e4-a52e-4f735466cecf'];
|
|
3693
|
-
if (!elementId) return null;
|
|
3694
|
-
|
|
3695
|
-
try {
|
|
3696
|
-
const [rectRes, labelRes] = await Promise.all([
|
|
3697
|
-
axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/rect`, { timeout: 1500 }),
|
|
3698
|
-
axios.get(`http://localhost:${port}/session/${sessionId}/element/${elementId}/attribute/label`, { timeout: 1500 }).catch(() => ({ data: { value: null } }))
|
|
3699
|
-
]);
|
|
3700
|
-
|
|
3701
|
-
if (rectRes.data?.value) {
|
|
3702
|
-
const rect = rectRes.data.value;
|
|
3703
|
-
const label = labelRes.data?.value;
|
|
3704
|
-
|
|
3705
|
-
// Simple fast filtering - back to original working logic
|
|
3706
|
-
if (rect.width > 3 && rect.height > 3) {
|
|
3707
|
-
return {
|
|
3708
|
-
type: query.value,
|
|
3709
|
-
label: label || 'Unlabeled',
|
|
3710
|
-
name: label || 'Unlabeled',
|
|
3711
|
-
accessibilityId: label,
|
|
3712
|
-
x: Math.round(rect.x),
|
|
3713
|
-
y: Math.round(rect.y),
|
|
3714
|
-
width: Math.round(rect.width),
|
|
3715
|
-
height: Math.round(rect.height),
|
|
3716
|
-
centerX: Math.round(rect.x + rect.width / 2),
|
|
3717
|
-
centerY: Math.round(rect.y + rect.height / 2)
|
|
3718
|
-
};
|
|
3719
|
-
}
|
|
3720
|
-
}
|
|
3721
|
-
} catch (err) { return null; }
|
|
3722
|
-
return null;
|
|
3723
|
-
});
|
|
3724
|
-
|
|
3725
|
-
const chunkResults = await Promise.all(chunkPromises);
|
|
3726
|
-
const validElements = chunkResults.filter(e => e !== null);
|
|
3727
|
-
|
|
3728
|
-
if (validElements.length > 0) {
|
|
3729
|
-
allElements.push(...validElements);
|
|
3730
|
-
|
|
3731
|
-
// Send progress updates
|
|
3732
|
-
this.broadcast({
|
|
3733
|
-
type: 'cached_elements',
|
|
3734
|
-
elements: validElements.map(e => ({
|
|
3735
|
-
label: e.label || e.name,
|
|
3736
|
-
type: e.type.replace('XCUIElementType', ''),
|
|
3737
|
-
x: e.x,
|
|
3738
|
-
y: e.y,
|
|
3739
|
-
width: e.width,
|
|
3740
|
-
height: e.height
|
|
3741
|
-
})),
|
|
3742
|
-
total: allElements.length,
|
|
3743
|
-
timestamp: Date.now()
|
|
3744
|
-
});
|
|
3745
|
-
}
|
|
3746
|
-
}
|
|
3747
|
-
}
|
|
3748
|
-
} catch (err) {
|
|
3749
|
-
this.logger.warn(`Full scan query ${query.value} failed: ${err.message}`);
|
|
3750
|
-
}
|
|
3751
|
-
}
|
|
3752
|
-
|
|
3753
|
-
this.cachedElements = allElements;
|
|
3754
|
-
this.lastCacheTime = Date.now();
|
|
3755
|
-
|
|
3756
|
-
const elapsed = Date.now() - startTime;
|
|
3757
|
-
this.logger.info(`✅ Full scan completed: ${allElements.length} elements in ${elapsed}ms`);
|
|
3758
|
-
|
|
3759
|
-
} finally {
|
|
3760
|
-
this.isCaching = false;
|
|
3761
|
-
}
|
|
3762
|
-
}
|
|
3763
|
-
|
|
3764
|
-
broadcastCommanderStatus() {
|
|
3765
|
-
// CRITICAL FIX: Proper null safety checks
|
|
3766
|
-
const commanderName = this.isActive && this.commanderDevice?.name ? this.commanderDevice.name : null;
|
|
3767
|
-
|
|
3768
|
-
// Debug logging to verify device name casing
|
|
3769
|
-
if (commanderName) {
|
|
3770
|
-
this.logger.info(`📡 Broadcasting commander status for device: "${commanderName}"`);
|
|
3771
|
-
}
|
|
3772
|
-
|
|
3773
|
-
const status = {
|
|
3774
|
-
type: 'commander_status',
|
|
3775
|
-
isActive: this.isActive,
|
|
3776
|
-
commander: commanderName,
|
|
3777
|
-
followers: Array.from(this.followerDevices || []).map(f => ({
|
|
3778
|
-
deviceId: f?.name || 'unknown',
|
|
3779
|
-
name: f?.name || 'unknown',
|
|
3780
|
-
platform: f?.platform || 'unknown',
|
|
3781
|
-
port: f?.port || 0
|
|
3782
|
-
})),
|
|
3783
|
-
followerCount: this.followerDevices ? this.followerDevices.size : 0,
|
|
3784
|
-
sessionId: this.sessionId
|
|
3785
|
-
};
|
|
3786
|
-
this.broadcast(status);
|
|
3787
|
-
}
|
|
3788
|
-
|
|
3789
|
-
broadcast(message) {
|
|
3790
|
-
if (this.wss?.clients) {
|
|
3791
|
-
this.wss.clients.forEach(client => {
|
|
3792
|
-
if (client.readyState === 1) client.send(JSON.stringify(message));
|
|
3793
|
-
});
|
|
3794
|
-
}
|
|
3795
|
-
}
|
|
3796
|
-
|
|
3797
|
-
async executeAppAction(appAction, appId) {
|
|
3798
|
-
const results = [];
|
|
3799
|
-
|
|
3800
|
-
// Execute on commander device first
|
|
3801
|
-
if (this.commanderDevice) {
|
|
3802
|
-
try {
|
|
3803
|
-
if (this.commanderDevice.platform === 'ios') {
|
|
3804
|
-
// iOS commander app control via WDA
|
|
3805
|
-
const session = await this.getWDASessionId();
|
|
3806
|
-
if (session) {
|
|
3807
|
-
const wdaUrl = `http://localhost:${this.commanderDevice.port || 8100}/session/${session}`;
|
|
3808
|
-
const axios = require('axios');
|
|
3809
|
-
|
|
3810
|
-
switch (appAction) {
|
|
3811
|
-
case 'openApp':
|
|
3812
|
-
case 'launchApp':
|
|
3813
|
-
await axios.post(`${wdaUrl}/wda/apps/launch`, { bundleId: appId });
|
|
3814
|
-
break;
|
|
3815
|
-
case 'closeApp':
|
|
3816
|
-
case 'terminateApp':
|
|
3817
|
-
await axios.post(`${wdaUrl}/wda/apps/terminate`, { bundleId: appId });
|
|
3818
|
-
break;
|
|
3819
|
-
case 'activateApp':
|
|
3820
|
-
await axios.post(`${wdaUrl}/wda/apps/activate`, { bundleId: appId });
|
|
3821
|
-
break;
|
|
3822
|
-
}
|
|
3823
|
-
results.push({ device: this.commanderDevice.name, success: true });
|
|
3824
|
-
this.logger.info(`✅ App action ${appAction} executed on commander: ${this.commanderDevice.name}`);
|
|
3825
|
-
}
|
|
3826
|
-
} else if (this.commanderDevice.platform === 'android') {
|
|
3827
|
-
// Android commander app control via UIAutomator2
|
|
3828
|
-
const session = sessionManager.getByName(this.commanderDevice.name);
|
|
3829
|
-
if (session?.sessionId && session?.port) {
|
|
3830
|
-
const axios = require('axios');
|
|
3831
|
-
const baseUrl = `http://localhost:${session.port}/session/${session.sessionId}`;
|
|
3832
|
-
|
|
3833
|
-
switch (appAction) {
|
|
3834
|
-
case 'openApp':
|
|
3835
|
-
case 'launchApp':
|
|
3836
|
-
await axios.post(`${baseUrl}/appium/device/activate_app`, {
|
|
3837
|
-
json: { appId: appId }
|
|
3838
|
-
});
|
|
3839
|
-
break;
|
|
3840
|
-
case 'closeApp':
|
|
3841
|
-
case 'terminateApp':
|
|
3842
|
-
await axios.post(`${baseUrl}/appium/device/terminate_app`, {
|
|
3843
|
-
json: { appId: appId }
|
|
3844
|
-
});
|
|
3845
|
-
break;
|
|
3846
|
-
}
|
|
3847
|
-
results.push({ device: this.commanderDevice.name, success: true });
|
|
3848
|
-
this.logger.info(`✅ App action ${appAction} executed on commander: ${this.commanderDevice.name}`);
|
|
3849
|
-
}
|
|
3850
|
-
}
|
|
3851
|
-
} catch (error) {
|
|
3852
|
-
this.logger.error(`❌ App action ${appAction} failed on commander: ${error.message}`);
|
|
3853
|
-
results.push({ device: this.commanderDevice.name, success: false, error: error.message });
|
|
3854
|
-
}
|
|
3855
|
-
}
|
|
3856
|
-
|
|
3857
|
-
// Broadcast to followers
|
|
3858
|
-
if (this.followerDevices.size > 0) {
|
|
3859
|
-
const broadcastResult = await this.broadcastCommand({
|
|
3860
|
-
action: 'app-control',
|
|
3861
|
-
appAction: appAction,
|
|
3862
|
-
appId: appId
|
|
3863
|
-
});
|
|
3864
|
-
results.push(...broadcastResult.results || []);
|
|
3865
|
-
}
|
|
3866
|
-
|
|
3867
|
-
return {
|
|
3868
|
-
success: true,
|
|
3869
|
-
results: results,
|
|
3870
|
-
executedOn: results.length
|
|
3871
|
-
};
|
|
3872
|
-
}
|
|
3873
|
-
|
|
3874
|
-
async broadcastCommand(commandData) {
|
|
3875
|
-
if (!this.isActive || this.followerDevices.size === 0) {
|
|
3876
|
-
this.logger.info('No followers to broadcast to');
|
|
3877
|
-
return { success: true, broadcastCount: 0 };
|
|
3878
|
-
}
|
|
3879
|
-
|
|
3880
|
-
this.logger.info(`📢 Broadcasting command to ${this.followerDevices.size} followers`);
|
|
3881
|
-
|
|
3882
|
-
const results = [];
|
|
3883
|
-
|
|
3884
|
-
for (const follower of this.followerDevices) {
|
|
3885
|
-
try {
|
|
3886
|
-
if (commandData.action === 'app-control') {
|
|
3887
|
-
// Handle app control commands for followers
|
|
3888
|
-
const { appAction, appId } = commandData;
|
|
3889
|
-
|
|
3890
|
-
if (follower.platform === 'ios') {
|
|
3891
|
-
// iOS follower app control via WDA
|
|
3892
|
-
const session = this.checkFollowerWDASession(follower);
|
|
3893
|
-
if (session?.sessionId) {
|
|
3894
|
-
const wdaUrl = `http://localhost:${follower.port || 8100}/session/${session.sessionId}`;
|
|
3895
|
-
const axios = require('axios');
|
|
3896
|
-
|
|
3897
|
-
switch (appAction) {
|
|
3898
|
-
case 'openApp':
|
|
3899
|
-
case 'launchApp':
|
|
3900
|
-
await axios.post(`${wdaUrl}/wda/apps/launch`, { bundleId: appId });
|
|
3901
|
-
break;
|
|
3902
|
-
case 'closeApp':
|
|
3903
|
-
case 'terminateApp':
|
|
3904
|
-
await axios.post(`${wdaUrl}/wda/apps/terminate`, { bundleId: appId });
|
|
3905
|
-
break;
|
|
3906
|
-
case 'activateApp':
|
|
3907
|
-
await axios.post(`${wdaUrl}/wda/apps/activate`, { bundleId: appId });
|
|
3908
|
-
break;
|
|
3909
|
-
}
|
|
3910
|
-
}
|
|
3911
|
-
} else if (follower.platform === 'android') {
|
|
3912
|
-
// Android follower app control via UIAutomator2
|
|
3913
|
-
const session = await this.getAndroidSession(follower);
|
|
3914
|
-
if (session?.sessionId && session?.port) {
|
|
3915
|
-
const axios = require('axios');
|
|
3916
|
-
const baseUrl = `http://localhost:${session.port}/session/${session.sessionId}`;
|
|
3917
|
-
|
|
3918
|
-
switch (appAction) {
|
|
3919
|
-
case 'openApp':
|
|
3920
|
-
case 'launchApp':
|
|
3921
|
-
await axios.post(`${baseUrl}/appium/device/activate_app`, {
|
|
3922
|
-
json: { appId: appId }
|
|
3923
|
-
});
|
|
3924
|
-
break;
|
|
3925
|
-
case 'closeApp':
|
|
3926
|
-
case 'terminateApp':
|
|
3927
|
-
await axios.post(`${baseUrl}/appium/device/terminate_app`, {
|
|
3928
|
-
json: { appId: appId }
|
|
3929
|
-
});
|
|
3930
|
-
break;
|
|
3931
|
-
}
|
|
3932
|
-
}
|
|
3933
|
-
}
|
|
3934
|
-
|
|
3935
|
-
results.push({ device: follower.name, success: true });
|
|
3936
|
-
this.logger.info(`✅ App control broadcast to ${follower.name}: ${appAction} ${appId}`);
|
|
3937
|
-
} else {
|
|
3938
|
-
// Handle general commands (home, swipe, etc.) by using shell scripts
|
|
3939
|
-
const command = commandData.action;
|
|
3940
|
-
this.logger.info(`📢 Broadcasting command '${command}' to follower ${follower.name} via shell script`);
|
|
3941
|
-
|
|
3942
|
-
// Use the existing executeCommand function that calls shell scripts
|
|
3943
|
-
// This is the same system that /api/execute uses
|
|
3944
|
-
const { spawn } = require('child_process');
|
|
3945
|
-
const executeCommand = require('util').promisify(require('child_process').exec);
|
|
3946
|
-
|
|
3947
|
-
try {
|
|
3948
|
-
// Get the correct script path based on platform and connection type
|
|
3949
|
-
let scriptPath;
|
|
3950
|
-
if (follower.platform === 'ios') {
|
|
3951
|
-
scriptPath = follower.connectionType === 'wireless'
|
|
3952
|
-
? './connect_ios_wireless_multi_final.sh'
|
|
3953
|
-
: './connect_ios_usb_multi_final.sh';
|
|
3954
|
-
} else if (follower.platform === 'android') {
|
|
3955
|
-
scriptPath = follower.connectionType === 'wireless'
|
|
3956
|
-
? './connect_android_wireless_multi_final.sh'
|
|
3957
|
-
: './connect_android_usb_multi_final.sh';
|
|
3958
|
-
}
|
|
3959
|
-
|
|
3960
|
-
if (scriptPath) {
|
|
3961
|
-
// Execute command on follower using the same shell script system
|
|
3962
|
-
const result = await executeCommand(`${scriptPath} "${follower.name}" "${command}"`);
|
|
3963
|
-
this.logger.info(`✅ Command '${command}' broadcast to follower ${follower.name}`);
|
|
3964
|
-
results.push({ device: follower.name, success: true });
|
|
3965
|
-
} else {
|
|
3966
|
-
throw new Error(`No script found for platform: ${follower.platform}`);
|
|
3967
|
-
}
|
|
3968
|
-
} catch (error) {
|
|
3969
|
-
this.logger.error(`❌ Failed to broadcast command '${command}' to ${follower.name}: ${error.message}`);
|
|
3970
|
-
results.push({ device: follower.name, success: false, error: error.message });
|
|
3971
|
-
}
|
|
3972
|
-
}
|
|
3973
|
-
} catch (error) {
|
|
3974
|
-
this.logger.error(`❌ Failed to broadcast to ${follower.name}: ${error.message}`);
|
|
3975
|
-
results.push({ device: follower.name, success: false, error: error.message });
|
|
3976
|
-
}
|
|
3977
|
-
}
|
|
3978
|
-
|
|
3979
|
-
const successCount = results.filter(r => r.success).length;
|
|
3980
|
-
this.logger.info(`📢 Broadcast complete: ${successCount}/${results.length} followers updated`);
|
|
3981
|
-
|
|
3982
|
-
return {
|
|
3983
|
-
success: true,
|
|
3984
|
-
broadcastCount: successCount,
|
|
3985
|
-
results
|
|
3986
|
-
};
|
|
3987
|
-
}
|
|
3988
|
-
|
|
3989
|
-
/**
|
|
3990
|
-
* GENIUS: Get specific device window size for coordinate scaling with caching
|
|
3991
|
-
*/
|
|
3992
|
-
async getDeviceWindowSizeForScaling(deviceName, port) {
|
|
3993
|
-
// Check cache first
|
|
3994
|
-
const cacheKey = `${deviceName}:${port}`;
|
|
3995
|
-
const cached = this.deviceSizeCache.get(cacheKey);
|
|
3996
|
-
if (cached && (Date.now() - cached.timestamp) < 30000) { // Cache for 30 seconds
|
|
3997
|
-
this.logger.debug(`📦 Using cached size for ${deviceName}: ${cached.size.width}×${cached.size.height}`);
|
|
3998
|
-
return cached.size;
|
|
3999
|
-
}
|
|
4000
|
-
|
|
4001
|
-
try {
|
|
4002
|
-
const session = global.sessionManager?.getByName(deviceName);
|
|
4003
|
-
if (!session?.sessionId) {
|
|
4004
|
-
this.logger.warn(`⚠️ No session found for device: ${deviceName}`);
|
|
4005
|
-
return null;
|
|
4006
|
-
}
|
|
4007
|
-
|
|
4008
|
-
const response = await axios.get(`http://localhost:${port}/session/${session.sessionId}/window/size`, {
|
|
4009
|
-
timeout: 3000
|
|
4010
|
-
});
|
|
4011
|
-
|
|
4012
|
-
if (response.data?.value) {
|
|
4013
|
-
const size = response.data.value;
|
|
4014
|
-
this.logger.debug(`📱 Device ${deviceName} window size: ${size.width}×${size.height}`);
|
|
4015
|
-
|
|
4016
|
-
// Cache the result
|
|
4017
|
-
this.deviceSizeCache.set(cacheKey, {
|
|
4018
|
-
size: size,
|
|
4019
|
-
timestamp: Date.now()
|
|
4020
|
-
});
|
|
4021
|
-
|
|
4022
|
-
return size;
|
|
4023
|
-
}
|
|
4024
|
-
} catch (error) {
|
|
4025
|
-
this.logger.warn(`⚠️ Failed to get window size for ${deviceName}: ${error.message}`);
|
|
4026
|
-
}
|
|
4027
|
-
return null;
|
|
4028
|
-
}
|
|
4029
|
-
|
|
4030
|
-
/**
|
|
4031
|
-
* GENIUS: Scale coordinates from commander to follower device
|
|
4032
|
-
*/
|
|
4033
|
-
scaleCoordinatesForDevice(commanderCoords, commanderSize, followerSize) {
|
|
4034
|
-
if (!commanderSize || !followerSize) {
|
|
4035
|
-
this.logger.warn(`⚠️ Missing device sizes for coordinate scaling`);
|
|
4036
|
-
return commanderCoords; // Fallback to original coordinates
|
|
4037
|
-
}
|
|
4038
|
-
|
|
4039
|
-
// Prevent division by zero
|
|
4040
|
-
if (commanderSize.width === 0 || commanderSize.height === 0) {
|
|
4041
|
-
this.logger.warn(`⚠️ Invalid commander size: ${commanderSize.width}×${commanderSize.height}`);
|
|
4042
|
-
return commanderCoords;
|
|
4043
|
-
}
|
|
4044
|
-
|
|
4045
|
-
const scaleX = followerSize.width / commanderSize.width;
|
|
4046
|
-
const scaleY = followerSize.height / commanderSize.height;
|
|
4047
|
-
|
|
4048
|
-
const scaledX = Math.round(commanderCoords.x * scaleX);
|
|
4049
|
-
const scaledY = Math.round(commanderCoords.y * scaleY);
|
|
4050
|
-
|
|
4051
|
-
// Ensure coordinates are within bounds
|
|
4052
|
-
const boundedX = Math.max(0, Math.min(scaledX, followerSize.width - 1));
|
|
4053
|
-
const boundedY = Math.max(0, Math.min(scaledY, followerSize.height - 1));
|
|
4054
|
-
|
|
4055
|
-
this.logger.info(`📐 Coordinate scaling: ${commanderCoords.x},${commanderCoords.y} → ${boundedX},${boundedY}`);
|
|
4056
|
-
this.logger.info(` Commander: ${commanderSize.width}×${commanderSize.height}`);
|
|
4057
|
-
this.logger.info(` Follower: ${followerSize.width}×${followerSize.height}`);
|
|
4058
|
-
this.logger.info(` Scale: ${scaleX.toFixed(3)}×${scaleY.toFixed(3)}`);
|
|
4059
|
-
|
|
4060
|
-
return { x: boundedX, y: boundedY };
|
|
4061
|
-
}
|
|
4062
|
-
|
|
4063
|
-
// General action broadcasting (tap, type, etc.) with intelligent coordinate transformation
|
|
4064
|
-
async broadcastAction({ action, locator, input, coordinates }) {
|
|
4065
|
-
if (!this.isActive || this.followerDevices.size === 0) {
|
|
4066
|
-
this.logger.info('No followers to broadcast action to');
|
|
4067
|
-
return { success: true, broadcastCount: 0 };
|
|
4068
|
-
}
|
|
4069
|
-
|
|
4070
|
-
this.logger.info(`📢 Broadcasting action '${action}' to ${this.followerDevices.size} followers`);
|
|
4071
|
-
|
|
4072
|
-
// GENIUS: Get commander device screen size for coordinate scaling
|
|
4073
|
-
let commanderSize = null;
|
|
4074
|
-
if (coordinates && this.commanderDevice) {
|
|
4075
|
-
const commanderPort = this.commanderDevice.port || 8100;
|
|
4076
|
-
commanderSize = await this.getDeviceWindowSizeForScaling(this.commanderDevice.name, commanderPort);
|
|
4077
|
-
if (commanderSize) {
|
|
4078
|
-
this.logger.info(`📱 Commander screen size: ${commanderSize.width}×${commanderSize.height}`);
|
|
4079
|
-
}
|
|
4080
|
-
}
|
|
4081
|
-
|
|
4082
|
-
const results = [];
|
|
4083
|
-
const axios = require('axios');
|
|
4084
|
-
|
|
4085
|
-
for (const follower of this.followerDevices) {
|
|
4086
|
-
try {
|
|
4087
|
-
const port = follower.port || (follower.platform === 'ios' ? 8100 : 4723);
|
|
4088
|
-
|
|
4089
|
-
// GENIUS: Get follower screen size and scale coordinates if needed
|
|
4090
|
-
let scaledCoordinates = coordinates;
|
|
4091
|
-
if (coordinates && commanderSize) {
|
|
4092
|
-
const followerSize = await this.getDeviceWindowSizeForScaling(follower.name, port);
|
|
4093
|
-
if (followerSize) {
|
|
4094
|
-
scaledCoordinates = this.scaleCoordinatesForDevice(coordinates, commanderSize, followerSize);
|
|
4095
|
-
} else {
|
|
4096
|
-
this.logger.warn(`⚠️ Could not get screen size for follower ${follower.name}, using original coordinates`);
|
|
4097
|
-
}
|
|
4098
|
-
}
|
|
4099
|
-
|
|
4100
|
-
if (follower.platform === 'ios') {
|
|
4101
|
-
// iOS WebDriverAgent actions
|
|
4102
|
-
// GENIUS: Get session from global manager for each follower
|
|
4103
|
-
const followerSession = global.sessionManager?.getByName(follower.name);
|
|
4104
|
-
const sessionId = followerSession?.sessionId || this.getWDASessionId(); // Fallback to commander's session
|
|
4105
|
-
const baseUrl = `http://localhost:${port}`;
|
|
4106
|
-
|
|
4107
|
-
switch (action) {
|
|
4108
|
-
case 'tap':
|
|
4109
|
-
if (scaledCoordinates) {
|
|
4110
|
-
try {
|
|
4111
|
-
this.logger.info(`🎯 Tapping follower ${follower.name} at scaled coordinates: (${scaledCoordinates.x}, ${scaledCoordinates.y})`);
|
|
4112
|
-
await axios.post(`${baseUrl}/session/${sessionId}/wda/tap/0`, {
|
|
4113
|
-
x: scaledCoordinates.x,
|
|
4114
|
-
y: scaledCoordinates.y
|
|
4115
|
-
});
|
|
4116
|
-
} catch (tapErr) {
|
|
4117
|
-
// If 404 error, follower session is invalid - try with fresh session
|
|
4118
|
-
if (tapErr.response?.status === 404) {
|
|
4119
|
-
this.logger.warn(`⚠️ Follower ${follower.name} session expired (404), attempting recovery...`);
|
|
4120
|
-
try {
|
|
4121
|
-
// Try to get a fresh session for the follower
|
|
4122
|
-
const newFollowerSession = global.sessionManager?.getByName(follower.name);
|
|
4123
|
-
const newSessionId = newFollowerSession?.sessionId;
|
|
4124
|
-
|
|
4125
|
-
if (newSessionId && newSessionId !== sessionId) {
|
|
4126
|
-
// Retry with new session and scaled coordinates
|
|
4127
|
-
this.logger.info(`🔄 Retry tapping ${follower.name} with recovered session at: (${scaledCoordinates.x}, ${scaledCoordinates.y})`);
|
|
4128
|
-
await axios.post(`${baseUrl}/session/${newSessionId}/wda/tap/0`, {
|
|
4129
|
-
x: scaledCoordinates.x,
|
|
4130
|
-
y: scaledCoordinates.y
|
|
4131
|
-
});
|
|
4132
|
-
this.logger.info(`✅ Follower ${follower.name} tap succeeded after session recovery`);
|
|
4133
|
-
} else {
|
|
4134
|
-
throw tapErr; // Re-throw if no new session available
|
|
4135
|
-
}
|
|
4136
|
-
} catch (recoveryErr) {
|
|
4137
|
-
throw new Error(`Follower tap failed: ${recoveryErr.message}`);
|
|
4138
|
-
}
|
|
4139
|
-
} else {
|
|
4140
|
-
throw tapErr;
|
|
4141
|
-
}
|
|
4142
|
-
}
|
|
4143
|
-
} else if (locator) {
|
|
4144
|
-
// Find element and tap
|
|
4145
|
-
const elementRes = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
4146
|
-
using: 'accessibility id',
|
|
4147
|
-
value: locator
|
|
4148
|
-
});
|
|
4149
|
-
const elementId = elementRes.data.value.ELEMENT;
|
|
4150
|
-
await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/click`);
|
|
4151
|
-
}
|
|
4152
|
-
break;
|
|
4153
|
-
|
|
4154
|
-
case 'type':
|
|
4155
|
-
if (input) {
|
|
4156
|
-
await axios.post(`${baseUrl}/session/${sessionId}/wda/keys`, {
|
|
4157
|
-
value: input.split('')
|
|
4158
|
-
});
|
|
4159
|
-
}
|
|
4160
|
-
break;
|
|
4161
|
-
|
|
4162
|
-
case 'swipe':
|
|
4163
|
-
if (scaledCoordinates && scaledCoordinates.fromX !== undefined) {
|
|
4164
|
-
// Scale swipe coordinates too
|
|
4165
|
-
let scaledSwipeCoords = scaledCoordinates;
|
|
4166
|
-
if (coordinates && commanderSize) {
|
|
4167
|
-
const followerSize = await this.getDeviceWindowSizeForScaling(follower.name, port);
|
|
4168
|
-
if (followerSize) {
|
|
4169
|
-
scaledSwipeCoords = {
|
|
4170
|
-
fromX: Math.round(coordinates.fromX * (followerSize.width / commanderSize.width)),
|
|
4171
|
-
fromY: Math.round(coordinates.fromY * (followerSize.height / commanderSize.height)),
|
|
4172
|
-
toX: Math.round(coordinates.toX * (followerSize.width / commanderSize.width)),
|
|
4173
|
-
toY: Math.round(coordinates.toY * (followerSize.height / commanderSize.height)),
|
|
4174
|
-
duration: coordinates.duration || 0.5
|
|
4175
|
-
};
|
|
4176
|
-
}
|
|
4177
|
-
}
|
|
4178
|
-
|
|
4179
|
-
await axios.post(`${baseUrl}/session/${sessionId}/wda/dragfromtoforduration`, scaledSwipeCoords);
|
|
4180
|
-
}
|
|
4181
|
-
break;
|
|
4182
|
-
|
|
4183
|
-
default:
|
|
4184
|
-
this.logger.warn(`Unsupported iOS action: ${action}`);
|
|
4185
|
-
}
|
|
4186
|
-
} else if (follower.platform === 'android') {
|
|
4187
|
-
// Android UIAutomator2 actions
|
|
4188
|
-
const baseUrl = `http://localhost:${port}/wd/hub`;
|
|
4189
|
-
|
|
4190
|
-
switch (action) {
|
|
4191
|
-
case 'tap':
|
|
4192
|
-
if (scaledCoordinates) {
|
|
4193
|
-
this.logger.info(`🎯 Tapping Android follower ${follower.name} at scaled coordinates: (${scaledCoordinates.x}, ${scaledCoordinates.y})`);
|
|
4194
|
-
await axios.post(`${baseUrl}/session/${sessionId}/touch/click`, {
|
|
4195
|
-
element: { x: scaledCoordinates.x, y: scaledCoordinates.y }
|
|
4196
|
-
});
|
|
4197
|
-
} else if (locator) {
|
|
4198
|
-
// Find element and tap
|
|
4199
|
-
const elementRes = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
4200
|
-
using: 'accessibility id',
|
|
4201
|
-
value: locator
|
|
4202
|
-
});
|
|
4203
|
-
const elementId = elementRes.data.value.ELEMENT;
|
|
4204
|
-
await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/click`);
|
|
4205
|
-
}
|
|
4206
|
-
break;
|
|
4207
|
-
|
|
4208
|
-
case 'type':
|
|
4209
|
-
if (input) {
|
|
4210
|
-
await axios.post(`${baseUrl}/session/${sessionId}/keys`, {
|
|
4211
|
-
value: [input]
|
|
4212
|
-
});
|
|
4213
|
-
}
|
|
4214
|
-
break;
|
|
4215
|
-
|
|
4216
|
-
default:
|
|
4217
|
-
this.logger.warn(`Unsupported Android action: ${action}`);
|
|
4218
|
-
}
|
|
4219
|
-
}
|
|
4220
|
-
|
|
4221
|
-
results.push({ device: follower.name, success: true });
|
|
4222
|
-
this.logger.info(`✅ Action '${action}' broadcast to ${follower.name}`);
|
|
4223
|
-
|
|
4224
|
-
} catch (error) {
|
|
4225
|
-
this.logger.error(`❌ Failed to broadcast action to ${follower.name}: ${error.message}`);
|
|
4226
|
-
results.push({ device: follower.name, success: false, error: error.message });
|
|
4227
|
-
}
|
|
4228
|
-
}
|
|
4229
|
-
|
|
4230
|
-
const successCount = results.filter(r => r.success).length;
|
|
4231
|
-
this.logger.info(`📢 Action broadcast complete: ${successCount}/${results.length} followers updated`);
|
|
4232
|
-
|
|
4233
|
-
return {
|
|
4234
|
-
success: true,
|
|
4235
|
-
broadcastCount: successCount,
|
|
4236
|
-
results,
|
|
4237
|
-
action
|
|
4238
|
-
};
|
|
4239
|
-
}
|
|
4240
|
-
|
|
4241
|
-
/**
|
|
4242
|
-
* 🎯 NEW GENIUS METHOD: Execute Intelligent Action on Follower
|
|
4243
|
-
* Uses the IntelligentLocatorService to perform actions with maximum accuracy
|
|
4244
|
-
*/
|
|
4245
|
-
async executeIntelligentActionOnFollower(follower, intelligentLocator, originalX, originalY) {
|
|
4246
|
-
// Use session lock to prevent concurrent access
|
|
4247
|
-
return this.withSessionLock(follower.name, async () => {
|
|
4248
|
-
return this._executeIntelligentActionOnFollower(follower, intelligentLocator, originalX, originalY);
|
|
4249
|
-
});
|
|
4250
|
-
}
|
|
4251
|
-
|
|
4252
|
-
async _executeIntelligentActionOnFollower(follower, intelligentLocator, originalX, originalY) {
|
|
4253
|
-
this.logger.info(`🎯 Executing intelligent action on ${follower.name} using ${intelligentLocator.method}`);
|
|
4254
|
-
|
|
4255
|
-
try {
|
|
4256
|
-
const startTime = Date.now();
|
|
4257
|
-
|
|
4258
|
-
// OPTIMIZED: Only invalidate follower cache if commander cache was invalidated
|
|
4259
|
-
// This prevents excessive cache clearing
|
|
4260
|
-
if (!this.intelligentLocator.hasCachedLocators(follower.name)) {
|
|
4261
|
-
this.logger.info(`🧹 Pre-invalidating locator cache for follower ${follower.name} (screen changed after commander action)`);
|
|
4262
|
-
this.intelligentLocator.invalidateCache(follower.name);
|
|
4263
|
-
}
|
|
4264
|
-
|
|
4265
|
-
// Step 1: Execute using IntelligentLocatorService
|
|
4266
|
-
const result = await this.intelligentLocator.executeIntelligentAction(
|
|
4267
|
-
follower,
|
|
4268
|
-
intelligentLocator,
|
|
4269
|
-
'tap'
|
|
4270
|
-
);
|
|
4271
|
-
|
|
4272
|
-
const executionTime = Date.now() - startTime;
|
|
4273
|
-
|
|
4274
|
-
if (result.success) {
|
|
4275
|
-
this.logger.info(`✅ Intelligent action succeeded on ${follower.name} in ${executionTime}ms (method: ${result.method})`);
|
|
4276
|
-
|
|
4277
|
-
// Update success metrics
|
|
4278
|
-
if (result.method === 'locator') {
|
|
4279
|
-
this.locatorSuccessCount++;
|
|
4280
|
-
}
|
|
4281
|
-
|
|
4282
|
-
// Update per-device success rate
|
|
4283
|
-
const deviceRate = this.locatorSuccessRate.get(follower.name) || { success: 0, total: 0 };
|
|
4284
|
-
deviceRate.success++;
|
|
4285
|
-
deviceRate.total++;
|
|
4286
|
-
this.locatorSuccessRate.set(follower.name, deviceRate);
|
|
4287
|
-
|
|
4288
|
-
return { success: true, method: result.method, executionTime };
|
|
4289
|
-
|
|
4290
|
-
} else {
|
|
4291
|
-
this.logger.error(`❌ Intelligent action failed on ${follower.name}: ${result.error}`);
|
|
4292
|
-
|
|
4293
|
-
// Update metrics
|
|
4294
|
-
const deviceRate = this.locatorSuccessRate.get(follower.name) || { success: 0, total: 0 };
|
|
4295
|
-
deviceRate.total++;
|
|
4296
|
-
this.locatorSuccessRate.set(follower.name, deviceRate);
|
|
4297
|
-
|
|
4298
|
-
return { success: false, error: result.error, method: result.method };
|
|
4299
|
-
}
|
|
4300
|
-
|
|
4301
|
-
} catch (error) {
|
|
4302
|
-
this.logger.error(`❌ Intelligent follower execution failed: ${error.message}`);
|
|
4303
|
-
|
|
4304
|
-
// Fallback to traditional coordinate scaling as last resort
|
|
4305
|
-
this.logger.warn(`🔄 Attempting traditional coordinate fallback for ${follower.name}`);
|
|
4306
|
-
|
|
4307
|
-
try {
|
|
4308
|
-
// Use the existing coordinate scaling logic as fallback
|
|
4309
|
-
const commanderSize = this.lastWindowSize;
|
|
4310
|
-
const followerSize = await this.getDeviceWindowSizeForScaling(follower.name, follower.port || 8100);
|
|
4311
|
-
|
|
4312
|
-
if (followerSize && commanderSize) {
|
|
4313
|
-
const scaledCoords = this.scaleCoordinatesForDevice(
|
|
4314
|
-
{ x: originalX, y: originalY },
|
|
4315
|
-
commanderSize,
|
|
4316
|
-
followerSize
|
|
4317
|
-
);
|
|
4318
|
-
|
|
4319
|
-
// Execute using coordinates
|
|
4320
|
-
const coordResult = await this.intelligentLocator.executeCoordinateAction(
|
|
4321
|
-
follower,
|
|
4322
|
-
scaledCoords,
|
|
4323
|
-
'tap'
|
|
4324
|
-
);
|
|
4325
|
-
|
|
4326
|
-
if (coordResult) {
|
|
4327
|
-
this.logger.info(`✅ Coordinate fallback succeeded on ${follower.name}`);
|
|
4328
|
-
return { success: true, method: 'coordinates_fallback' };
|
|
4329
|
-
}
|
|
4330
|
-
}
|
|
4331
|
-
|
|
4332
|
-
throw new Error('Both intelligent and coordinate methods failed');
|
|
4333
|
-
|
|
4334
|
-
} catch (fallbackError) {
|
|
4335
|
-
this.logger.error(`❌ All methods failed for ${follower.name}: ${fallbackError.message}`);
|
|
4336
|
-
return { success: false, error: `All methods failed: ${error.message}` };
|
|
4337
|
-
}
|
|
4338
|
-
}
|
|
4339
|
-
}
|
|
4340
|
-
|
|
4341
|
-
/**
|
|
4342
|
-
* 🎯 Get Intelligent Locator Performance Statistics
|
|
4343
|
-
*/
|
|
4344
|
-
getIntelligentLocatorStats() {
|
|
4345
|
-
const totalRequests = this.locatorSuccessCount + this.coordinateFallbackCount;
|
|
4346
|
-
const locatorSuccessRate = totalRequests > 0 ?
|
|
4347
|
-
((this.locatorSuccessCount / totalRequests) * 100).toFixed(1) : 0;
|
|
4348
|
-
|
|
4349
|
-
const deviceStats = {};
|
|
4350
|
-
for (const [deviceName, stats] of this.locatorSuccessRate.entries()) {
|
|
4351
|
-
const rate = stats.total > 0 ? ((stats.success / stats.total) * 100).toFixed(1) : 0;
|
|
4352
|
-
deviceStats[deviceName] = {
|
|
4353
|
-
successRate: `${rate}%`,
|
|
4354
|
-
successful: stats.success,
|
|
4355
|
-
total: stats.total
|
|
4356
|
-
};
|
|
4357
|
-
}
|
|
4358
|
-
|
|
4359
|
-
return {
|
|
4360
|
-
overallLocatorSuccessRate: `${locatorSuccessRate}%`,
|
|
4361
|
-
locatorSuccesses: this.locatorSuccessCount,
|
|
4362
|
-
coordinateFallbacks: this.coordinateFallbackCount,
|
|
4363
|
-
totalRequests: totalRequests,
|
|
4364
|
-
deviceStats: deviceStats,
|
|
4365
|
-
intelligentLocatorMetrics: this.intelligentLocator?.getPerformanceMetrics()
|
|
4366
|
-
};
|
|
4367
|
-
}
|
|
4368
|
-
|
|
4369
|
-
/**
|
|
4370
|
-
* 🎯 Force Refresh Locator Cache for All Devices
|
|
4371
|
-
*/
|
|
4372
|
-
async refreshIntelligentLocatorCache() {
|
|
4373
|
-
this.logger.info('🔄 Refreshing intelligent locator cache for all devices...');
|
|
4374
|
-
|
|
4375
|
-
const devices = [this.commanderDevice, ...Array.from(this.followerDevices)].filter(d => d);
|
|
4376
|
-
const refreshResults = [];
|
|
4377
|
-
|
|
4378
|
-
for (const device of devices) {
|
|
4379
|
-
try {
|
|
4380
|
-
const locators = await this.intelligentLocator.getCachedLocators(device.name, true);
|
|
4381
|
-
refreshResults.push({
|
|
4382
|
-
device: device.name,
|
|
4383
|
-
success: true,
|
|
4384
|
-
locatorCount: locators.length
|
|
4385
|
-
});
|
|
4386
|
-
this.logger.info(`✅ Refreshed ${locators.length} locators for ${device.name}`);
|
|
4387
|
-
} catch (error) {
|
|
4388
|
-
refreshResults.push({
|
|
4389
|
-
device: device.name,
|
|
4390
|
-
success: false,
|
|
4391
|
-
error: error.message
|
|
4392
|
-
});
|
|
4393
|
-
this.logger.error(`❌ Failed to refresh locators for ${device.name}: ${error.message}`);
|
|
4394
|
-
}
|
|
4395
|
-
}
|
|
4396
|
-
|
|
4397
|
-
return {
|
|
4398
|
-
success: true,
|
|
4399
|
-
refreshedDevices: refreshResults.filter(r => r.success).length,
|
|
4400
|
-
totalDevices: refreshResults.length,
|
|
4401
|
-
results: refreshResults
|
|
4402
|
-
};
|
|
4403
|
-
}
|
|
4404
|
-
|
|
4405
|
-
getStatus() {
|
|
4406
|
-
const baseStatus = {
|
|
4407
|
-
isActive: this.isActive,
|
|
4408
|
-
commander: this.commanderDevice?.name,
|
|
4409
|
-
followers: Array.from(this.followerDevices).map(f => ({
|
|
4410
|
-
deviceId: f.name,
|
|
4411
|
-
name: f.name,
|
|
4412
|
-
platform: f.platform,
|
|
4413
|
-
port: f.port
|
|
4414
|
-
})),
|
|
4415
|
-
followerCount: this.followerDevices.size
|
|
4416
|
-
};
|
|
4417
|
-
|
|
4418
|
-
// Add intelligent locator stats if available
|
|
4419
|
-
if (this.useIntelligentLocators) {
|
|
4420
|
-
baseStatus.intelligentLocatorStats = this.getIntelligentLocatorStats();
|
|
4421
|
-
}
|
|
4422
|
-
|
|
4423
|
-
return baseStatus;
|
|
4424
|
-
}
|
|
4425
|
-
}
|
|
4426
|
-
|
|
4427
|
-
module.exports = CommanderService;
|