devicely 2.2.12 โ†’ 2.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +182 -81
  2. package/bin/devicely.js +1 -1
  3. package/config/devices.conf +2 -2
  4. package/lib/.logging-backup/aiProviders.js.backup +654 -0
  5. package/lib/.logging-backup/appMappings.js.backup +337 -0
  6. package/lib/.logging-backup/commanderService.js.backup +4427 -0
  7. package/lib/.logging-backup/devices.js.backup +54 -0
  8. package/lib/.logging-backup/doctor.js.backup +94 -0
  9. package/lib/.logging-backup/encryption.js.backup +61 -0
  10. package/lib/.logging-backup/executor.js.backup +104 -0
  11. package/lib/.logging-backup/hybridAI.js.backup +154 -0
  12. package/lib/.logging-backup/intelligentLocatorService.js.backup +1541 -0
  13. package/lib/.logging-backup/locatorStrategy.js.backup +342 -0
  14. package/lib/.logging-backup/scriptLoader.js.backup +13 -0
  15. package/lib/.logging-backup/server.js.backup +6298 -0
  16. package/lib/.logging-backup/tensorflowAI.js.backup +714 -0
  17. package/lib/.logging-backup/universalSessionManager.js.backup +370 -0
  18. package/lib/.logging-enhanced-backup/server.js.enhanced-backup +6298 -0
  19. package/lib/advanced-logger.js +1 -0
  20. package/lib/aiProviders.js +154 -15
  21. package/lib/aiProviders.js.strategic-backup +657 -0
  22. package/lib/aiProvidersConfig.js +61 -151
  23. package/lib/aiProvidersConfig.js.backup +218 -0
  24. package/lib/androidDeviceDetection.js +1 -1
  25. package/lib/appMappings.js +1 -1
  26. package/lib/commanderService.js +1 -1
  27. package/lib/commanderService.js.backup +5552 -0
  28. package/lib/deviceDetection.js +1 -1
  29. package/lib/devices.js +1 -1
  30. package/lib/devices.js.strategic-backup +57 -0
  31. package/lib/doctor.js +1 -1
  32. package/lib/encryption.js +1 -1
  33. package/lib/encryption.js.strategic-backup +61 -0
  34. package/lib/executor.js +1 -1
  35. package/lib/executor.js.strategic-backup +107 -0
  36. package/lib/frontend/asset-manifest.json +5 -3
  37. package/lib/frontend/index.html +1 -1
  38. package/lib/hybridAI.js +1 -0
  39. package/lib/intelligentLocatorService.js +1 -0
  40. package/lib/lightweightAI.js +1 -0
  41. package/lib/localBuiltInAI.js +1 -0
  42. package/lib/localBuiltInAI_backup.js +1 -0
  43. package/lib/localBuiltInAI_simple.js +1 -0
  44. package/lib/locatorStrategy.js +1 -1
  45. package/lib/logger-demo.js +2 -0
  46. package/lib/logger-integration-examples.js +102 -0
  47. package/lib/logger.js +1 -1
  48. package/lib/package.json +5 -0
  49. package/lib/public/asset-manifest.json +3 -3
  50. package/lib/public/index.html +1 -1
  51. package/lib/quick-start-logger.js +2 -0
  52. package/lib/scriptLoader.js +1 -1
  53. package/lib/server.js +1 -1
  54. package/lib/server.js.strategic-backup +6298 -0
  55. package/lib/tensorflowAI.js +1 -0
  56. package/lib/tensorflowAI.js.strategic-backup +717 -0
  57. package/lib/tinyAI.js +1 -0
  58. package/lib/universalSessionManager.js +1 -0
  59. package/package.json +1 -1
  60. package/scripts/shell/android_device_control.enc +1 -1
  61. package/scripts/shell/connect_ios_usb_multi_final.enc +1 -1
  62. package/scripts/shell/connect_ios_wireless_multi_final.enc +1 -1
  63. package/lib/public/index.html.bak +0 -1
@@ -0,0 +1,1541 @@
1
+ /**
2
+ * ๐ŸŽฏ INTELLIGENT LOCATOR SERVICE - GENIUS SOLUTION
3
+ * ================================================
4
+ *
5
+ * This service implements a revolutionary locator-based Commander-Follower system:
6
+ * 1. Caches all screen locators when screen loads
7
+ * 2. Uses tiny AI models for intelligent locator matching
8
+ * 3. Converts coordinates to precise locators instantly
9
+ * 4. 100% accurate cross-device element targeting
10
+ * 5. Zero-cost, open-source, local processing
11
+ */
12
+
13
+ const Logger = require('./logger');
14
+ const axios = require('axios');
15
+ const fs = require('fs').promises;
16
+ const path = require('path');
17
+ const universalSessionManager = require('./universalSessionManager');
18
+
19
+ class IntelligentLocatorService {
20
+ constructor(connectedDevices = null) {
21
+ this.logger = new Logger('IntelligentLocatorService');
22
+
23
+ // Store reference to connected devices
24
+ this.connectedDevices = connectedDevices;
25
+
26
+ // REVOLUTIONARY: Universal Session Management
27
+ this.universalSessionManager = universalSessionManager;
28
+
29
+ // Locator Cache Management - BALANCED for reliability and speed
30
+ this.locatorCache = new Map(); // Map<deviceName, CachedLocators>
31
+ this.cacheTimestamps = new Map(); // Map<deviceName, timestamp>
32
+ this.cacheValidityMs = 25000; // BALANCED: 25 seconds cache validity (was 20s, originally 30s)
33
+ this.refreshingDevices = new Set(); // Track devices currently being refreshed
34
+
35
+ // AI Model for Locator Matching
36
+ this.aiModel = null; // Will hold local tiny AI model
37
+ this.modelPath = path.join(__dirname, 'ai', 'locator_model');
38
+
39
+ // Element Similarity Scoring
40
+ this.similarityThreshold = 0.40; // 40% similarity threshold - more flexible
41
+
42
+ // Performance Metrics
43
+ this.performanceMetrics = {
44
+ cacheHits: 0,
45
+ cacheMisses: 0,
46
+ accurateMatches: 0,
47
+ fallbackToCoordinates: 0
48
+ };
49
+
50
+ this.initializeAIModel();
51
+ }
52
+
53
+ /**
54
+ * ๐Ÿ”ง Decode HTML entities from text
55
+ */
56
+ decodeHtmlEntities(text) {
57
+ if (!text) return text;
58
+
59
+ return text
60
+ .replace(/&amp;/g, '&')
61
+ .replace(/&lt;/g, '<')
62
+ .replace(/&gt;/g, '>')
63
+ .replace(/&quot;/g, '"')
64
+ .replace(/&#39;/g, "'")
65
+ .replace(/&nbsp;/g, ' ');
66
+ }
67
+
68
+ /**
69
+ * ๐Ÿค– Initialize Local Tiny AI Model for Locator Intelligence
70
+ * Uses TensorFlow.js with a lightweight model for element classification
71
+ */
72
+ async initializeAIModel() {
73
+ try {
74
+ // Check if we have TensorFlow.js available
75
+ let tf;
76
+ try {
77
+ tf = require('@tensorflow/tfjs-node');
78
+ } catch (error) {
79
+ this.logger.warn('๐Ÿค– TensorFlow.js not available, falling back to rule-based matching');
80
+ this.useRuleBasedMatching = true;
81
+ return;
82
+ }
83
+
84
+ // Try to load existing model or create a simple one
85
+ const modelExists = await this.checkModelExists();
86
+
87
+ if (modelExists) {
88
+ this.aiModel = await tf.loadLayersModel(`file://${this.modelPath}/model.json`);
89
+ this.logger.info('๐Ÿค– โœ… Loaded existing AI locator model');
90
+ } else {
91
+ // Create a simple neural network for element classification
92
+ this.aiModel = await this.createSimpleLocatorModel(tf);
93
+ this.logger.info('๐Ÿค– โœ… Created new AI locator model');
94
+ }
95
+
96
+ } catch (error) {
97
+ this.logger.warn(`๐Ÿค– AI model initialization failed: ${error.message}, using rule-based matching`);
98
+ this.useRuleBasedMatching = true;
99
+ }
100
+ }
101
+
102
+ /**
103
+ * ๐Ÿง  Create a Simple Locator Classification Model
104
+ * Lightweight model for element type and text similarity classification
105
+ */
106
+ async createSimpleLocatorModel(tf) {
107
+ // Simple neural network for element feature matching
108
+ const model = tf.sequential({
109
+ layers: [
110
+ tf.layers.dense({ inputShape: [10], units: 16, activation: 'relu' }),
111
+ tf.layers.dropout({ rate: 0.2 }),
112
+ tf.layers.dense({ units: 8, activation: 'relu' }),
113
+ tf.layers.dense({ units: 3, activation: 'softmax' }) // 3 classes: button, text, input
114
+ ]
115
+ });
116
+
117
+ model.compile({
118
+ optimizer: 'adam',
119
+ loss: 'categoricalCrossentropy',
120
+ metrics: ['accuracy']
121
+ });
122
+
123
+ return model;
124
+ }
125
+
126
+ async checkModelExists() {
127
+ try {
128
+ await fs.access(path.join(this.modelPath, 'model.json'));
129
+ return true;
130
+ } catch {
131
+ return false;
132
+ }
133
+ }
134
+
135
+ /**
136
+ * ๐ŸŽฏ MAIN GENIUS METHOD: Convert Click Coordinates to Intelligent Locator
137
+ */
138
+ async getIntelligentLocator(deviceName, x, y, forceRefresh = false) {
139
+ this.logger.info(`๐ŸŽฏ Finding intelligent locator for ${deviceName} at (${x}, ${y})`);
140
+
141
+ try {
142
+ // Step 1: Ensure we have fresh locator cache - FORCE REFRESH AFTER EVERY ACTION
143
+ const shouldForceRefresh = forceRefresh || this.shouldForceRefreshCache(deviceName);
144
+
145
+ if (shouldForceRefresh) {
146
+ this.logger.info(`๐Ÿ”„ FORCING cache refresh for ${deviceName} (forceRefresh=${forceRefresh})`);
147
+ }
148
+
149
+ const cachedLocators = await this.getCachedLocators(deviceName, shouldForceRefresh);
150
+
151
+ if (!cachedLocators || cachedLocators.length === 0) {
152
+ this.logger.warn(`โŒ No cached locators for ${deviceName}, falling back to coordinates`);
153
+ this.performanceMetrics.fallbackToCoordinates++;
154
+ return { method: 'coordinates', x, y };
155
+ }
156
+
157
+ this.logger.debug(`๐Ÿ“ฆ Using ${cachedLocators.length} cached locators for matching`);
158
+
159
+ // Step 2: Find best matching element using AI/rules
160
+ const bestMatch = await this.findBestMatchingElement(cachedLocators, x, y);
161
+
162
+ if (bestMatch && bestMatch.confidence > this.similarityThreshold) {
163
+ this.logger.info(`โœ… High-confidence locator match found (${(bestMatch.confidence * 100).toFixed(1)}%)`);
164
+ this.performanceMetrics.accurateMatches++;
165
+
166
+ return {
167
+ method: 'locator',
168
+ locator: bestMatch.locator,
169
+ confidence: bestMatch.confidence,
170
+ element: bestMatch.element,
171
+ fallbackCoords: { x, y } // Always include fallback
172
+ };
173
+ }
174
+
175
+ // Step 3: Fallback to coordinate-based if no good match
176
+ this.logger.warn(`โš ๏ธ Low confidence match (${bestMatch ? (bestMatch.confidence * 100).toFixed(1) : 0}%), using coordinates`);
177
+ this.performanceMetrics.fallbackToCoordinates++;
178
+
179
+ return {
180
+ method: 'coordinates',
181
+ x,
182
+ y,
183
+ nearestElement: bestMatch ? bestMatch.element : null
184
+ };
185
+
186
+ } catch (error) {
187
+ this.logger.error(`โŒ Intelligent locator failed: ${error.message}`);
188
+ return { method: 'coordinates', x, y, error: error.message };
189
+ }
190
+ }
191
+
192
+ /**
193
+ * ๐ŸŽฏ Determine if cache should be forcefully refreshed - OPTIMIZED for speed
194
+ */
195
+ shouldForceRefreshCache(deviceName) {
196
+ const now = Date.now();
197
+ const lastCached = this.cacheTimestamps.get(deviceName);
198
+
199
+ // PERFORMANCE: More aggressive caching for commander speed
200
+ // Force refresh only if:
201
+ // 1. Never cached
202
+ // 2. Cache is older than 15 seconds (increased from 10s for speed)
203
+ // 3. Cache was explicitly invalidated
204
+
205
+ if (!lastCached) {
206
+ this.logger.debug(`๐Ÿ”„ Force refresh: ${deviceName} never cached`);
207
+ return true;
208
+ }
209
+
210
+ const cacheAge = now - lastCached;
211
+ if (cacheAge > 15000) { // OPTIMIZED: Increased from 10s to 15s for better performance
212
+ this.logger.debug(`๐Ÿ”„ Force refresh: ${deviceName} cache is ${Math.round(cacheAge/1000)}s old`);
213
+ return true;
214
+ }
215
+
216
+ // Check if we have actual cached data
217
+ const cachedData = this.locatorCache.get(deviceName);
218
+ if (!cachedData || cachedData.length === 0) {
219
+ this.logger.debug(`๐Ÿ”„ Force refresh: ${deviceName} has no cached data`);
220
+ return true;
221
+ }
222
+
223
+ return false;
224
+ }
225
+
226
+ /**
227
+ * ๐Ÿ“ฆ Get or Refresh Cached Locators for Device (ENHANCED)
228
+ */
229
+ async getCachedLocators(deviceName, forceRefresh = false) {
230
+ const cacheKey = deviceName;
231
+ const now = Date.now();
232
+ const lastCached = this.cacheTimestamps.get(cacheKey);
233
+
234
+ // Check if cache is valid and not forcing refresh
235
+ if (!forceRefresh && lastCached && this.locatorCache.has(cacheKey)) {
236
+ const cacheAge = now - lastCached;
237
+ if (cacheAge < this.cacheValidityMs) {
238
+ this.logger.debug(`๐Ÿ“ฆ Using cached locators for ${deviceName} (age: ${Math.round(cacheAge/1000)}s)`);
239
+ this.performanceMetrics.cacheHits++;
240
+ return this.locatorCache.get(cacheKey);
241
+ }
242
+ }
243
+
244
+ // ๐ŸŽฏ ENHANCED: If cache was recently invalidated, always force fresh fetch
245
+ const wasRecentlyInvalidated = !lastCached || (now - lastCached) > 60000; // If older than 1 minute, was likely invalidated
246
+ const effectiveForceRefresh = forceRefresh || wasRecentlyInvalidated;
247
+
248
+ // ๐ŸŽฏ PREVENT DUPLICATE REFRESHES: If already refreshing this device, return cached data quickly
249
+ if (this.refreshingDevices.has(deviceName)) {
250
+ this.logger.debug(`โณ Device ${deviceName} is already being refreshed, using existing cache`);
251
+ // Don't wait - just return existing cache if available
252
+ const existingCache = this.locatorCache.get(cacheKey);
253
+ if (existingCache && existingCache.length > 0) {
254
+ this.logger.debug(`๐Ÿ“ฆ Using existing ${existingCache.length} locators for ${deviceName}`);
255
+ return existingCache;
256
+ }
257
+ }
258
+
259
+ // Mark as refreshing
260
+ this.refreshingDevices.add(deviceName);
261
+ this.logger.debug(`๐Ÿ”„ Marked ${deviceName} as refreshing`);
262
+
263
+ // Refresh cache
264
+ const ageText = lastCached ? `${Math.round((now - lastCached)/1000)}s` : 'never cached';
265
+ this.logger.info(`๐Ÿ”„ Refreshing locator cache for ${deviceName} (force: ${effectiveForceRefresh}, age: ${ageText})`);
266
+ this.performanceMetrics.cacheMisses++;
267
+
268
+ try {
269
+ const freshLocators = await this.fetchAllLocators(deviceName);
270
+
271
+ if (freshLocators && freshLocators.length > 0) {
272
+ this.locatorCache.set(cacheKey, freshLocators);
273
+ this.cacheTimestamps.set(cacheKey, now);
274
+ this.logger.info(`โœ… Cached ${freshLocators.length} locators for ${deviceName}`);
275
+ return freshLocators;
276
+ } else {
277
+ this.logger.warn(`โš ๏ธ No locators found for ${deviceName}`);
278
+ return [];
279
+ }
280
+
281
+ } catch (error) {
282
+ this.logger.error(`โŒ Failed to refresh locators for ${deviceName}: ${error.message}`);
283
+ // Return stale cache if available, but mark it as potentially stale
284
+ const staleCache = this.locatorCache.get(cacheKey) || [];
285
+ if (staleCache.length > 0) {
286
+ this.logger.warn(`โš ๏ธ Using stale cache with ${staleCache.length} locators for ${deviceName}`);
287
+ return staleCache;
288
+ } else {
289
+ // If no cache available, return empty array but don't throw error
290
+ this.logger.warn(`โš ๏ธ No fallback cache available for ${deviceName}, returning empty cache`);
291
+ return [];
292
+ }
293
+ } finally {
294
+ // Always remove from refreshing set
295
+ this.refreshingDevices.delete(deviceName);
296
+ this.logger.debug(`๐Ÿ”“ Removed ${deviceName} from refreshing set`);
297
+ }
298
+ }
299
+
300
+ /**
301
+ * ๐Ÿ” Fetch All Locators from Device (Enhanced Direct WDA Method)
302
+ */
303
+ async fetchAllLocators(deviceName) {
304
+ try {
305
+ // For iOS devices, use direct WDA page source (faster and more reliable)
306
+ const device = this.connectedDevices?.find(d => d.name === deviceName);
307
+ if (!device) {
308
+ throw new Error(`Device ${deviceName} not found in connected devices`);
309
+ }
310
+
311
+ if (device.platform === 'ios') {
312
+ return await this.fetchIOSLocatorsDirect(deviceName, device);
313
+ }
314
+
315
+ // For Android, use the existing API endpoint
316
+ const response = await axios.post('http://localhost:3001/api/locators', {
317
+ deviceName: deviceName
318
+ }, { timeout: 15000 });
319
+
320
+ if (response.data.success && response.data.locators) {
321
+ const locators = response.data.locators;
322
+
323
+ // Process and enhance locators with our intelligence
324
+ const enhancedLocators = locators.map(locator => ({
325
+ ...locator,
326
+ // Add computed properties for matching
327
+ centerX: locator.bounds ? (locator.bounds.x + locator.bounds.width / 2) : null,
328
+ centerY: locator.bounds ? (locator.bounds.y + locator.bounds.height / 2) : null,
329
+ area: locator.bounds ? (locator.bounds.width * locator.bounds.height) : 0,
330
+ // Element classification
331
+ isClickable: this.isElementClickable(locator),
332
+ elementType: this.classifyElementType(locator),
333
+ // Text features for matching
334
+ hasText: !!(locator.text || locator.accessibilityLabel || locator.contentDesc),
335
+ textLength: (locator.text || locator.accessibilityLabel || locator.contentDesc || '').length,
336
+ // Priority scoring
337
+ priority: this.calculateElementPriority(locator)
338
+ }));
339
+
340
+ // Sort by priority (most likely to be interaction targets first)
341
+ enhancedLocators.sort((a, b) => b.priority - a.priority);
342
+
343
+ return enhancedLocators;
344
+ }
345
+
346
+ return [];
347
+
348
+ } catch (error) {
349
+ this.logger.error(`Failed to fetch locators: ${error.message}`);
350
+ throw error;
351
+ }
352
+ }
353
+
354
+ /**
355
+ * ๐Ÿ“ฑ Fetch iOS Locators Directly from WDA (NEW METHOD)
356
+ */
357
+ async fetchIOSLocatorsDirect(deviceName, device) {
358
+ const port = device.port || 8100;
359
+ let attempts = 0;
360
+ const maxAttempts = 3;
361
+
362
+ while (attempts < maxAttempts) {
363
+ attempts++;
364
+
365
+ try {
366
+ // REVOLUTIONARY: Use universal session manager for consistent sessions
367
+ const sessionId = await this.universalSessionManager.getSessionWithRetry(deviceName, port);
368
+ if (!sessionId) {
369
+ throw new Error(`No valid session for iOS device ${deviceName} on attempt ${attempts}`);
370
+ }
371
+
372
+ this.logger.debug(`๐Ÿ“ฑ Fetching iOS locators directly from WDA for ${deviceName} (session: ${sessionId.substring(0, 8)}..., attempt: ${attempts})`);
373
+
374
+ // Try multiple WDA endpoints for getting page source
375
+ let response;
376
+ let pageSource;
377
+
378
+ // Method 1: Standard source endpoint
379
+ try {
380
+ response = await axios.get(`http://localhost:${port}/session/${sessionId}/source`, {
381
+ timeout: 8000
382
+ });
383
+ pageSource = response.data?.value;
384
+ } catch (error) {
385
+ this.logger.warn(`๐Ÿ“ฑ Standard source endpoint failed: ${error.message}`);
386
+ }
387
+
388
+ // Method 2: If first method fails, try WDA-specific source endpoint
389
+ if (!pageSource) {
390
+ try {
391
+ response = await axios.get(`http://localhost:${port}/session/${sessionId}/wda/source`, {
392
+ timeout: 8000
393
+ });
394
+ pageSource = response.data?.value;
395
+ } catch (error) {
396
+ this.logger.warn(`๐Ÿ“ฑ WDA source endpoint also failed: ${error.message}`);
397
+ }
398
+ }
399
+
400
+ // Method 4: Try getting all elements and extracting from that
401
+ if (!pageSource) {
402
+ try {
403
+ response = await axios.get(`http://localhost:${port}/session/${sessionId}/elements`, {
404
+ timeout: 8000,
405
+ params: { using: 'xpath', value: '//*' }
406
+ });
407
+
408
+ if (response.data?.value && Array.isArray(response.data.value)) {
409
+ // Convert elements array to simple XML-like structure for parsing
410
+ pageSource = this.convertElementsToPageSource(response.data.value, port, sessionId);
411
+ }
412
+ } catch (error) {
413
+ this.logger.warn(`๐Ÿ“ฑ Elements endpoint also failed: ${error.message}`);
414
+ }
415
+ }
416
+
417
+ // Method 5: Fallback to a basic hierarchical page source
418
+ if (!pageSource) {
419
+ try {
420
+ response = await axios.get(`http://localhost:${port}/session/${sessionId}/wda/element/0/tree`, {
421
+ timeout: 8000
422
+ });
423
+ pageSource = JSON.stringify(response.data);
424
+ } catch (error) {
425
+ this.logger.warn(`๐Ÿ“ฑ Element tree with root also failed: ${error.message}`);
426
+ }
427
+ }
428
+
429
+ if (!pageSource) {
430
+ if (attempts < maxAttempts) {
431
+ this.logger.warn(`๐Ÿ“ฑ All source methods failed for ${deviceName}, attempt ${attempts}/${maxAttempts}. Retrying...`);
432
+ await new Promise(resolve => setTimeout(resolve, 1000)); // Wait 1s before retry
433
+ continue;
434
+ } else {
435
+ throw new Error('No page source returned from any WDA endpoint after all attempts');
436
+ }
437
+ }
438
+
439
+ const parsedElements = this.parseIOSPageSource(pageSource);
440
+
441
+ this.logger.info(`โœ… Parsed ${parsedElements.length} iOS locators directly from WDA (attempt ${attempts})`);
442
+
443
+ return parsedElements;
444
+
445
+ } catch (error) {
446
+ if (attempts < maxAttempts) {
447
+ this.logger.warn(`๐Ÿ“ฑ iOS locator fetch attempt ${attempts}/${maxAttempts} failed: ${error.message}. Retrying...`);
448
+ await new Promise(resolve => setTimeout(resolve, 1000)); // Wait 1s before retry
449
+ } else {
450
+ throw error;
451
+ }
452
+ }
453
+ }
454
+
455
+ throw new Error(`Failed to fetch iOS locators after ${maxAttempts} attempts`);
456
+ }
457
+
458
+ /**
459
+ * ๐Ÿ“ Parse iOS Page Source to Extract Locators
460
+ */
461
+ parseIOSPageSource(pageSource) {
462
+ const elements = [];
463
+
464
+ // More comprehensive regex to catch interactive elements
465
+ const xmlRegex = /<XCUIElementType(Button|TextField|SecureTextField|Switch|Link|StaticText|SearchField|Slider|Stepper|DatePicker|PickerWheel|PageIndicator|SegmentedControl|TabBar|NavigationBar|Cell|Image|TextView|WebView)[^>]*>/g;
466
+
467
+ let match;
468
+ while ((match = xmlRegex.exec(pageSource)) !== null) {
469
+ const elementTag = match[0];
470
+
471
+ // Extract attributes with better parsing
472
+ const attributes = this.parseElementAttributes(elementTag);
473
+
474
+ if (attributes.x !== undefined && attributes.y !== undefined &&
475
+ attributes.width !== undefined && attributes.height !== undefined) {
476
+
477
+ // More permissive filtering for iOS elements
478
+ if (attributes.width > 5 && attributes.height > 5 && attributes.enabled) {
479
+
480
+ const elementType = `XCUIElementType${match[1]}`;
481
+ const isInteractive = this.isInteractiveElement(elementType, attributes);
482
+
483
+ // Include both interactive elements and meaningful text elements
484
+ if (isInteractive || (attributes.label && attributes.label.length > 0)) {
485
+ elements.push({
486
+ type: elementType,
487
+ text: attributes.label || attributes.name,
488
+ accessibilityLabel: attributes.label,
489
+ name: attributes.name,
490
+ x: Math.round(attributes.x),
491
+ y: Math.round(attributes.y),
492
+ width: Math.round(attributes.width),
493
+ height: Math.round(attributes.height),
494
+ bounds: {
495
+ x: Math.round(attributes.x),
496
+ y: Math.round(attributes.y),
497
+ width: Math.round(attributes.width),
498
+ height: Math.round(attributes.height)
499
+ },
500
+ centerX: Math.round(attributes.x + attributes.width / 2),
501
+ centerY: Math.round(attributes.y + attributes.height / 2),
502
+ enabled: attributes.enabled,
503
+ visible: attributes.visible,
504
+ // Enhanced properties for intelligent matching
505
+ isClickable: this.isInteractiveElement(elementType, attributes),
506
+ elementType: this.classifyElementType({ type: elementType, text: attributes.label }),
507
+ hasText: !!(attributes.label || attributes.name),
508
+ textLength: (attributes.label || attributes.name || '').length,
509
+ priority: this.calculateElementPriority({
510
+ type: elementType,
511
+ text: attributes.label,
512
+ width: attributes.width,
513
+ height: attributes.height,
514
+ enabled: attributes.enabled
515
+ })
516
+ });
517
+ }
518
+ }
519
+ }
520
+ }
521
+
522
+ // Sort by priority (most likely to be interaction targets first)
523
+ elements.sort((a, b) => b.priority - a.priority);
524
+
525
+ this.logger.debug(`โœ… Parsed and sorted ${elements.length} iOS elements by priority`);
526
+ return elements;
527
+ }
528
+
529
+ /**
530
+ * ๐Ÿ”ง Parse Element Attributes from XML Tag
531
+ */
532
+ parseElementAttributes(elementTag) {
533
+ const getAttribute = (name) => {
534
+ const match = elementTag.match(new RegExp(`${name}="([^"]*)"`, 'i'));
535
+ return match ? match[1] : null;
536
+ };
537
+
538
+ return {
539
+ x: parseFloat(getAttribute('x')) || 0,
540
+ y: parseFloat(getAttribute('y')) || 0,
541
+ width: parseFloat(getAttribute('width')) || 0,
542
+ height: parseFloat(getAttribute('height')) || 0,
543
+ label: getAttribute('label'),
544
+ name: getAttribute('name'),
545
+ enabled: getAttribute('enabled') !== 'false',
546
+ visible: getAttribute('visible') !== 'false',
547
+ accessible: getAttribute('accessible') === 'true'
548
+ };
549
+ }
550
+
551
+ /**
552
+ * ๐ŸŽฏ Check if Element is Interactive
553
+ */
554
+ isInteractiveElement(elementType, attributes) {
555
+ const interactiveTypes = [
556
+ 'Button', 'TextField', 'SecureTextField', 'Switch', 'Link',
557
+ 'SearchField', 'Slider', 'Stepper', 'DatePicker', 'PickerWheel',
558
+ 'SegmentedControl', 'Cell'
559
+ ];
560
+
561
+ const typeMatch = interactiveTypes.some(type => elementType.includes(type));
562
+ const hasAction = attributes.label && (
563
+ attributes.label.toLowerCase().includes('button') ||
564
+ attributes.label.toLowerCase().includes('tap') ||
565
+ attributes.label.toLowerCase().includes('click')
566
+ );
567
+
568
+ return typeMatch || hasAction;
569
+ }
570
+
571
+ /**
572
+ * ๐ŸŽฏ Find Best Matching Element using AI and Geometric Analysis
573
+ */
574
+ async findBestMatchingElement(cachedLocators, x, y) {
575
+ const candidates = [];
576
+
577
+ // PERFORMANCE: Limit search to top priority elements first for speed
578
+ const maxCandidates = 20; // OPTIMIZED: Process only top 20 elements for speed
579
+ let candidatesFound = 0;
580
+
581
+ // Step 1: Find elements near the click coordinates - OPTIMIZED
582
+ for (const locator of cachedLocators) {
583
+ if (!locator.bounds || !locator.centerX || !locator.centerY) continue;
584
+
585
+ // PERFORMANCE: Break early if we have enough good candidates
586
+ if (candidatesFound >= maxCandidates) break;
587
+
588
+ // Calculate distance from click point
589
+ const distance = Math.sqrt(
590
+ Math.pow(x - locator.centerX, 2) +
591
+ Math.pow(y - locator.centerY, 2)
592
+ );
593
+
594
+ // Check if click is within element bounds (with generous tolerance)
595
+ const tolerance = 25; // Increased tolerance for mobile UI
596
+ const isWithinBounds = (
597
+ x >= locator.bounds.x - tolerance &&
598
+ x <= locator.bounds.x + locator.bounds.width + tolerance &&
599
+ y >= locator.bounds.y - tolerance &&
600
+ y <= locator.bounds.y + locator.bounds.height + tolerance
601
+ );
602
+
603
+ if (isWithinBounds || distance < 80) { // Within element or nearby (increased from 50)
604
+ candidates.push({
605
+ locator: locator,
606
+ distance: distance,
607
+ isWithinBounds: isWithinBounds,
608
+ geometricScore: this.calculateGeometricScore(locator, x, y),
609
+ elementScore: this.calculateElementScore(locator)
610
+ });
611
+ candidatesFound++;
612
+ }
613
+ }
614
+
615
+ if (candidates.length === 0) {
616
+ this.logger.warn(`No candidate elements found near (${x}, ${y})`);
617
+ return null;
618
+ }
619
+
620
+ // Step 2: Score candidates using AI or rules
621
+ let scoredCandidates;
622
+
623
+ if (this.aiModel && !this.useRuleBasedMatching) {
624
+ scoredCandidates = await this.scoreWithAI(candidates, x, y);
625
+ } else {
626
+ scoredCandidates = this.scoreWithRules(candidates, x, y);
627
+ }
628
+
629
+ // Step 3: Select best candidate
630
+ scoredCandidates.sort((a, b) => b.totalScore - a.totalScore);
631
+ const bestCandidate = scoredCandidates[0];
632
+
633
+ this.logger.info(`๐ŸŽฏ Best candidate: ${bestCandidate.locator.elementType} with score ${bestCandidate.totalScore.toFixed(3)}`);
634
+
635
+ return {
636
+ locator: this.buildOptimalLocator(bestCandidate.locator),
637
+ confidence: bestCandidate.totalScore,
638
+ element: bestCandidate.locator,
639
+ allCandidates: scoredCandidates.slice(0, 3) // Top 3 for debugging
640
+ };
641
+ }
642
+
643
+ /**
644
+ * ๐Ÿค– Score Candidates using AI Model
645
+ */
646
+ async scoreWithAI(candidates, x, y) {
647
+ // This would use the TensorFlow model to score candidates
648
+ // For now, we'll fall back to rule-based scoring
649
+ return this.scoreWithRules(candidates, x, y);
650
+ }
651
+
652
+ /**
653
+ * ๐Ÿ“ Score Candidates using Geometric and Heuristic Rules
654
+ */
655
+ scoreWithRules(candidates, x, y) {
656
+ return candidates.map(candidate => {
657
+ const { locator, distance, isWithinBounds, geometricScore, elementScore } = candidate;
658
+
659
+ // Distance score (closer is better, more generous)
660
+ const maxDistance = 150; // Increased from 100
661
+ const distanceScore = Math.max(0.1, 1 - (distance / maxDistance)); // Min score of 0.1
662
+
663
+ // Bounds score (inside is much better, but outside isn't terrible)
664
+ const boundsScore = isWithinBounds ? 1.0 : 0.5; // Improved from 0.3
665
+
666
+ // Clickability score
667
+ const clickabilityScore = locator.isClickable ? 1.0 : 0.5;
668
+
669
+ // Size appropriateness (not too small, not too large)
670
+ const idealArea = 1000; // ~30x30 px
671
+ const sizeScore = locator.area > 0 ?
672
+ Math.exp(-Math.abs(Math.log(locator.area / idealArea))) : 0.5;
673
+
674
+ // Element type preference (iOS-specific types, improved for links)
675
+ const elementTypeLower = (locator.elementType || '').toLowerCase();
676
+ let typeScore = 0.6; // Default score
677
+
678
+ // HIGH PRIORITY: Link elements (especially for URL text)
679
+ if (elementTypeLower.includes('link')) typeScore = 1.0;
680
+ else if (elementTypeLower.includes('button') || elementTypeLower.includes('cell')) typeScore = 0.95;
681
+ else if (elementTypeLower.includes('switch') || elementTypeLower.includes('slider')) typeScore = 0.9;
682
+ else if (elementTypeLower.includes('textfield') || elementTypeLower.includes('input')) typeScore = 0.85;
683
+ else if (elementTypeLower.includes('image')) typeScore = 0.8;
684
+ else if (elementTypeLower.includes('text') || elementTypeLower.includes('label')) {
685
+ // Give higher score to text that looks like links
686
+ const textContent = (locator.text || '').toLowerCase();
687
+ if (textContent.includes('learn more') || textContent.includes('coverage') ||
688
+ textContent.includes('...') || textContent.includes('http')) {
689
+ typeScore = 0.9; // Boost for link-like text
690
+ } else {
691
+ typeScore = 0.75;
692
+ }
693
+ }
694
+ else if (locator.isClickable) typeScore = 0.8; // Any clickable element
695
+
696
+ // Combine all scores with improved weights (favor proximity and bounds)
697
+ const totalScore = (
698
+ distanceScore * 0.35 + // Increased: proximity is very important
699
+ boundsScore * 0.35 + // Increased: being within bounds is crucial
700
+ clickabilityScore * 0.15 + // Reduced: many elements are clickable
701
+ typeScore * 0.10 + // Added: element type matters
702
+ geometricScore * 0.03 + // Reduced: less important than proximity
703
+ elementScore * 0.02 // Reduced: less important than proximity
704
+ );
705
+
706
+ return {
707
+ ...candidate,
708
+ scores: {
709
+ distance: distanceScore,
710
+ bounds: boundsScore,
711
+ clickability: clickabilityScore,
712
+ geometric: geometricScore,
713
+ element: elementScore,
714
+ size: sizeScore,
715
+ type: typeScore
716
+ },
717
+ totalScore: Math.min(1.0, totalScore) // Cap at 1.0
718
+ };
719
+ });
720
+ }
721
+
722
+ /**
723
+ * ๐Ÿ“ Calculate Geometric Score
724
+ */
725
+ calculateGeometricScore(locator, x, y) {
726
+ if (!locator.bounds) return 0;
727
+
728
+ // Prefer elements that are centered around the click point
729
+ const centerDistanceScore = 1 / (1 + Math.abs(locator.centerX - x) + Math.abs(locator.centerY - y));
730
+
731
+ // Prefer reasonable aspect ratios
732
+ const aspectRatio = locator.bounds.width / locator.bounds.height;
733
+ const aspectScore = Math.exp(-Math.abs(Math.log(aspectRatio / 1.5))); // Prefer ~1.5:1 ratio
734
+
735
+ return (centerDistanceScore + aspectScore) / 2;
736
+ }
737
+
738
+ /**
739
+ * ๐Ÿท๏ธ Calculate Element Score based on properties (Enhanced for App Icons)
740
+ */
741
+ calculateElementScore(locator) {
742
+ let score = 0.5; // Base score
743
+ const className = (locator.className || locator.type || '').toLowerCase();
744
+
745
+ // ๐ŸŽฏ HIGH PRIORITY: App icons with text labels
746
+ if ((className.includes('cell') || className.includes('icon') || className.includes('app')) &&
747
+ locator.text && locator.text.trim().length > 0) {
748
+ score += 0.4; // Very high boost for app icons with names
749
+ this.logger.debug(`๐Ÿ“ฑ App icon detected: "${locator.text}" - boosted score`);
750
+ }
751
+
752
+ // Has meaningful text
753
+ if (locator.hasText && locator.textLength > 2) score += 0.2;
754
+
755
+ // Has accessibility info
756
+ if (locator.accessibilityId || locator.accessibilityLabel) score += 0.15;
757
+
758
+ // Has resource ID (Android) or name (iOS)
759
+ if (locator.resourceId || locator.name) score += 0.15;
760
+
761
+ // Clickable elements get priority
762
+ if (this.isElementClickable(locator)) score += 0.1;
763
+
764
+ return Math.min(1.0, score);
765
+ }
766
+
767
+ /**
768
+ * ๐Ÿท๏ธ Classify Element Type
769
+ */
770
+ classifyElementType(locator) {
771
+ const text = (locator.text || '').toLowerCase();
772
+ const className = (locator.className || locator.type || '').toLowerCase();
773
+ const resourceId = (locator.resourceId || '').toLowerCase();
774
+
775
+ // Link detection (NEW - high priority for URL links)
776
+ if (className.includes('link') ||
777
+ className.includes('xcuielementtypelink') ||
778
+ text.includes('http') ||
779
+ text.includes('www.') ||
780
+ text.includes('learn more') ||
781
+ text.includes('coverage') ||
782
+ text.includes('more info') ||
783
+ (text.includes('...') && text.length > 10)) { // Truncated link text
784
+ return 'link';
785
+ }
786
+
787
+ // Button detection (enhanced for back buttons, navigation, and app icons)
788
+ if (className.includes('button') ||
789
+ resourceId.includes('btn') ||
790
+ // Back button specific patterns
791
+ className.includes('back') ||
792
+ className.includes('navigation') ||
793
+ locator.accessibilityLabel?.toLowerCase().includes('back') ||
794
+ locator.name?.toLowerCase().includes('back') ||
795
+ // iOS specific button types
796
+ className.includes('xcuielementtypebutton') ||
797
+ // App icon patterns
798
+ className.includes('icon') ||
799
+ className.includes('app') ||
800
+ (className.includes('cell') && locator.text) || // App cells with text
801
+ // Generic button text patterns
802
+ text.match(/^(ok|cancel|submit|save|send|login|sign|tap|click|back|close|done)$/i)) {
803
+ return 'button';
804
+ }
805
+
806
+ // Input field detection
807
+ if (className.includes('edit') ||
808
+ className.includes('input') ||
809
+ className.includes('field') ||
810
+ resourceId.includes('input') ||
811
+ resourceId.includes('edit')) {
812
+ return 'input';
813
+ }
814
+
815
+ // App icon detection
816
+ if (className.includes('icon') ||
817
+ className.includes('app') ||
818
+ (className.includes('cell') && locator.text) || // App grid cells
819
+ (className.includes('collection') && locator.text)) { // Collection view cells
820
+ return 'app_icon';
821
+ }
822
+
823
+ // Image detection
824
+ if (className.includes('image') || className.includes('img')) {
825
+ return 'image';
826
+ }
827
+
828
+ // Text detection
829
+ if (className.includes('text') || locator.hasText) {
830
+ return 'text';
831
+ }
832
+
833
+ return 'unknown';
834
+ }
835
+
836
+ /**
837
+ * โœ… Check if Element is Clickable (Enhanced for Navigation Elements)
838
+ */
839
+ isElementClickable(locator) {
840
+ const className = (locator.className || locator.type || '').toLowerCase();
841
+ const text = (locator.text || '').toLowerCase();
842
+ const accessibilityLabel = (locator.accessibilityLabel || locator.label || '').toLowerCase();
843
+ const name = (locator.name || '').toLowerCase();
844
+
845
+ // Explicitly clickable elements
846
+ if (className.includes('button') ||
847
+ className.includes('clickable') ||
848
+ locator.clickable === true) {
849
+ return true;
850
+ }
851
+
852
+ // Back buttons and navigation elements (often without text)
853
+ if (className.includes('back') ||
854
+ className.includes('navigation') ||
855
+ className.includes('xcuielementtypebutton') ||
856
+ accessibilityLabel.includes('back') ||
857
+ name.includes('back') ||
858
+ accessibilityLabel.includes('close') ||
859
+ accessibilityLabel.includes('done')) {
860
+ return true;
861
+ }
862
+
863
+ // App icons and collection view cells (always clickable)
864
+ if (className.includes('icon') ||
865
+ className.includes('app') ||
866
+ className.includes('cell') || // Collection/table view cells
867
+ className.includes('collection') ||
868
+ (className.includes('xcuielementtypecell') && locator.text)) { // iOS cells with app names
869
+ return true;
870
+ }
871
+
872
+ // Interactive elements
873
+ if (className.includes('edit') ||
874
+ className.includes('input') ||
875
+ className.includes('switch') ||
876
+ className.includes('checkbox')) {
877
+ return true;
878
+ }
879
+
880
+ // Has click handlers (iOS) or meaningful accessibility info
881
+ if (locator.enabled !== false && (locator.text || locator.accessibilityLabel || locator.name)) {
882
+ return true;
883
+ }
884
+
885
+ return false;
886
+ }
887
+
888
+ /**
889
+ * โญ Calculate Element Priority for Caching Order
890
+ */
891
+ calculateElementPriority(locator) {
892
+ let priority = 0;
893
+
894
+ // Clickable elements get higher priority
895
+ if (this.isElementClickable(locator)) priority += 10;
896
+
897
+ // Elements with text get priority
898
+ if (locator.hasText) priority += 5;
899
+
900
+ // Elements with IDs get priority
901
+ if (locator.resourceId || locator.accessibilityId) priority += 3;
902
+
903
+ // Reasonable size elements
904
+ if (locator.area > 100 && locator.area < 10000) priority += 2;
905
+
906
+ return priority;
907
+ }
908
+
909
+ /**
910
+ * ๐ŸŽฏ Build Optimal Locator Strategy
911
+ */
912
+ buildOptimalLocator(element) {
913
+ const locators = [];
914
+
915
+ // Priority 1: Accessibility ID (most reliable)
916
+ if (element.accessibilityId) {
917
+ locators.push({
918
+ strategy: 'accessibility_id',
919
+ selector: element.accessibilityId,
920
+ priority: 1
921
+ });
922
+ }
923
+
924
+ // Priority 2: Resource ID (Android)
925
+ if (element.resourceId) {
926
+ locators.push({
927
+ strategy: 'id',
928
+ selector: element.resourceId,
929
+ priority: 1
930
+ });
931
+ }
932
+
933
+ // Priority 3: XPath (if available and short)
934
+ if (element.xpath && element.xpath.length < 200) {
935
+ locators.push({
936
+ strategy: 'xpath',
937
+ selector: element.xpath,
938
+ priority: 2
939
+ });
940
+ }
941
+
942
+ // Priority 4: Text-based (if unique)
943
+ if (element.text && element.text.length > 2) {
944
+ const decodedText = this.decodeHtmlEntities(element.text);
945
+ locators.push({
946
+ strategy: 'text',
947
+ selector: decodedText,
948
+ priority: 3
949
+ });
950
+ }
951
+
952
+ // Priority 5: Class name + index
953
+ if (element.className) {
954
+ locators.push({
955
+ strategy: 'class_name',
956
+ selector: element.className,
957
+ index: element.index || 0,
958
+ priority: 4
959
+ });
960
+ }
961
+
962
+ // Sort by priority and return the best strategy
963
+ locators.sort((a, b) => a.priority - b.priority);
964
+
965
+ return locators.length > 0 ? locators[0] : null;
966
+ }
967
+
968
+ /**
969
+ * ๐ŸŽฏ MAIN EXECUTION METHOD: Apply Intelligent Locator on Follower
970
+ */
971
+ async executeIntelligentAction(followerDevice, intelligentLocator, action = 'tap') {
972
+ this.logger.info(`๐ŸŽฏ Executing ${action} on ${followerDevice.name} using intelligent locator`);
973
+
974
+ // ENHANCED: Special handling for URL links that often fail with locators
975
+ const isUrlLink = intelligentLocator.locator &&
976
+ intelligentLocator.locator.strategy === 'text' &&
977
+ intelligentLocator.locator.selector &&
978
+ (intelligentLocator.locator.selector.toLowerCase().includes('learn more') ||
979
+ intelligentLocator.locator.selector.toLowerCase().includes('coverage') ||
980
+ intelligentLocator.locator.selector.toLowerCase().includes('http') ||
981
+ intelligentLocator.locator.selector.includes('...'));
982
+
983
+ if (isUrlLink) {
984
+ this.logger.warn(`๐Ÿ”— URL link detected: "${intelligentLocator.locator.selector}"`);
985
+ this.logger.warn('๐Ÿ’ก URL links often fail with locators - using coordinate fallback for reliability');
986
+
987
+ // For URL links, skip locator attempts and go straight to coordinates
988
+ if (intelligentLocator.fallbackCoords || intelligentLocator.method === 'coordinates') {
989
+ const coords = intelligentLocator.fallbackCoords || intelligentLocator;
990
+ const success = await this.executeCoordinateAction(followerDevice, coords, action);
991
+
992
+ return {
993
+ success: success,
994
+ method: 'coordinates',
995
+ reason: 'URL link - forced coordinate fallback for reliability',
996
+ coordinates: coords
997
+ };
998
+ }
999
+ }
1000
+
1001
+ try {
1002
+ if (intelligentLocator.method === 'locator' && intelligentLocator.locator && !isUrlLink) {
1003
+ // Try locator-based execution first (except for URL links)
1004
+ const success = await this.executeLocatorAction(followerDevice, intelligentLocator.locator, action);
1005
+
1006
+ if (success) {
1007
+ this.logger.info(`โœ… Locator-based ${action} succeeded on ${followerDevice.name}`);
1008
+ return { success: true, method: 'locator' };
1009
+ } else {
1010
+ this.logger.warn(`โš ๏ธ Locator failed, falling back to coordinates on ${followerDevice.name}`);
1011
+ }
1012
+ }
1013
+
1014
+ // Fallback to coordinate-based execution
1015
+ if (intelligentLocator.fallbackCoords || intelligentLocator.method === 'coordinates') {
1016
+ const coords = intelligentLocator.fallbackCoords || intelligentLocator;
1017
+ const success = await this.executeCoordinateAction(followerDevice, coords, action);
1018
+
1019
+ return {
1020
+ success: success,
1021
+ method: 'coordinates',
1022
+ coordinates: coords
1023
+ };
1024
+ }
1025
+
1026
+ throw new Error('No valid execution method available');
1027
+
1028
+ } catch (error) {
1029
+ this.logger.error(`โŒ Intelligent action failed: ${error.message}`);
1030
+ return { success: false, error: error.message };
1031
+ }
1032
+ }
1033
+
1034
+ /**
1035
+ * ๐ŸŽฏ Execute Action using Locator
1036
+ */
1037
+ async executeLocatorAction(device, locator, action) {
1038
+ const port = device.port || 8100;
1039
+ const platform = device.platform || 'ios';
1040
+
1041
+ try {
1042
+ if (platform.toLowerCase() === 'ios') {
1043
+ return await this.executeIOSLocatorAction(device, locator, action, port);
1044
+ } else {
1045
+ return await this.executeAndroidLocatorAction(device, locator, action, port);
1046
+ }
1047
+ } catch (error) {
1048
+ this.logger.error(`Locator action failed: ${error.message}`);
1049
+ return false;
1050
+ }
1051
+ }
1052
+
1053
+ /**
1054
+ * ๐Ÿ”„ Get Fresh Session ID with Universal Session Manager (UPDATED)
1055
+ * Uses the universal session manager for consistent session handling
1056
+ */
1057
+ async getFreshSessionId(deviceName, validateWithSourceEndpoint = false) {
1058
+ try {
1059
+ this.logger.debug(`[${deviceName}] Getting session via Universal Session Manager...`);
1060
+
1061
+ // REVOLUTIONARY: Use Universal Session Manager
1062
+ const sessionId = await this.universalSessionManager.getSessionWithRetry(deviceName);
1063
+
1064
+ if (sessionId) {
1065
+ this.logger.debug(`โœ… Session retrieved via Universal Manager: ${sessionId.substring(0, 8)}... for ${deviceName}`);
1066
+ return sessionId;
1067
+ }
1068
+
1069
+ throw new Error(`No session available via Universal Session Manager`);
1070
+
1071
+ } catch (error) {
1072
+ this.logger.error(`โŒ Universal session lookup failed for ${deviceName}: ${error.message}`);
1073
+ throw error;
1074
+ }
1075
+ }
1076
+
1077
+ /**
1078
+ * ๐Ÿฉบ Check if device disconnect should trigger immediate cleanup (DISABLED)
1079
+ */
1080
+ async checkDeviceDisconnection(deviceName, error) {
1081
+ // DISABLED: Was too aggressive and cleaning up healthy devices
1082
+ // TODO: Fix health check logic before re-enabling
1083
+ /*
1084
+ const disconnectionKeywords = [
1085
+ 'ECONNREFUSED', 'ENOTFOUND', 'ETIMEDOUT',
1086
+ 'Connection refused', 'connect ECONNREFUSED',
1087
+ 'Session does not exist', 'invalid session id'
1088
+ ];
1089
+
1090
+ const isDisconnectionError = disconnectionKeywords.some(keyword =>
1091
+ error.message?.includes(keyword)
1092
+ );
1093
+
1094
+ if (isDisconnectionError && global.performDeviceHealthCheck) {
1095
+ this.logger.warn(`๐Ÿฉบ Device ${deviceName} shows disconnection signs, triggering immediate health check...`);
1096
+
1097
+ setTimeout(async () => {
1098
+ try {
1099
+ await global.performDeviceHealthCheck();
1100
+ } catch (healthError) {
1101
+ this.logger.error(`Health check failed: ${healthError.message}`);
1102
+ }
1103
+ }, 1000);
1104
+ }
1105
+ */
1106
+
1107
+ // Just log the error for now without triggering cleanup
1108
+ this.logger.debug(`Device ${deviceName} session error (auto-cleanup disabled): ${error.message}`);
1109
+ }
1110
+
1111
+ /**
1112
+ * ๐Ÿ“ฑ Execute iOS Locator Action (Updated with Fresh Session Management)
1113
+ */
1114
+ async executeIOSLocatorAction(device, locator, action, port) {
1115
+ // REVOLUTIONARY: Use universal session manager for consistent sessions
1116
+ let sessionId;
1117
+ try {
1118
+ sessionId = await this.universalSessionManager.getSessionWithRetry(device.name, port);
1119
+ this.logger.debug(`โœ… Using universal session: ${sessionId.substring(0, 8)}... for ${device.name}`);
1120
+ } catch (sessionError) {
1121
+ this.logger.warn(`โš ๏ธ Universal session failed: ${sessionError.message}`);
1122
+ throw new Error('No valid iOS session available');
1123
+ }
1124
+
1125
+ const baseUrl = `http://localhost:${port}`;
1126
+
1127
+ try {
1128
+ let elementId = null;
1129
+
1130
+ // Find element using the best locator strategy
1131
+ switch (locator.strategy) {
1132
+ case 'accessibility_id':
1133
+ const accessResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1134
+ using: 'accessibility id',
1135
+ value: locator.selector
1136
+ });
1137
+ elementId = accessResponse.data.value.ELEMENT;
1138
+ break;
1139
+
1140
+ case 'xpath':
1141
+ const xpathResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1142
+ using: 'xpath',
1143
+ value: locator.selector
1144
+ });
1145
+ elementId = xpathResponse.data.value.ELEMENT;
1146
+ break;
1147
+
1148
+ case 'text':
1149
+ // Try multiple iOS text matching strategies
1150
+ const decodedText = this.decodeHtmlEntities(locator.selector);
1151
+ try {
1152
+ // Strategy 1: Accessibility label (most common for iOS buttons)
1153
+ const labelResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1154
+ using: 'accessibility id',
1155
+ value: decodedText
1156
+ });
1157
+ elementId = labelResponse.data.value.ELEMENT;
1158
+ } catch {
1159
+ try {
1160
+ // Strategy 2: XPath with text content for iOS (improved for links)
1161
+ const xpathResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1162
+ using: 'xpath',
1163
+ value: `//*[@name="${decodedText}" or @label="${decodedText}" or @value="${decodedText}" or contains(@name, "${decodedText}")]`
1164
+ });
1165
+ elementId = xpathResponse.data.value.ELEMENT;
1166
+ } catch {
1167
+ try {
1168
+ // Strategy 3: Predicate string (iOS-specific, improved)
1169
+ const predicateResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1170
+ using: '-ios predicate string',
1171
+ value: `name CONTAINS "${decodedText}" OR label CONTAINS "${decodedText}" OR value CONTAINS "${decodedText}"`
1172
+ });
1173
+ elementId = predicateResponse.data.value.ELEMENT;
1174
+ } catch {
1175
+ try {
1176
+ // Strategy 4: Class chain for buttons (original)
1177
+ const classChainResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1178
+ using: '-ios class chain',
1179
+ value: `**/XCUIElementTypeButton[\`name == "${decodedText}" OR label == "${decodedText}"\`]`
1180
+ });
1181
+ elementId = classChainResponse.data.value.ELEMENT;
1182
+ } catch {
1183
+ // Strategy 5: Class chain for links and text elements (NEW - specifically for URLs)
1184
+ const linkChainResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1185
+ using: '-ios class chain',
1186
+ value: `**/XCUIElementTypeLink[\`name CONTAINS "${decodedText}" OR label CONTAINS "${decodedText}"\`]`
1187
+ });
1188
+ if (linkChainResponse.data?.value?.ELEMENT) {
1189
+ elementId = linkChainResponse.data.value.ELEMENT;
1190
+ } else {
1191
+ // Strategy 6: Generic text elements (for text that might be clickable)
1192
+ const textChainResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1193
+ using: '-ios class chain',
1194
+ value: `**/XCUIElementTypeStaticText[\`name CONTAINS "${decodedText}" OR label CONTAINS "${decodedText}"\`]`
1195
+ });
1196
+ elementId = textChainResponse.data.value.ELEMENT;
1197
+ }
1198
+ }
1199
+ }
1200
+ }
1201
+ }
1202
+ break;
1203
+
1204
+ case 'class_name':
1205
+ const classResponse = await axios.post(`${baseUrl}/session/${sessionId}/elements`, {
1206
+ using: 'class name',
1207
+ value: locator.selector
1208
+ });
1209
+ const elements = classResponse.data.value;
1210
+ if (elements && elements.length > locator.index) {
1211
+ elementId = elements[locator.index].ELEMENT;
1212
+ }
1213
+ break;
1214
+ }
1215
+
1216
+ if (!elementId) {
1217
+ throw new Error('Element not found with locator strategy');
1218
+ }
1219
+
1220
+ // Execute action on found element
1221
+ switch (action) {
1222
+ case 'tap':
1223
+ case 'click':
1224
+ await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/click`);
1225
+ break;
1226
+ case 'sendKeys':
1227
+ await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/value`, {
1228
+ value: [locator.text || '']
1229
+ });
1230
+ break;
1231
+ }
1232
+
1233
+ // ๐ŸŽฏ Invalidate cache after successful iOS locator action (screen likely changed)
1234
+ this.invalidateCache(device.name);
1235
+
1236
+ return true;
1237
+
1238
+ } catch (error) {
1239
+ this.logger.warn(`iOS locator action failed: ${error.message}`);
1240
+ return false;
1241
+ }
1242
+ }
1243
+
1244
+ /**
1245
+ * ๐Ÿค– Execute Android Locator Action (Updated with Fresh Session Management)
1246
+ */
1247
+ async executeAndroidLocatorAction(device, locator, action, port) {
1248
+ // Get fresh session ID
1249
+ const sessionId = await this.getFreshSessionId(device.name);
1250
+ if (!sessionId) {
1251
+ throw new Error('No valid Android session available');
1252
+ }
1253
+
1254
+ const baseUrl = `http://localhost:${port}/wd/hub`;
1255
+
1256
+ try {
1257
+ let elementId = null;
1258
+
1259
+ // Find element using the best locator strategy
1260
+ switch (locator.strategy) {
1261
+ case 'id':
1262
+ const idResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1263
+ using: 'id',
1264
+ value: locator.selector
1265
+ });
1266
+ elementId = idResponse.data.value.ELEMENT;
1267
+ break;
1268
+
1269
+ case 'accessibility_id':
1270
+ const accessResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1271
+ using: 'accessibility id',
1272
+ value: locator.selector
1273
+ });
1274
+ elementId = accessResponse.data.value.ELEMENT;
1275
+ break;
1276
+
1277
+ case 'xpath':
1278
+ const xpathResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1279
+ using: 'xpath',
1280
+ value: locator.selector
1281
+ });
1282
+ elementId = xpathResponse.data.value.ELEMENT;
1283
+ break;
1284
+
1285
+ case 'text':
1286
+ const textResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
1287
+ using: 'partial link text',
1288
+ value: locator.selector
1289
+ });
1290
+ elementId = textResponse.data.value.ELEMENT;
1291
+ break;
1292
+
1293
+ case 'class_name':
1294
+ const classResponse = await axios.post(`${baseUrl}/session/${sessionId}/elements`, {
1295
+ using: 'class name',
1296
+ value: locator.selector
1297
+ });
1298
+ const elements = classResponse.data.value;
1299
+ if (elements && elements.length > locator.index) {
1300
+ elementId = elements[locator.index].ELEMENT;
1301
+ }
1302
+ break;
1303
+ }
1304
+
1305
+ if (!elementId) {
1306
+ throw new Error('Element not found with locator strategy');
1307
+ }
1308
+
1309
+ // Execute action on found element
1310
+ switch (action) {
1311
+ case 'tap':
1312
+ case 'click':
1313
+ await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/click`);
1314
+ break;
1315
+ case 'sendKeys':
1316
+ await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/value`, {
1317
+ value: [locator.text || '']
1318
+ });
1319
+ break;
1320
+ }
1321
+
1322
+ // ๐ŸŽฏ Invalidate cache after successful Android locator action (screen likely changed)
1323
+ this.invalidateCache(device.name);
1324
+
1325
+ return true;
1326
+
1327
+ } catch (error) {
1328
+ this.logger.warn(`Android locator action failed: ${error.message}`);
1329
+ return false;
1330
+ }
1331
+ }
1332
+
1333
+ /**
1334
+ * ๐Ÿ“ Execute Coordinate Action (Fallback) - Optimized Performance
1335
+ */
1336
+ async executeCoordinateAction(device, coordinates, action) {
1337
+ const port = device.port || 8100;
1338
+ const platform = device.platform || 'ios';
1339
+
1340
+ // ๐ŸŽฏ FAST COORDINATE PROCESSING: Only scale if obviously wrong
1341
+ let scaledX = coordinates.x;
1342
+ let scaledY = coordinates.y;
1343
+
1344
+ // Quick check: Only scale if coordinates are suspiciously small (< 50 pixels)
1345
+ if (scaledX < 50 || scaledY < 100) {
1346
+ // These are likely incorrectly scaled - apply 3x scaling
1347
+ scaledX = scaledX * 3;
1348
+ scaledY = scaledY * 3;
1349
+ this.logger.info(`๐Ÿ”ง Quick-scaled tiny coordinates (${coordinates.x}, ${coordinates.y}) โ†’ (${scaledX}, ${scaledY})`);
1350
+ }
1351
+
1352
+ // Fast bounds check - only clamp if outside reasonable limits
1353
+ if (scaledX > 500) scaledX = 450;
1354
+ if (scaledY > 1000) scaledY = 900;
1355
+ if (scaledX < 5) scaledX = 10;
1356
+ if (scaledY < 20) scaledY = 30;
1357
+
1358
+ try {
1359
+ if (platform.toLowerCase() === 'ios') {
1360
+ // REVOLUTIONARY: Use universal session manager for coordinate actions
1361
+ let sessionId;
1362
+ try {
1363
+ sessionId = await this.universalSessionManager.getSessionWithRetry(device.name, port);
1364
+ this.logger.debug(`โœ… Using universal session for coordinates: ${sessionId.substring(0, 8)}... for ${device.name}`);
1365
+ } catch (sessionError) {
1366
+ this.logger.warn(`โš ๏ธ Universal session failed for coordinates: ${sessionError.message}`);
1367
+ throw new Error('No valid iOS session available for coordinate action');
1368
+ }
1369
+
1370
+ // Use WDA tap endpoint for iOS
1371
+ await axios.post(`http://localhost:${port}/session/${sessionId}/wda/tap`, {
1372
+ x: scaledX,
1373
+ y: scaledY
1374
+ }, { timeout: 5000 });
1375
+
1376
+ } else {
1377
+ // REVOLUTIONARY: Use universal session manager for Android too
1378
+ let sessionId;
1379
+ try {
1380
+ sessionId = await this.universalSessionManager.getSessionWithRetry(device.name, port);
1381
+ } catch (sessionError) {
1382
+ throw new Error('No valid Android session available for coordinate action');
1383
+ }
1384
+
1385
+ await axios.post(`http://localhost:${port}/wd/hub/session/${sessionId}/touch/click`, {
1386
+ element: { x: scaledX, y: scaledY }
1387
+ }, { timeout: 5000 });
1388
+ }
1389
+
1390
+ this.logger.info(`โœ… Coordinate action successful on ${device.name} at (${scaledX}, ${scaledY})`);
1391
+
1392
+ // ๐ŸŽฏ Invalidate cache after successful coordinate action (screen likely changed)
1393
+ this.invalidateCache(device.name);
1394
+
1395
+ return true;
1396
+
1397
+ } catch (error) {
1398
+ // If 404 error, try session recovery once
1399
+ if (error.response?.status === 404) {
1400
+ this.logger.warn(`โš ๏ธ Session validation failed for ${device.name}: ${error.message}`);
1401
+
1402
+ try {
1403
+ // Try to get a fresh session directly from WDA status
1404
+ const statusRes = await axios.get(`http://localhost:${port}/status`, { timeout: 3000 });
1405
+ const freshSessionId = statusRes.data?.sessionId;
1406
+
1407
+ if (freshSessionId) {
1408
+ this.logger.info(`๐Ÿ”„ Found different session after validation failure: ${freshSessionId.substring(0, 8)}...`);
1409
+
1410
+ // Update universal session manager port mapping
1411
+ this.universalSessionManager.setDevicePort(device.name, port);
1412
+
1413
+ // Retry the coordinate action with fresh session
1414
+ if (platform.toLowerCase() === 'ios') {
1415
+ await axios.post(`http://localhost:${port}/session/${freshSessionId}/wda/tap`, {
1416
+ x: coordinates.x,
1417
+ y: coordinates.y
1418
+ }, { timeout: 5000 });
1419
+ } else {
1420
+ await axios.post(`http://localhost:${port}/wd/hub/session/${freshSessionId}/touch/click`, {
1421
+ element: { x: coordinates.x, y: coordinates.y }
1422
+ }, { timeout: 5000 });
1423
+ }
1424
+
1425
+ this.logger.info(`โœ… Coordinate action successful on ${device.name} at (${coordinates.x}, ${coordinates.y}) after session recovery`);
1426
+
1427
+ // ๐ŸŽฏ Invalidate cache after successful coordinate recovery action (screen likely changed)
1428
+ this.invalidateCache(device.name);
1429
+
1430
+ return true;
1431
+ }
1432
+ } catch (recoveryError) {
1433
+ this.logger.error(`Failed to recover session for ${device.name}: ${recoveryError.message}`);
1434
+ }
1435
+ }
1436
+
1437
+ this.logger.error(`Coordinate action failed: ${error.message}`);
1438
+ return false;
1439
+ }
1440
+ }
1441
+
1442
+ /**
1443
+ * ๐Ÿ“Š Get Performance Metrics
1444
+ */
1445
+ getPerformanceMetrics() {
1446
+ const total = this.performanceMetrics.cacheHits + this.performanceMetrics.cacheMisses;
1447
+ const cacheHitRate = total > 0 ? (this.performanceMetrics.cacheHits / total * 100).toFixed(1) : 0;
1448
+
1449
+ return {
1450
+ ...this.performanceMetrics,
1451
+ cacheHitRate: `${cacheHitRate}%`,
1452
+ totalRequests: total,
1453
+ accuracyRate: `${((this.performanceMetrics.accurateMatches / (this.performanceMetrics.accurateMatches + this.performanceMetrics.fallbackToCoordinates)) * 100).toFixed(1)}%`
1454
+ };
1455
+ }
1456
+
1457
+ /**
1458
+ * ๐Ÿ”„ Convert elements array to parseable page source format
1459
+ */
1460
+ convertElementsToPageSource(elements, port, sessionId) {
1461
+ // This is a fallback method to create a basic parseable structure
1462
+ // when proper page source endpoints fail
1463
+ let xmlContent = '<?xml version="1.0" encoding="UTF-8"?>\n<hierarchy>\n';
1464
+
1465
+ elements.forEach((element, index) => {
1466
+ try {
1467
+ // Get element details if possible
1468
+ const elementId = element.ELEMENT;
1469
+ // For now, just create a basic structure - this would need enhancement
1470
+ // to fetch actual element properties via separate API calls
1471
+ xmlContent += ` <XCUIElementTypeButton name="" label="" x="0" y="0" width="100" height="50" />\n`;
1472
+ } catch (error) {
1473
+ // Skip problematic elements
1474
+ }
1475
+ });
1476
+
1477
+ xmlContent += '</hierarchy>';
1478
+ return xmlContent;
1479
+ }
1480
+
1481
+ /**
1482
+ * ๐Ÿงน Clear Cache (for testing or troubleshooting)
1483
+ */
1484
+ clearCache() {
1485
+ this.locatorCache.clear();
1486
+ this.cacheTimestamps.clear();
1487
+ this.logger.info('๐Ÿงน Locator cache cleared');
1488
+ }
1489
+
1490
+ /**
1491
+ * ๐Ÿ” Check if device has cached locators
1492
+ */
1493
+ hasCachedLocators(deviceName) {
1494
+ return this.locatorCache.has(deviceName) && this.locatorCache.get(deviceName).length > 0;
1495
+ }
1496
+
1497
+ /**
1498
+ * ๐ŸŽฏ Invalidate Cache for Specific Device (after screen changes)
1499
+ */
1500
+ invalidateCache(deviceName) {
1501
+ const hadCache = this.locatorCache.has(deviceName);
1502
+ const cacheSize = hadCache ? this.locatorCache.get(deviceName).length : 0;
1503
+
1504
+ this.locatorCache.delete(deviceName);
1505
+ this.cacheTimestamps.delete(deviceName);
1506
+
1507
+ if (hadCache) {
1508
+ this.logger.info(`๐Ÿงน Locator cache invalidated for ${deviceName} (had ${cacheSize} locators)`);
1509
+ } else {
1510
+ this.logger.debug(`๐Ÿงน Locator cache invalidated for ${deviceName} (was empty)`);
1511
+ }
1512
+
1513
+ // Also remove from refreshing set to allow immediate refresh
1514
+ this.refreshingDevices.delete(deviceName);
1515
+ }
1516
+
1517
+ /**
1518
+ * ๐Ÿ• Get Cache Age in milliseconds
1519
+ * Returns null if no cache exists, or age in ms since last refresh
1520
+ */
1521
+ getCacheAge(deviceName) {
1522
+ const timestamp = this.cacheTimestamps.get(deviceName);
1523
+ if (!timestamp) {
1524
+ return null; // No cache exists
1525
+ }
1526
+
1527
+ return Date.now() - timestamp;
1528
+ }
1529
+
1530
+ /**
1531
+ * ๐ŸŽฏ Invalidate All Device Caches (after major screen changes)
1532
+ */
1533
+ invalidateAllCaches() {
1534
+ const deviceCount = this.locatorCache.size;
1535
+ this.locatorCache.clear();
1536
+ this.cacheTimestamps.clear();
1537
+ this.logger.info(`๐Ÿงน All locator caches invalidated (${deviceCount} devices)`);
1538
+ }
1539
+ }
1540
+
1541
+ module.exports = IntelligentLocatorService;