devicely 2.2.12 โ 2.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +182 -81
- package/bin/devicely.js +1 -1
- package/config/devices.conf +2 -2
- package/lib/.logging-backup/aiProviders.js.backup +654 -0
- package/lib/.logging-backup/appMappings.js.backup +337 -0
- package/lib/.logging-backup/commanderService.js.backup +4427 -0
- package/lib/.logging-backup/devices.js.backup +54 -0
- package/lib/.logging-backup/doctor.js.backup +94 -0
- package/lib/.logging-backup/encryption.js.backup +61 -0
- package/lib/.logging-backup/executor.js.backup +104 -0
- package/lib/.logging-backup/hybridAI.js.backup +154 -0
- package/lib/.logging-backup/intelligentLocatorService.js.backup +1541 -0
- package/lib/.logging-backup/locatorStrategy.js.backup +342 -0
- package/lib/.logging-backup/scriptLoader.js.backup +13 -0
- package/lib/.logging-backup/server.js.backup +6298 -0
- package/lib/.logging-backup/tensorflowAI.js.backup +714 -0
- package/lib/.logging-backup/universalSessionManager.js.backup +370 -0
- package/lib/.logging-enhanced-backup/server.js.enhanced-backup +6298 -0
- package/lib/advanced-logger.js +1 -0
- package/lib/aiProviders.js +154 -15
- package/lib/aiProviders.js.strategic-backup +657 -0
- package/lib/aiProvidersConfig.js +61 -151
- package/lib/aiProvidersConfig.js.backup +218 -0
- package/lib/androidDeviceDetection.js +1 -1
- package/lib/appMappings.js +1 -1
- package/lib/commanderService.js +1 -1
- package/lib/commanderService.js.backup +5552 -0
- package/lib/deviceDetection.js +1 -1
- package/lib/devices.js +1 -1
- package/lib/devices.js.strategic-backup +57 -0
- package/lib/doctor.js +1 -1
- package/lib/encryption.js +1 -1
- package/lib/encryption.js.strategic-backup +61 -0
- package/lib/executor.js +1 -1
- package/lib/executor.js.strategic-backup +107 -0
- package/lib/frontend/asset-manifest.json +5 -3
- package/lib/frontend/index.html +1 -1
- package/lib/hybridAI.js +1 -0
- package/lib/intelligentLocatorService.js +1 -0
- package/lib/lightweightAI.js +1 -0
- package/lib/localBuiltInAI.js +1 -0
- package/lib/localBuiltInAI_backup.js +1 -0
- package/lib/localBuiltInAI_simple.js +1 -0
- package/lib/locatorStrategy.js +1 -1
- package/lib/logger-demo.js +2 -0
- package/lib/logger-integration-examples.js +102 -0
- package/lib/logger.js +1 -1
- package/lib/package.json +5 -0
- package/lib/public/asset-manifest.json +3 -3
- package/lib/public/index.html +1 -1
- package/lib/quick-start-logger.js +2 -0
- package/lib/scriptLoader.js +1 -1
- package/lib/server.js +1 -1
- package/lib/server.js.strategic-backup +6298 -0
- package/lib/tensorflowAI.js +1 -0
- package/lib/tensorflowAI.js.strategic-backup +717 -0
- package/lib/tinyAI.js +1 -0
- package/lib/universalSessionManager.js +1 -0
- package/package.json +1 -1
- package/scripts/shell/android_device_control.enc +1 -1
- package/scripts/shell/connect_ios_usb_multi_final.enc +1 -1
- package/scripts/shell/connect_ios_wireless_multi_final.enc +1 -1
- package/lib/public/index.html.bak +0 -1
|
@@ -0,0 +1,1541 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ๐ฏ INTELLIGENT LOCATOR SERVICE - GENIUS SOLUTION
|
|
3
|
+
* ================================================
|
|
4
|
+
*
|
|
5
|
+
* This service implements a revolutionary locator-based Commander-Follower system:
|
|
6
|
+
* 1. Caches all screen locators when screen loads
|
|
7
|
+
* 2. Uses tiny AI models for intelligent locator matching
|
|
8
|
+
* 3. Converts coordinates to precise locators instantly
|
|
9
|
+
* 4. 100% accurate cross-device element targeting
|
|
10
|
+
* 5. Zero-cost, open-source, local processing
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const Logger = require('./logger');
|
|
14
|
+
const axios = require('axios');
|
|
15
|
+
const fs = require('fs').promises;
|
|
16
|
+
const path = require('path');
|
|
17
|
+
const universalSessionManager = require('./universalSessionManager');
|
|
18
|
+
|
|
19
|
+
class IntelligentLocatorService {
|
|
20
|
+
constructor(connectedDevices = null) {
|
|
21
|
+
this.logger = new Logger('IntelligentLocatorService');
|
|
22
|
+
|
|
23
|
+
// Store reference to connected devices
|
|
24
|
+
this.connectedDevices = connectedDevices;
|
|
25
|
+
|
|
26
|
+
// REVOLUTIONARY: Universal Session Management
|
|
27
|
+
this.universalSessionManager = universalSessionManager;
|
|
28
|
+
|
|
29
|
+
// Locator Cache Management - BALANCED for reliability and speed
|
|
30
|
+
this.locatorCache = new Map(); // Map<deviceName, CachedLocators>
|
|
31
|
+
this.cacheTimestamps = new Map(); // Map<deviceName, timestamp>
|
|
32
|
+
this.cacheValidityMs = 25000; // BALANCED: 25 seconds cache validity (was 20s, originally 30s)
|
|
33
|
+
this.refreshingDevices = new Set(); // Track devices currently being refreshed
|
|
34
|
+
|
|
35
|
+
// AI Model for Locator Matching
|
|
36
|
+
this.aiModel = null; // Will hold local tiny AI model
|
|
37
|
+
this.modelPath = path.join(__dirname, 'ai', 'locator_model');
|
|
38
|
+
|
|
39
|
+
// Element Similarity Scoring
|
|
40
|
+
this.similarityThreshold = 0.40; // 40% similarity threshold - more flexible
|
|
41
|
+
|
|
42
|
+
// Performance Metrics
|
|
43
|
+
this.performanceMetrics = {
|
|
44
|
+
cacheHits: 0,
|
|
45
|
+
cacheMisses: 0,
|
|
46
|
+
accurateMatches: 0,
|
|
47
|
+
fallbackToCoordinates: 0
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
this.initializeAIModel();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* ๐ง Decode HTML entities from text
|
|
55
|
+
*/
|
|
56
|
+
decodeHtmlEntities(text) {
|
|
57
|
+
if (!text) return text;
|
|
58
|
+
|
|
59
|
+
return text
|
|
60
|
+
.replace(/&/g, '&')
|
|
61
|
+
.replace(/</g, '<')
|
|
62
|
+
.replace(/>/g, '>')
|
|
63
|
+
.replace(/"/g, '"')
|
|
64
|
+
.replace(/'/g, "'")
|
|
65
|
+
.replace(/ /g, ' ');
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* ๐ค Initialize Local Tiny AI Model for Locator Intelligence
|
|
70
|
+
* Uses TensorFlow.js with a lightweight model for element classification
|
|
71
|
+
*/
|
|
72
|
+
async initializeAIModel() {
|
|
73
|
+
try {
|
|
74
|
+
// Check if we have TensorFlow.js available
|
|
75
|
+
let tf;
|
|
76
|
+
try {
|
|
77
|
+
tf = require('@tensorflow/tfjs-node');
|
|
78
|
+
} catch (error) {
|
|
79
|
+
this.logger.warn('๐ค TensorFlow.js not available, falling back to rule-based matching');
|
|
80
|
+
this.useRuleBasedMatching = true;
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Try to load existing model or create a simple one
|
|
85
|
+
const modelExists = await this.checkModelExists();
|
|
86
|
+
|
|
87
|
+
if (modelExists) {
|
|
88
|
+
this.aiModel = await tf.loadLayersModel(`file://${this.modelPath}/model.json`);
|
|
89
|
+
this.logger.info('๐ค โ
Loaded existing AI locator model');
|
|
90
|
+
} else {
|
|
91
|
+
// Create a simple neural network for element classification
|
|
92
|
+
this.aiModel = await this.createSimpleLocatorModel(tf);
|
|
93
|
+
this.logger.info('๐ค โ
Created new AI locator model');
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
} catch (error) {
|
|
97
|
+
this.logger.warn(`๐ค AI model initialization failed: ${error.message}, using rule-based matching`);
|
|
98
|
+
this.useRuleBasedMatching = true;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* ๐ง Create a Simple Locator Classification Model
|
|
104
|
+
* Lightweight model for element type and text similarity classification
|
|
105
|
+
*/
|
|
106
|
+
async createSimpleLocatorModel(tf) {
|
|
107
|
+
// Simple neural network for element feature matching
|
|
108
|
+
const model = tf.sequential({
|
|
109
|
+
layers: [
|
|
110
|
+
tf.layers.dense({ inputShape: [10], units: 16, activation: 'relu' }),
|
|
111
|
+
tf.layers.dropout({ rate: 0.2 }),
|
|
112
|
+
tf.layers.dense({ units: 8, activation: 'relu' }),
|
|
113
|
+
tf.layers.dense({ units: 3, activation: 'softmax' }) // 3 classes: button, text, input
|
|
114
|
+
]
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
model.compile({
|
|
118
|
+
optimizer: 'adam',
|
|
119
|
+
loss: 'categoricalCrossentropy',
|
|
120
|
+
metrics: ['accuracy']
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
return model;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
async checkModelExists() {
|
|
127
|
+
try {
|
|
128
|
+
await fs.access(path.join(this.modelPath, 'model.json'));
|
|
129
|
+
return true;
|
|
130
|
+
} catch {
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* ๐ฏ MAIN GENIUS METHOD: Convert Click Coordinates to Intelligent Locator
|
|
137
|
+
*/
|
|
138
|
+
async getIntelligentLocator(deviceName, x, y, forceRefresh = false) {
|
|
139
|
+
this.logger.info(`๐ฏ Finding intelligent locator for ${deviceName} at (${x}, ${y})`);
|
|
140
|
+
|
|
141
|
+
try {
|
|
142
|
+
// Step 1: Ensure we have fresh locator cache - FORCE REFRESH AFTER EVERY ACTION
|
|
143
|
+
const shouldForceRefresh = forceRefresh || this.shouldForceRefreshCache(deviceName);
|
|
144
|
+
|
|
145
|
+
if (shouldForceRefresh) {
|
|
146
|
+
this.logger.info(`๐ FORCING cache refresh for ${deviceName} (forceRefresh=${forceRefresh})`);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const cachedLocators = await this.getCachedLocators(deviceName, shouldForceRefresh);
|
|
150
|
+
|
|
151
|
+
if (!cachedLocators || cachedLocators.length === 0) {
|
|
152
|
+
this.logger.warn(`โ No cached locators for ${deviceName}, falling back to coordinates`);
|
|
153
|
+
this.performanceMetrics.fallbackToCoordinates++;
|
|
154
|
+
return { method: 'coordinates', x, y };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
this.logger.debug(`๐ฆ Using ${cachedLocators.length} cached locators for matching`);
|
|
158
|
+
|
|
159
|
+
// Step 2: Find best matching element using AI/rules
|
|
160
|
+
const bestMatch = await this.findBestMatchingElement(cachedLocators, x, y);
|
|
161
|
+
|
|
162
|
+
if (bestMatch && bestMatch.confidence > this.similarityThreshold) {
|
|
163
|
+
this.logger.info(`โ
High-confidence locator match found (${(bestMatch.confidence * 100).toFixed(1)}%)`);
|
|
164
|
+
this.performanceMetrics.accurateMatches++;
|
|
165
|
+
|
|
166
|
+
return {
|
|
167
|
+
method: 'locator',
|
|
168
|
+
locator: bestMatch.locator,
|
|
169
|
+
confidence: bestMatch.confidence,
|
|
170
|
+
element: bestMatch.element,
|
|
171
|
+
fallbackCoords: { x, y } // Always include fallback
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Step 3: Fallback to coordinate-based if no good match
|
|
176
|
+
this.logger.warn(`โ ๏ธ Low confidence match (${bestMatch ? (bestMatch.confidence * 100).toFixed(1) : 0}%), using coordinates`);
|
|
177
|
+
this.performanceMetrics.fallbackToCoordinates++;
|
|
178
|
+
|
|
179
|
+
return {
|
|
180
|
+
method: 'coordinates',
|
|
181
|
+
x,
|
|
182
|
+
y,
|
|
183
|
+
nearestElement: bestMatch ? bestMatch.element : null
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
} catch (error) {
|
|
187
|
+
this.logger.error(`โ Intelligent locator failed: ${error.message}`);
|
|
188
|
+
return { method: 'coordinates', x, y, error: error.message };
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* ๐ฏ Determine if cache should be forcefully refreshed - OPTIMIZED for speed
|
|
194
|
+
*/
|
|
195
|
+
shouldForceRefreshCache(deviceName) {
|
|
196
|
+
const now = Date.now();
|
|
197
|
+
const lastCached = this.cacheTimestamps.get(deviceName);
|
|
198
|
+
|
|
199
|
+
// PERFORMANCE: More aggressive caching for commander speed
|
|
200
|
+
// Force refresh only if:
|
|
201
|
+
// 1. Never cached
|
|
202
|
+
// 2. Cache is older than 15 seconds (increased from 10s for speed)
|
|
203
|
+
// 3. Cache was explicitly invalidated
|
|
204
|
+
|
|
205
|
+
if (!lastCached) {
|
|
206
|
+
this.logger.debug(`๐ Force refresh: ${deviceName} never cached`);
|
|
207
|
+
return true;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const cacheAge = now - lastCached;
|
|
211
|
+
if (cacheAge > 15000) { // OPTIMIZED: Increased from 10s to 15s for better performance
|
|
212
|
+
this.logger.debug(`๐ Force refresh: ${deviceName} cache is ${Math.round(cacheAge/1000)}s old`);
|
|
213
|
+
return true;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// Check if we have actual cached data
|
|
217
|
+
const cachedData = this.locatorCache.get(deviceName);
|
|
218
|
+
if (!cachedData || cachedData.length === 0) {
|
|
219
|
+
this.logger.debug(`๐ Force refresh: ${deviceName} has no cached data`);
|
|
220
|
+
return true;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
return false;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* ๐ฆ Get or Refresh Cached Locators for Device (ENHANCED)
|
|
228
|
+
*/
|
|
229
|
+
async getCachedLocators(deviceName, forceRefresh = false) {
|
|
230
|
+
const cacheKey = deviceName;
|
|
231
|
+
const now = Date.now();
|
|
232
|
+
const lastCached = this.cacheTimestamps.get(cacheKey);
|
|
233
|
+
|
|
234
|
+
// Check if cache is valid and not forcing refresh
|
|
235
|
+
if (!forceRefresh && lastCached && this.locatorCache.has(cacheKey)) {
|
|
236
|
+
const cacheAge = now - lastCached;
|
|
237
|
+
if (cacheAge < this.cacheValidityMs) {
|
|
238
|
+
this.logger.debug(`๐ฆ Using cached locators for ${deviceName} (age: ${Math.round(cacheAge/1000)}s)`);
|
|
239
|
+
this.performanceMetrics.cacheHits++;
|
|
240
|
+
return this.locatorCache.get(cacheKey);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// ๐ฏ ENHANCED: If cache was recently invalidated, always force fresh fetch
|
|
245
|
+
const wasRecentlyInvalidated = !lastCached || (now - lastCached) > 60000; // If older than 1 minute, was likely invalidated
|
|
246
|
+
const effectiveForceRefresh = forceRefresh || wasRecentlyInvalidated;
|
|
247
|
+
|
|
248
|
+
// ๐ฏ PREVENT DUPLICATE REFRESHES: If already refreshing this device, return cached data quickly
|
|
249
|
+
if (this.refreshingDevices.has(deviceName)) {
|
|
250
|
+
this.logger.debug(`โณ Device ${deviceName} is already being refreshed, using existing cache`);
|
|
251
|
+
// Don't wait - just return existing cache if available
|
|
252
|
+
const existingCache = this.locatorCache.get(cacheKey);
|
|
253
|
+
if (existingCache && existingCache.length > 0) {
|
|
254
|
+
this.logger.debug(`๐ฆ Using existing ${existingCache.length} locators for ${deviceName}`);
|
|
255
|
+
return existingCache;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// Mark as refreshing
|
|
260
|
+
this.refreshingDevices.add(deviceName);
|
|
261
|
+
this.logger.debug(`๐ Marked ${deviceName} as refreshing`);
|
|
262
|
+
|
|
263
|
+
// Refresh cache
|
|
264
|
+
const ageText = lastCached ? `${Math.round((now - lastCached)/1000)}s` : 'never cached';
|
|
265
|
+
this.logger.info(`๐ Refreshing locator cache for ${deviceName} (force: ${effectiveForceRefresh}, age: ${ageText})`);
|
|
266
|
+
this.performanceMetrics.cacheMisses++;
|
|
267
|
+
|
|
268
|
+
try {
|
|
269
|
+
const freshLocators = await this.fetchAllLocators(deviceName);
|
|
270
|
+
|
|
271
|
+
if (freshLocators && freshLocators.length > 0) {
|
|
272
|
+
this.locatorCache.set(cacheKey, freshLocators);
|
|
273
|
+
this.cacheTimestamps.set(cacheKey, now);
|
|
274
|
+
this.logger.info(`โ
Cached ${freshLocators.length} locators for ${deviceName}`);
|
|
275
|
+
return freshLocators;
|
|
276
|
+
} else {
|
|
277
|
+
this.logger.warn(`โ ๏ธ No locators found for ${deviceName}`);
|
|
278
|
+
return [];
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
} catch (error) {
|
|
282
|
+
this.logger.error(`โ Failed to refresh locators for ${deviceName}: ${error.message}`);
|
|
283
|
+
// Return stale cache if available, but mark it as potentially stale
|
|
284
|
+
const staleCache = this.locatorCache.get(cacheKey) || [];
|
|
285
|
+
if (staleCache.length > 0) {
|
|
286
|
+
this.logger.warn(`โ ๏ธ Using stale cache with ${staleCache.length} locators for ${deviceName}`);
|
|
287
|
+
return staleCache;
|
|
288
|
+
} else {
|
|
289
|
+
// If no cache available, return empty array but don't throw error
|
|
290
|
+
this.logger.warn(`โ ๏ธ No fallback cache available for ${deviceName}, returning empty cache`);
|
|
291
|
+
return [];
|
|
292
|
+
}
|
|
293
|
+
} finally {
|
|
294
|
+
// Always remove from refreshing set
|
|
295
|
+
this.refreshingDevices.delete(deviceName);
|
|
296
|
+
this.logger.debug(`๐ Removed ${deviceName} from refreshing set`);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* ๐ Fetch All Locators from Device (Enhanced Direct WDA Method)
|
|
302
|
+
*/
|
|
303
|
+
async fetchAllLocators(deviceName) {
|
|
304
|
+
try {
|
|
305
|
+
// For iOS devices, use direct WDA page source (faster and more reliable)
|
|
306
|
+
const device = this.connectedDevices?.find(d => d.name === deviceName);
|
|
307
|
+
if (!device) {
|
|
308
|
+
throw new Error(`Device ${deviceName} not found in connected devices`);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
if (device.platform === 'ios') {
|
|
312
|
+
return await this.fetchIOSLocatorsDirect(deviceName, device);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// For Android, use the existing API endpoint
|
|
316
|
+
const response = await axios.post('http://localhost:3001/api/locators', {
|
|
317
|
+
deviceName: deviceName
|
|
318
|
+
}, { timeout: 15000 });
|
|
319
|
+
|
|
320
|
+
if (response.data.success && response.data.locators) {
|
|
321
|
+
const locators = response.data.locators;
|
|
322
|
+
|
|
323
|
+
// Process and enhance locators with our intelligence
|
|
324
|
+
const enhancedLocators = locators.map(locator => ({
|
|
325
|
+
...locator,
|
|
326
|
+
// Add computed properties for matching
|
|
327
|
+
centerX: locator.bounds ? (locator.bounds.x + locator.bounds.width / 2) : null,
|
|
328
|
+
centerY: locator.bounds ? (locator.bounds.y + locator.bounds.height / 2) : null,
|
|
329
|
+
area: locator.bounds ? (locator.bounds.width * locator.bounds.height) : 0,
|
|
330
|
+
// Element classification
|
|
331
|
+
isClickable: this.isElementClickable(locator),
|
|
332
|
+
elementType: this.classifyElementType(locator),
|
|
333
|
+
// Text features for matching
|
|
334
|
+
hasText: !!(locator.text || locator.accessibilityLabel || locator.contentDesc),
|
|
335
|
+
textLength: (locator.text || locator.accessibilityLabel || locator.contentDesc || '').length,
|
|
336
|
+
// Priority scoring
|
|
337
|
+
priority: this.calculateElementPriority(locator)
|
|
338
|
+
}));
|
|
339
|
+
|
|
340
|
+
// Sort by priority (most likely to be interaction targets first)
|
|
341
|
+
enhancedLocators.sort((a, b) => b.priority - a.priority);
|
|
342
|
+
|
|
343
|
+
return enhancedLocators;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
return [];
|
|
347
|
+
|
|
348
|
+
} catch (error) {
|
|
349
|
+
this.logger.error(`Failed to fetch locators: ${error.message}`);
|
|
350
|
+
throw error;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* ๐ฑ Fetch iOS Locators Directly from WDA (NEW METHOD)
|
|
356
|
+
*/
|
|
357
|
+
async fetchIOSLocatorsDirect(deviceName, device) {
|
|
358
|
+
const port = device.port || 8100;
|
|
359
|
+
let attempts = 0;
|
|
360
|
+
const maxAttempts = 3;
|
|
361
|
+
|
|
362
|
+
while (attempts < maxAttempts) {
|
|
363
|
+
attempts++;
|
|
364
|
+
|
|
365
|
+
try {
|
|
366
|
+
// REVOLUTIONARY: Use universal session manager for consistent sessions
|
|
367
|
+
const sessionId = await this.universalSessionManager.getSessionWithRetry(deviceName, port);
|
|
368
|
+
if (!sessionId) {
|
|
369
|
+
throw new Error(`No valid session for iOS device ${deviceName} on attempt ${attempts}`);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
this.logger.debug(`๐ฑ Fetching iOS locators directly from WDA for ${deviceName} (session: ${sessionId.substring(0, 8)}..., attempt: ${attempts})`);
|
|
373
|
+
|
|
374
|
+
// Try multiple WDA endpoints for getting page source
|
|
375
|
+
let response;
|
|
376
|
+
let pageSource;
|
|
377
|
+
|
|
378
|
+
// Method 1: Standard source endpoint
|
|
379
|
+
try {
|
|
380
|
+
response = await axios.get(`http://localhost:${port}/session/${sessionId}/source`, {
|
|
381
|
+
timeout: 8000
|
|
382
|
+
});
|
|
383
|
+
pageSource = response.data?.value;
|
|
384
|
+
} catch (error) {
|
|
385
|
+
this.logger.warn(`๐ฑ Standard source endpoint failed: ${error.message}`);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// Method 2: If first method fails, try WDA-specific source endpoint
|
|
389
|
+
if (!pageSource) {
|
|
390
|
+
try {
|
|
391
|
+
response = await axios.get(`http://localhost:${port}/session/${sessionId}/wda/source`, {
|
|
392
|
+
timeout: 8000
|
|
393
|
+
});
|
|
394
|
+
pageSource = response.data?.value;
|
|
395
|
+
} catch (error) {
|
|
396
|
+
this.logger.warn(`๐ฑ WDA source endpoint also failed: ${error.message}`);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
// Method 4: Try getting all elements and extracting from that
|
|
401
|
+
if (!pageSource) {
|
|
402
|
+
try {
|
|
403
|
+
response = await axios.get(`http://localhost:${port}/session/${sessionId}/elements`, {
|
|
404
|
+
timeout: 8000,
|
|
405
|
+
params: { using: 'xpath', value: '//*' }
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
if (response.data?.value && Array.isArray(response.data.value)) {
|
|
409
|
+
// Convert elements array to simple XML-like structure for parsing
|
|
410
|
+
pageSource = this.convertElementsToPageSource(response.data.value, port, sessionId);
|
|
411
|
+
}
|
|
412
|
+
} catch (error) {
|
|
413
|
+
this.logger.warn(`๐ฑ Elements endpoint also failed: ${error.message}`);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// Method 5: Fallback to a basic hierarchical page source
|
|
418
|
+
if (!pageSource) {
|
|
419
|
+
try {
|
|
420
|
+
response = await axios.get(`http://localhost:${port}/session/${sessionId}/wda/element/0/tree`, {
|
|
421
|
+
timeout: 8000
|
|
422
|
+
});
|
|
423
|
+
pageSource = JSON.stringify(response.data);
|
|
424
|
+
} catch (error) {
|
|
425
|
+
this.logger.warn(`๐ฑ Element tree with root also failed: ${error.message}`);
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
if (!pageSource) {
|
|
430
|
+
if (attempts < maxAttempts) {
|
|
431
|
+
this.logger.warn(`๐ฑ All source methods failed for ${deviceName}, attempt ${attempts}/${maxAttempts}. Retrying...`);
|
|
432
|
+
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait 1s before retry
|
|
433
|
+
continue;
|
|
434
|
+
} else {
|
|
435
|
+
throw new Error('No page source returned from any WDA endpoint after all attempts');
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
const parsedElements = this.parseIOSPageSource(pageSource);
|
|
440
|
+
|
|
441
|
+
this.logger.info(`โ
Parsed ${parsedElements.length} iOS locators directly from WDA (attempt ${attempts})`);
|
|
442
|
+
|
|
443
|
+
return parsedElements;
|
|
444
|
+
|
|
445
|
+
} catch (error) {
|
|
446
|
+
if (attempts < maxAttempts) {
|
|
447
|
+
this.logger.warn(`๐ฑ iOS locator fetch attempt ${attempts}/${maxAttempts} failed: ${error.message}. Retrying...`);
|
|
448
|
+
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait 1s before retry
|
|
449
|
+
} else {
|
|
450
|
+
throw error;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
throw new Error(`Failed to fetch iOS locators after ${maxAttempts} attempts`);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* ๐ Parse iOS Page Source to Extract Locators
|
|
460
|
+
*/
|
|
461
|
+
parseIOSPageSource(pageSource) {
|
|
462
|
+
const elements = [];
|
|
463
|
+
|
|
464
|
+
// More comprehensive regex to catch interactive elements
|
|
465
|
+
const xmlRegex = /<XCUIElementType(Button|TextField|SecureTextField|Switch|Link|StaticText|SearchField|Slider|Stepper|DatePicker|PickerWheel|PageIndicator|SegmentedControl|TabBar|NavigationBar|Cell|Image|TextView|WebView)[^>]*>/g;
|
|
466
|
+
|
|
467
|
+
let match;
|
|
468
|
+
while ((match = xmlRegex.exec(pageSource)) !== null) {
|
|
469
|
+
const elementTag = match[0];
|
|
470
|
+
|
|
471
|
+
// Extract attributes with better parsing
|
|
472
|
+
const attributes = this.parseElementAttributes(elementTag);
|
|
473
|
+
|
|
474
|
+
if (attributes.x !== undefined && attributes.y !== undefined &&
|
|
475
|
+
attributes.width !== undefined && attributes.height !== undefined) {
|
|
476
|
+
|
|
477
|
+
// More permissive filtering for iOS elements
|
|
478
|
+
if (attributes.width > 5 && attributes.height > 5 && attributes.enabled) {
|
|
479
|
+
|
|
480
|
+
const elementType = `XCUIElementType${match[1]}`;
|
|
481
|
+
const isInteractive = this.isInteractiveElement(elementType, attributes);
|
|
482
|
+
|
|
483
|
+
// Include both interactive elements and meaningful text elements
|
|
484
|
+
if (isInteractive || (attributes.label && attributes.label.length > 0)) {
|
|
485
|
+
elements.push({
|
|
486
|
+
type: elementType,
|
|
487
|
+
text: attributes.label || attributes.name,
|
|
488
|
+
accessibilityLabel: attributes.label,
|
|
489
|
+
name: attributes.name,
|
|
490
|
+
x: Math.round(attributes.x),
|
|
491
|
+
y: Math.round(attributes.y),
|
|
492
|
+
width: Math.round(attributes.width),
|
|
493
|
+
height: Math.round(attributes.height),
|
|
494
|
+
bounds: {
|
|
495
|
+
x: Math.round(attributes.x),
|
|
496
|
+
y: Math.round(attributes.y),
|
|
497
|
+
width: Math.round(attributes.width),
|
|
498
|
+
height: Math.round(attributes.height)
|
|
499
|
+
},
|
|
500
|
+
centerX: Math.round(attributes.x + attributes.width / 2),
|
|
501
|
+
centerY: Math.round(attributes.y + attributes.height / 2),
|
|
502
|
+
enabled: attributes.enabled,
|
|
503
|
+
visible: attributes.visible,
|
|
504
|
+
// Enhanced properties for intelligent matching
|
|
505
|
+
isClickable: this.isInteractiveElement(elementType, attributes),
|
|
506
|
+
elementType: this.classifyElementType({ type: elementType, text: attributes.label }),
|
|
507
|
+
hasText: !!(attributes.label || attributes.name),
|
|
508
|
+
textLength: (attributes.label || attributes.name || '').length,
|
|
509
|
+
priority: this.calculateElementPriority({
|
|
510
|
+
type: elementType,
|
|
511
|
+
text: attributes.label,
|
|
512
|
+
width: attributes.width,
|
|
513
|
+
height: attributes.height,
|
|
514
|
+
enabled: attributes.enabled
|
|
515
|
+
})
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// Sort by priority (most likely to be interaction targets first)
|
|
523
|
+
elements.sort((a, b) => b.priority - a.priority);
|
|
524
|
+
|
|
525
|
+
this.logger.debug(`โ
Parsed and sorted ${elements.length} iOS elements by priority`);
|
|
526
|
+
return elements;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* ๐ง Parse Element Attributes from XML Tag
|
|
531
|
+
*/
|
|
532
|
+
parseElementAttributes(elementTag) {
|
|
533
|
+
const getAttribute = (name) => {
|
|
534
|
+
const match = elementTag.match(new RegExp(`${name}="([^"]*)"`, 'i'));
|
|
535
|
+
return match ? match[1] : null;
|
|
536
|
+
};
|
|
537
|
+
|
|
538
|
+
return {
|
|
539
|
+
x: parseFloat(getAttribute('x')) || 0,
|
|
540
|
+
y: parseFloat(getAttribute('y')) || 0,
|
|
541
|
+
width: parseFloat(getAttribute('width')) || 0,
|
|
542
|
+
height: parseFloat(getAttribute('height')) || 0,
|
|
543
|
+
label: getAttribute('label'),
|
|
544
|
+
name: getAttribute('name'),
|
|
545
|
+
enabled: getAttribute('enabled') !== 'false',
|
|
546
|
+
visible: getAttribute('visible') !== 'false',
|
|
547
|
+
accessible: getAttribute('accessible') === 'true'
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* ๐ฏ Check if Element is Interactive
|
|
553
|
+
*/
|
|
554
|
+
isInteractiveElement(elementType, attributes) {
|
|
555
|
+
const interactiveTypes = [
|
|
556
|
+
'Button', 'TextField', 'SecureTextField', 'Switch', 'Link',
|
|
557
|
+
'SearchField', 'Slider', 'Stepper', 'DatePicker', 'PickerWheel',
|
|
558
|
+
'SegmentedControl', 'Cell'
|
|
559
|
+
];
|
|
560
|
+
|
|
561
|
+
const typeMatch = interactiveTypes.some(type => elementType.includes(type));
|
|
562
|
+
const hasAction = attributes.label && (
|
|
563
|
+
attributes.label.toLowerCase().includes('button') ||
|
|
564
|
+
attributes.label.toLowerCase().includes('tap') ||
|
|
565
|
+
attributes.label.toLowerCase().includes('click')
|
|
566
|
+
);
|
|
567
|
+
|
|
568
|
+
return typeMatch || hasAction;
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* ๐ฏ Find Best Matching Element using AI and Geometric Analysis
|
|
573
|
+
*/
|
|
574
|
+
async findBestMatchingElement(cachedLocators, x, y) {
|
|
575
|
+
const candidates = [];
|
|
576
|
+
|
|
577
|
+
// PERFORMANCE: Limit search to top priority elements first for speed
|
|
578
|
+
const maxCandidates = 20; // OPTIMIZED: Process only top 20 elements for speed
|
|
579
|
+
let candidatesFound = 0;
|
|
580
|
+
|
|
581
|
+
// Step 1: Find elements near the click coordinates - OPTIMIZED
|
|
582
|
+
for (const locator of cachedLocators) {
|
|
583
|
+
if (!locator.bounds || !locator.centerX || !locator.centerY) continue;
|
|
584
|
+
|
|
585
|
+
// PERFORMANCE: Break early if we have enough good candidates
|
|
586
|
+
if (candidatesFound >= maxCandidates) break;
|
|
587
|
+
|
|
588
|
+
// Calculate distance from click point
|
|
589
|
+
const distance = Math.sqrt(
|
|
590
|
+
Math.pow(x - locator.centerX, 2) +
|
|
591
|
+
Math.pow(y - locator.centerY, 2)
|
|
592
|
+
);
|
|
593
|
+
|
|
594
|
+
// Check if click is within element bounds (with generous tolerance)
|
|
595
|
+
const tolerance = 25; // Increased tolerance for mobile UI
|
|
596
|
+
const isWithinBounds = (
|
|
597
|
+
x >= locator.bounds.x - tolerance &&
|
|
598
|
+
x <= locator.bounds.x + locator.bounds.width + tolerance &&
|
|
599
|
+
y >= locator.bounds.y - tolerance &&
|
|
600
|
+
y <= locator.bounds.y + locator.bounds.height + tolerance
|
|
601
|
+
);
|
|
602
|
+
|
|
603
|
+
if (isWithinBounds || distance < 80) { // Within element or nearby (increased from 50)
|
|
604
|
+
candidates.push({
|
|
605
|
+
locator: locator,
|
|
606
|
+
distance: distance,
|
|
607
|
+
isWithinBounds: isWithinBounds,
|
|
608
|
+
geometricScore: this.calculateGeometricScore(locator, x, y),
|
|
609
|
+
elementScore: this.calculateElementScore(locator)
|
|
610
|
+
});
|
|
611
|
+
candidatesFound++;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
if (candidates.length === 0) {
|
|
616
|
+
this.logger.warn(`No candidate elements found near (${x}, ${y})`);
|
|
617
|
+
return null;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
// Step 2: Score candidates using AI or rules
|
|
621
|
+
let scoredCandidates;
|
|
622
|
+
|
|
623
|
+
if (this.aiModel && !this.useRuleBasedMatching) {
|
|
624
|
+
scoredCandidates = await this.scoreWithAI(candidates, x, y);
|
|
625
|
+
} else {
|
|
626
|
+
scoredCandidates = this.scoreWithRules(candidates, x, y);
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
// Step 3: Select best candidate
|
|
630
|
+
scoredCandidates.sort((a, b) => b.totalScore - a.totalScore);
|
|
631
|
+
const bestCandidate = scoredCandidates[0];
|
|
632
|
+
|
|
633
|
+
this.logger.info(`๐ฏ Best candidate: ${bestCandidate.locator.elementType} with score ${bestCandidate.totalScore.toFixed(3)}`);
|
|
634
|
+
|
|
635
|
+
return {
|
|
636
|
+
locator: this.buildOptimalLocator(bestCandidate.locator),
|
|
637
|
+
confidence: bestCandidate.totalScore,
|
|
638
|
+
element: bestCandidate.locator,
|
|
639
|
+
allCandidates: scoredCandidates.slice(0, 3) // Top 3 for debugging
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
/**
|
|
644
|
+
* ๐ค Score Candidates using AI Model
|
|
645
|
+
*/
|
|
646
|
+
async scoreWithAI(candidates, x, y) {
|
|
647
|
+
// This would use the TensorFlow model to score candidates
|
|
648
|
+
// For now, we'll fall back to rule-based scoring
|
|
649
|
+
return this.scoreWithRules(candidates, x, y);
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
/**
|
|
653
|
+
* ๐ Score Candidates using Geometric and Heuristic Rules
|
|
654
|
+
*/
|
|
655
|
+
scoreWithRules(candidates, x, y) {
|
|
656
|
+
return candidates.map(candidate => {
|
|
657
|
+
const { locator, distance, isWithinBounds, geometricScore, elementScore } = candidate;
|
|
658
|
+
|
|
659
|
+
// Distance score (closer is better, more generous)
|
|
660
|
+
const maxDistance = 150; // Increased from 100
|
|
661
|
+
const distanceScore = Math.max(0.1, 1 - (distance / maxDistance)); // Min score of 0.1
|
|
662
|
+
|
|
663
|
+
// Bounds score (inside is much better, but outside isn't terrible)
|
|
664
|
+
const boundsScore = isWithinBounds ? 1.0 : 0.5; // Improved from 0.3
|
|
665
|
+
|
|
666
|
+
// Clickability score
|
|
667
|
+
const clickabilityScore = locator.isClickable ? 1.0 : 0.5;
|
|
668
|
+
|
|
669
|
+
// Size appropriateness (not too small, not too large)
|
|
670
|
+
const idealArea = 1000; // ~30x30 px
|
|
671
|
+
const sizeScore = locator.area > 0 ?
|
|
672
|
+
Math.exp(-Math.abs(Math.log(locator.area / idealArea))) : 0.5;
|
|
673
|
+
|
|
674
|
+
// Element type preference (iOS-specific types, improved for links)
|
|
675
|
+
const elementTypeLower = (locator.elementType || '').toLowerCase();
|
|
676
|
+
let typeScore = 0.6; // Default score
|
|
677
|
+
|
|
678
|
+
// HIGH PRIORITY: Link elements (especially for URL text)
|
|
679
|
+
if (elementTypeLower.includes('link')) typeScore = 1.0;
|
|
680
|
+
else if (elementTypeLower.includes('button') || elementTypeLower.includes('cell')) typeScore = 0.95;
|
|
681
|
+
else if (elementTypeLower.includes('switch') || elementTypeLower.includes('slider')) typeScore = 0.9;
|
|
682
|
+
else if (elementTypeLower.includes('textfield') || elementTypeLower.includes('input')) typeScore = 0.85;
|
|
683
|
+
else if (elementTypeLower.includes('image')) typeScore = 0.8;
|
|
684
|
+
else if (elementTypeLower.includes('text') || elementTypeLower.includes('label')) {
|
|
685
|
+
// Give higher score to text that looks like links
|
|
686
|
+
const textContent = (locator.text || '').toLowerCase();
|
|
687
|
+
if (textContent.includes('learn more') || textContent.includes('coverage') ||
|
|
688
|
+
textContent.includes('...') || textContent.includes('http')) {
|
|
689
|
+
typeScore = 0.9; // Boost for link-like text
|
|
690
|
+
} else {
|
|
691
|
+
typeScore = 0.75;
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
else if (locator.isClickable) typeScore = 0.8; // Any clickable element
|
|
695
|
+
|
|
696
|
+
// Combine all scores with improved weights (favor proximity and bounds)
|
|
697
|
+
const totalScore = (
|
|
698
|
+
distanceScore * 0.35 + // Increased: proximity is very important
|
|
699
|
+
boundsScore * 0.35 + // Increased: being within bounds is crucial
|
|
700
|
+
clickabilityScore * 0.15 + // Reduced: many elements are clickable
|
|
701
|
+
typeScore * 0.10 + // Added: element type matters
|
|
702
|
+
geometricScore * 0.03 + // Reduced: less important than proximity
|
|
703
|
+
elementScore * 0.02 // Reduced: less important than proximity
|
|
704
|
+
);
|
|
705
|
+
|
|
706
|
+
return {
|
|
707
|
+
...candidate,
|
|
708
|
+
scores: {
|
|
709
|
+
distance: distanceScore,
|
|
710
|
+
bounds: boundsScore,
|
|
711
|
+
clickability: clickabilityScore,
|
|
712
|
+
geometric: geometricScore,
|
|
713
|
+
element: elementScore,
|
|
714
|
+
size: sizeScore,
|
|
715
|
+
type: typeScore
|
|
716
|
+
},
|
|
717
|
+
totalScore: Math.min(1.0, totalScore) // Cap at 1.0
|
|
718
|
+
};
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/**
|
|
723
|
+
* ๐ Calculate Geometric Score
|
|
724
|
+
*/
|
|
725
|
+
calculateGeometricScore(locator, x, y) {
|
|
726
|
+
if (!locator.bounds) return 0;
|
|
727
|
+
|
|
728
|
+
// Prefer elements that are centered around the click point
|
|
729
|
+
const centerDistanceScore = 1 / (1 + Math.abs(locator.centerX - x) + Math.abs(locator.centerY - y));
|
|
730
|
+
|
|
731
|
+
// Prefer reasonable aspect ratios
|
|
732
|
+
const aspectRatio = locator.bounds.width / locator.bounds.height;
|
|
733
|
+
const aspectScore = Math.exp(-Math.abs(Math.log(aspectRatio / 1.5))); // Prefer ~1.5:1 ratio
|
|
734
|
+
|
|
735
|
+
return (centerDistanceScore + aspectScore) / 2;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/**
|
|
739
|
+
* ๐ท๏ธ Calculate Element Score based on properties (Enhanced for App Icons)
|
|
740
|
+
*/
|
|
741
|
+
calculateElementScore(locator) {
|
|
742
|
+
let score = 0.5; // Base score
|
|
743
|
+
const className = (locator.className || locator.type || '').toLowerCase();
|
|
744
|
+
|
|
745
|
+
// ๐ฏ HIGH PRIORITY: App icons with text labels
|
|
746
|
+
if ((className.includes('cell') || className.includes('icon') || className.includes('app')) &&
|
|
747
|
+
locator.text && locator.text.trim().length > 0) {
|
|
748
|
+
score += 0.4; // Very high boost for app icons with names
|
|
749
|
+
this.logger.debug(`๐ฑ App icon detected: "${locator.text}" - boosted score`);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
// Has meaningful text
|
|
753
|
+
if (locator.hasText && locator.textLength > 2) score += 0.2;
|
|
754
|
+
|
|
755
|
+
// Has accessibility info
|
|
756
|
+
if (locator.accessibilityId || locator.accessibilityLabel) score += 0.15;
|
|
757
|
+
|
|
758
|
+
// Has resource ID (Android) or name (iOS)
|
|
759
|
+
if (locator.resourceId || locator.name) score += 0.15;
|
|
760
|
+
|
|
761
|
+
// Clickable elements get priority
|
|
762
|
+
if (this.isElementClickable(locator)) score += 0.1;
|
|
763
|
+
|
|
764
|
+
return Math.min(1.0, score);
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* ๐ท๏ธ Classify Element Type
|
|
769
|
+
*/
|
|
770
|
+
classifyElementType(locator) {
|
|
771
|
+
const text = (locator.text || '').toLowerCase();
|
|
772
|
+
const className = (locator.className || locator.type || '').toLowerCase();
|
|
773
|
+
const resourceId = (locator.resourceId || '').toLowerCase();
|
|
774
|
+
|
|
775
|
+
// Link detection (NEW - high priority for URL links)
|
|
776
|
+
if (className.includes('link') ||
|
|
777
|
+
className.includes('xcuielementtypelink') ||
|
|
778
|
+
text.includes('http') ||
|
|
779
|
+
text.includes('www.') ||
|
|
780
|
+
text.includes('learn more') ||
|
|
781
|
+
text.includes('coverage') ||
|
|
782
|
+
text.includes('more info') ||
|
|
783
|
+
(text.includes('...') && text.length > 10)) { // Truncated link text
|
|
784
|
+
return 'link';
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
// Button detection (enhanced for back buttons, navigation, and app icons)
|
|
788
|
+
if (className.includes('button') ||
|
|
789
|
+
resourceId.includes('btn') ||
|
|
790
|
+
// Back button specific patterns
|
|
791
|
+
className.includes('back') ||
|
|
792
|
+
className.includes('navigation') ||
|
|
793
|
+
locator.accessibilityLabel?.toLowerCase().includes('back') ||
|
|
794
|
+
locator.name?.toLowerCase().includes('back') ||
|
|
795
|
+
// iOS specific button types
|
|
796
|
+
className.includes('xcuielementtypebutton') ||
|
|
797
|
+
// App icon patterns
|
|
798
|
+
className.includes('icon') ||
|
|
799
|
+
className.includes('app') ||
|
|
800
|
+
(className.includes('cell') && locator.text) || // App cells with text
|
|
801
|
+
// Generic button text patterns
|
|
802
|
+
text.match(/^(ok|cancel|submit|save|send|login|sign|tap|click|back|close|done)$/i)) {
|
|
803
|
+
return 'button';
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
// Input field detection
|
|
807
|
+
if (className.includes('edit') ||
|
|
808
|
+
className.includes('input') ||
|
|
809
|
+
className.includes('field') ||
|
|
810
|
+
resourceId.includes('input') ||
|
|
811
|
+
resourceId.includes('edit')) {
|
|
812
|
+
return 'input';
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
// App icon detection
|
|
816
|
+
if (className.includes('icon') ||
|
|
817
|
+
className.includes('app') ||
|
|
818
|
+
(className.includes('cell') && locator.text) || // App grid cells
|
|
819
|
+
(className.includes('collection') && locator.text)) { // Collection view cells
|
|
820
|
+
return 'app_icon';
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
// Image detection
|
|
824
|
+
if (className.includes('image') || className.includes('img')) {
|
|
825
|
+
return 'image';
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
// Text detection
|
|
829
|
+
if (className.includes('text') || locator.hasText) {
|
|
830
|
+
return 'text';
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
return 'unknown';
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
/**
|
|
837
|
+
* โ
Check if Element is Clickable (Enhanced for Navigation Elements)
|
|
838
|
+
*/
|
|
839
|
+
isElementClickable(locator) {
|
|
840
|
+
const className = (locator.className || locator.type || '').toLowerCase();
|
|
841
|
+
const text = (locator.text || '').toLowerCase();
|
|
842
|
+
const accessibilityLabel = (locator.accessibilityLabel || locator.label || '').toLowerCase();
|
|
843
|
+
const name = (locator.name || '').toLowerCase();
|
|
844
|
+
|
|
845
|
+
// Explicitly clickable elements
|
|
846
|
+
if (className.includes('button') ||
|
|
847
|
+
className.includes('clickable') ||
|
|
848
|
+
locator.clickable === true) {
|
|
849
|
+
return true;
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
// Back buttons and navigation elements (often without text)
|
|
853
|
+
if (className.includes('back') ||
|
|
854
|
+
className.includes('navigation') ||
|
|
855
|
+
className.includes('xcuielementtypebutton') ||
|
|
856
|
+
accessibilityLabel.includes('back') ||
|
|
857
|
+
name.includes('back') ||
|
|
858
|
+
accessibilityLabel.includes('close') ||
|
|
859
|
+
accessibilityLabel.includes('done')) {
|
|
860
|
+
return true;
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// App icons and collection view cells (always clickable)
|
|
864
|
+
if (className.includes('icon') ||
|
|
865
|
+
className.includes('app') ||
|
|
866
|
+
className.includes('cell') || // Collection/table view cells
|
|
867
|
+
className.includes('collection') ||
|
|
868
|
+
(className.includes('xcuielementtypecell') && locator.text)) { // iOS cells with app names
|
|
869
|
+
return true;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
// Interactive elements
|
|
873
|
+
if (className.includes('edit') ||
|
|
874
|
+
className.includes('input') ||
|
|
875
|
+
className.includes('switch') ||
|
|
876
|
+
className.includes('checkbox')) {
|
|
877
|
+
return true;
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
// Has click handlers (iOS) or meaningful accessibility info
|
|
881
|
+
if (locator.enabled !== false && (locator.text || locator.accessibilityLabel || locator.name)) {
|
|
882
|
+
return true;
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
return false;
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
/**
|
|
889
|
+
* โญ Calculate Element Priority for Caching Order
|
|
890
|
+
*/
|
|
891
|
+
calculateElementPriority(locator) {
|
|
892
|
+
let priority = 0;
|
|
893
|
+
|
|
894
|
+
// Clickable elements get higher priority
|
|
895
|
+
if (this.isElementClickable(locator)) priority += 10;
|
|
896
|
+
|
|
897
|
+
// Elements with text get priority
|
|
898
|
+
if (locator.hasText) priority += 5;
|
|
899
|
+
|
|
900
|
+
// Elements with IDs get priority
|
|
901
|
+
if (locator.resourceId || locator.accessibilityId) priority += 3;
|
|
902
|
+
|
|
903
|
+
// Reasonable size elements
|
|
904
|
+
if (locator.area > 100 && locator.area < 10000) priority += 2;
|
|
905
|
+
|
|
906
|
+
return priority;
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
/**
|
|
910
|
+
* ๐ฏ Build Optimal Locator Strategy
|
|
911
|
+
*/
|
|
912
|
+
buildOptimalLocator(element) {
|
|
913
|
+
const locators = [];
|
|
914
|
+
|
|
915
|
+
// Priority 1: Accessibility ID (most reliable)
|
|
916
|
+
if (element.accessibilityId) {
|
|
917
|
+
locators.push({
|
|
918
|
+
strategy: 'accessibility_id',
|
|
919
|
+
selector: element.accessibilityId,
|
|
920
|
+
priority: 1
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
// Priority 2: Resource ID (Android)
|
|
925
|
+
if (element.resourceId) {
|
|
926
|
+
locators.push({
|
|
927
|
+
strategy: 'id',
|
|
928
|
+
selector: element.resourceId,
|
|
929
|
+
priority: 1
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
// Priority 3: XPath (if available and short)
|
|
934
|
+
if (element.xpath && element.xpath.length < 200) {
|
|
935
|
+
locators.push({
|
|
936
|
+
strategy: 'xpath',
|
|
937
|
+
selector: element.xpath,
|
|
938
|
+
priority: 2
|
|
939
|
+
});
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
// Priority 4: Text-based (if unique)
|
|
943
|
+
if (element.text && element.text.length > 2) {
|
|
944
|
+
const decodedText = this.decodeHtmlEntities(element.text);
|
|
945
|
+
locators.push({
|
|
946
|
+
strategy: 'text',
|
|
947
|
+
selector: decodedText,
|
|
948
|
+
priority: 3
|
|
949
|
+
});
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
// Priority 5: Class name + index
|
|
953
|
+
if (element.className) {
|
|
954
|
+
locators.push({
|
|
955
|
+
strategy: 'class_name',
|
|
956
|
+
selector: element.className,
|
|
957
|
+
index: element.index || 0,
|
|
958
|
+
priority: 4
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
// Sort by priority and return the best strategy
|
|
963
|
+
locators.sort((a, b) => a.priority - b.priority);
|
|
964
|
+
|
|
965
|
+
return locators.length > 0 ? locators[0] : null;
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
/**
|
|
969
|
+
* ๐ฏ MAIN EXECUTION METHOD: Apply Intelligent Locator on Follower
|
|
970
|
+
*/
|
|
971
|
+
async executeIntelligentAction(followerDevice, intelligentLocator, action = 'tap') {
|
|
972
|
+
this.logger.info(`๐ฏ Executing ${action} on ${followerDevice.name} using intelligent locator`);
|
|
973
|
+
|
|
974
|
+
// ENHANCED: Special handling for URL links that often fail with locators
|
|
975
|
+
const isUrlLink = intelligentLocator.locator &&
|
|
976
|
+
intelligentLocator.locator.strategy === 'text' &&
|
|
977
|
+
intelligentLocator.locator.selector &&
|
|
978
|
+
(intelligentLocator.locator.selector.toLowerCase().includes('learn more') ||
|
|
979
|
+
intelligentLocator.locator.selector.toLowerCase().includes('coverage') ||
|
|
980
|
+
intelligentLocator.locator.selector.toLowerCase().includes('http') ||
|
|
981
|
+
intelligentLocator.locator.selector.includes('...'));
|
|
982
|
+
|
|
983
|
+
if (isUrlLink) {
|
|
984
|
+
this.logger.warn(`๐ URL link detected: "${intelligentLocator.locator.selector}"`);
|
|
985
|
+
this.logger.warn('๐ก URL links often fail with locators - using coordinate fallback for reliability');
|
|
986
|
+
|
|
987
|
+
// For URL links, skip locator attempts and go straight to coordinates
|
|
988
|
+
if (intelligentLocator.fallbackCoords || intelligentLocator.method === 'coordinates') {
|
|
989
|
+
const coords = intelligentLocator.fallbackCoords || intelligentLocator;
|
|
990
|
+
const success = await this.executeCoordinateAction(followerDevice, coords, action);
|
|
991
|
+
|
|
992
|
+
return {
|
|
993
|
+
success: success,
|
|
994
|
+
method: 'coordinates',
|
|
995
|
+
reason: 'URL link - forced coordinate fallback for reliability',
|
|
996
|
+
coordinates: coords
|
|
997
|
+
};
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
try {
|
|
1002
|
+
if (intelligentLocator.method === 'locator' && intelligentLocator.locator && !isUrlLink) {
|
|
1003
|
+
// Try locator-based execution first (except for URL links)
|
|
1004
|
+
const success = await this.executeLocatorAction(followerDevice, intelligentLocator.locator, action);
|
|
1005
|
+
|
|
1006
|
+
if (success) {
|
|
1007
|
+
this.logger.info(`โ
Locator-based ${action} succeeded on ${followerDevice.name}`);
|
|
1008
|
+
return { success: true, method: 'locator' };
|
|
1009
|
+
} else {
|
|
1010
|
+
this.logger.warn(`โ ๏ธ Locator failed, falling back to coordinates on ${followerDevice.name}`);
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
// Fallback to coordinate-based execution
|
|
1015
|
+
if (intelligentLocator.fallbackCoords || intelligentLocator.method === 'coordinates') {
|
|
1016
|
+
const coords = intelligentLocator.fallbackCoords || intelligentLocator;
|
|
1017
|
+
const success = await this.executeCoordinateAction(followerDevice, coords, action);
|
|
1018
|
+
|
|
1019
|
+
return {
|
|
1020
|
+
success: success,
|
|
1021
|
+
method: 'coordinates',
|
|
1022
|
+
coordinates: coords
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
throw new Error('No valid execution method available');
|
|
1027
|
+
|
|
1028
|
+
} catch (error) {
|
|
1029
|
+
this.logger.error(`โ Intelligent action failed: ${error.message}`);
|
|
1030
|
+
return { success: false, error: error.message };
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
/**
|
|
1035
|
+
* ๐ฏ Execute Action using Locator
|
|
1036
|
+
*/
|
|
1037
|
+
async executeLocatorAction(device, locator, action) {
|
|
1038
|
+
const port = device.port || 8100;
|
|
1039
|
+
const platform = device.platform || 'ios';
|
|
1040
|
+
|
|
1041
|
+
try {
|
|
1042
|
+
if (platform.toLowerCase() === 'ios') {
|
|
1043
|
+
return await this.executeIOSLocatorAction(device, locator, action, port);
|
|
1044
|
+
} else {
|
|
1045
|
+
return await this.executeAndroidLocatorAction(device, locator, action, port);
|
|
1046
|
+
}
|
|
1047
|
+
} catch (error) {
|
|
1048
|
+
this.logger.error(`Locator action failed: ${error.message}`);
|
|
1049
|
+
return false;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
/**
|
|
1054
|
+
* ๐ Get Fresh Session ID with Universal Session Manager (UPDATED)
|
|
1055
|
+
* Uses the universal session manager for consistent session handling
|
|
1056
|
+
*/
|
|
1057
|
+
async getFreshSessionId(deviceName, validateWithSourceEndpoint = false) {
|
|
1058
|
+
try {
|
|
1059
|
+
this.logger.debug(`[${deviceName}] Getting session via Universal Session Manager...`);
|
|
1060
|
+
|
|
1061
|
+
// REVOLUTIONARY: Use Universal Session Manager
|
|
1062
|
+
const sessionId = await this.universalSessionManager.getSessionWithRetry(deviceName);
|
|
1063
|
+
|
|
1064
|
+
if (sessionId) {
|
|
1065
|
+
this.logger.debug(`โ
Session retrieved via Universal Manager: ${sessionId.substring(0, 8)}... for ${deviceName}`);
|
|
1066
|
+
return sessionId;
|
|
1067
|
+
}
|
|
1068
|
+
|
|
1069
|
+
throw new Error(`No session available via Universal Session Manager`);
|
|
1070
|
+
|
|
1071
|
+
} catch (error) {
|
|
1072
|
+
this.logger.error(`โ Universal session lookup failed for ${deviceName}: ${error.message}`);
|
|
1073
|
+
throw error;
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
/**
|
|
1078
|
+
* ๐ฉบ Check if device disconnect should trigger immediate cleanup (DISABLED)
|
|
1079
|
+
*/
|
|
1080
|
+
async checkDeviceDisconnection(deviceName, error) {
|
|
1081
|
+
// DISABLED: Was too aggressive and cleaning up healthy devices
|
|
1082
|
+
// TODO: Fix health check logic before re-enabling
|
|
1083
|
+
/*
|
|
1084
|
+
const disconnectionKeywords = [
|
|
1085
|
+
'ECONNREFUSED', 'ENOTFOUND', 'ETIMEDOUT',
|
|
1086
|
+
'Connection refused', 'connect ECONNREFUSED',
|
|
1087
|
+
'Session does not exist', 'invalid session id'
|
|
1088
|
+
];
|
|
1089
|
+
|
|
1090
|
+
const isDisconnectionError = disconnectionKeywords.some(keyword =>
|
|
1091
|
+
error.message?.includes(keyword)
|
|
1092
|
+
);
|
|
1093
|
+
|
|
1094
|
+
if (isDisconnectionError && global.performDeviceHealthCheck) {
|
|
1095
|
+
this.logger.warn(`๐ฉบ Device ${deviceName} shows disconnection signs, triggering immediate health check...`);
|
|
1096
|
+
|
|
1097
|
+
setTimeout(async () => {
|
|
1098
|
+
try {
|
|
1099
|
+
await global.performDeviceHealthCheck();
|
|
1100
|
+
} catch (healthError) {
|
|
1101
|
+
this.logger.error(`Health check failed: ${healthError.message}`);
|
|
1102
|
+
}
|
|
1103
|
+
}, 1000);
|
|
1104
|
+
}
|
|
1105
|
+
*/
|
|
1106
|
+
|
|
1107
|
+
// Just log the error for now without triggering cleanup
|
|
1108
|
+
this.logger.debug(`Device ${deviceName} session error (auto-cleanup disabled): ${error.message}`);
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
/**
|
|
1112
|
+
* ๐ฑ Execute iOS Locator Action (Updated with Fresh Session Management)
|
|
1113
|
+
*/
|
|
1114
|
+
async executeIOSLocatorAction(device, locator, action, port) {
|
|
1115
|
+
// REVOLUTIONARY: Use universal session manager for consistent sessions
|
|
1116
|
+
let sessionId;
|
|
1117
|
+
try {
|
|
1118
|
+
sessionId = await this.universalSessionManager.getSessionWithRetry(device.name, port);
|
|
1119
|
+
this.logger.debug(`โ
Using universal session: ${sessionId.substring(0, 8)}... for ${device.name}`);
|
|
1120
|
+
} catch (sessionError) {
|
|
1121
|
+
this.logger.warn(`โ ๏ธ Universal session failed: ${sessionError.message}`);
|
|
1122
|
+
throw new Error('No valid iOS session available');
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
const baseUrl = `http://localhost:${port}`;
|
|
1126
|
+
|
|
1127
|
+
try {
|
|
1128
|
+
let elementId = null;
|
|
1129
|
+
|
|
1130
|
+
// Find element using the best locator strategy
|
|
1131
|
+
switch (locator.strategy) {
|
|
1132
|
+
case 'accessibility_id':
|
|
1133
|
+
const accessResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1134
|
+
using: 'accessibility id',
|
|
1135
|
+
value: locator.selector
|
|
1136
|
+
});
|
|
1137
|
+
elementId = accessResponse.data.value.ELEMENT;
|
|
1138
|
+
break;
|
|
1139
|
+
|
|
1140
|
+
case 'xpath':
|
|
1141
|
+
const xpathResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1142
|
+
using: 'xpath',
|
|
1143
|
+
value: locator.selector
|
|
1144
|
+
});
|
|
1145
|
+
elementId = xpathResponse.data.value.ELEMENT;
|
|
1146
|
+
break;
|
|
1147
|
+
|
|
1148
|
+
case 'text':
|
|
1149
|
+
// Try multiple iOS text matching strategies
|
|
1150
|
+
const decodedText = this.decodeHtmlEntities(locator.selector);
|
|
1151
|
+
try {
|
|
1152
|
+
// Strategy 1: Accessibility label (most common for iOS buttons)
|
|
1153
|
+
const labelResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1154
|
+
using: 'accessibility id',
|
|
1155
|
+
value: decodedText
|
|
1156
|
+
});
|
|
1157
|
+
elementId = labelResponse.data.value.ELEMENT;
|
|
1158
|
+
} catch {
|
|
1159
|
+
try {
|
|
1160
|
+
// Strategy 2: XPath with text content for iOS (improved for links)
|
|
1161
|
+
const xpathResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1162
|
+
using: 'xpath',
|
|
1163
|
+
value: `//*[@name="${decodedText}" or @label="${decodedText}" or @value="${decodedText}" or contains(@name, "${decodedText}")]`
|
|
1164
|
+
});
|
|
1165
|
+
elementId = xpathResponse.data.value.ELEMENT;
|
|
1166
|
+
} catch {
|
|
1167
|
+
try {
|
|
1168
|
+
// Strategy 3: Predicate string (iOS-specific, improved)
|
|
1169
|
+
const predicateResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1170
|
+
using: '-ios predicate string',
|
|
1171
|
+
value: `name CONTAINS "${decodedText}" OR label CONTAINS "${decodedText}" OR value CONTAINS "${decodedText}"`
|
|
1172
|
+
});
|
|
1173
|
+
elementId = predicateResponse.data.value.ELEMENT;
|
|
1174
|
+
} catch {
|
|
1175
|
+
try {
|
|
1176
|
+
// Strategy 4: Class chain for buttons (original)
|
|
1177
|
+
const classChainResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1178
|
+
using: '-ios class chain',
|
|
1179
|
+
value: `**/XCUIElementTypeButton[\`name == "${decodedText}" OR label == "${decodedText}"\`]`
|
|
1180
|
+
});
|
|
1181
|
+
elementId = classChainResponse.data.value.ELEMENT;
|
|
1182
|
+
} catch {
|
|
1183
|
+
// Strategy 5: Class chain for links and text elements (NEW - specifically for URLs)
|
|
1184
|
+
const linkChainResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1185
|
+
using: '-ios class chain',
|
|
1186
|
+
value: `**/XCUIElementTypeLink[\`name CONTAINS "${decodedText}" OR label CONTAINS "${decodedText}"\`]`
|
|
1187
|
+
});
|
|
1188
|
+
if (linkChainResponse.data?.value?.ELEMENT) {
|
|
1189
|
+
elementId = linkChainResponse.data.value.ELEMENT;
|
|
1190
|
+
} else {
|
|
1191
|
+
// Strategy 6: Generic text elements (for text that might be clickable)
|
|
1192
|
+
const textChainResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1193
|
+
using: '-ios class chain',
|
|
1194
|
+
value: `**/XCUIElementTypeStaticText[\`name CONTAINS "${decodedText}" OR label CONTAINS "${decodedText}"\`]`
|
|
1195
|
+
});
|
|
1196
|
+
elementId = textChainResponse.data.value.ELEMENT;
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
break;
|
|
1203
|
+
|
|
1204
|
+
case 'class_name':
|
|
1205
|
+
const classResponse = await axios.post(`${baseUrl}/session/${sessionId}/elements`, {
|
|
1206
|
+
using: 'class name',
|
|
1207
|
+
value: locator.selector
|
|
1208
|
+
});
|
|
1209
|
+
const elements = classResponse.data.value;
|
|
1210
|
+
if (elements && elements.length > locator.index) {
|
|
1211
|
+
elementId = elements[locator.index].ELEMENT;
|
|
1212
|
+
}
|
|
1213
|
+
break;
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
if (!elementId) {
|
|
1217
|
+
throw new Error('Element not found with locator strategy');
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
// Execute action on found element
|
|
1221
|
+
switch (action) {
|
|
1222
|
+
case 'tap':
|
|
1223
|
+
case 'click':
|
|
1224
|
+
await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/click`);
|
|
1225
|
+
break;
|
|
1226
|
+
case 'sendKeys':
|
|
1227
|
+
await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/value`, {
|
|
1228
|
+
value: [locator.text || '']
|
|
1229
|
+
});
|
|
1230
|
+
break;
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
// ๐ฏ Invalidate cache after successful iOS locator action (screen likely changed)
|
|
1234
|
+
this.invalidateCache(device.name);
|
|
1235
|
+
|
|
1236
|
+
return true;
|
|
1237
|
+
|
|
1238
|
+
} catch (error) {
|
|
1239
|
+
this.logger.warn(`iOS locator action failed: ${error.message}`);
|
|
1240
|
+
return false;
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
/**
|
|
1245
|
+
* ๐ค Execute Android Locator Action (Updated with Fresh Session Management)
|
|
1246
|
+
*/
|
|
1247
|
+
async executeAndroidLocatorAction(device, locator, action, port) {
|
|
1248
|
+
// Get fresh session ID
|
|
1249
|
+
const sessionId = await this.getFreshSessionId(device.name);
|
|
1250
|
+
if (!sessionId) {
|
|
1251
|
+
throw new Error('No valid Android session available');
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
const baseUrl = `http://localhost:${port}/wd/hub`;
|
|
1255
|
+
|
|
1256
|
+
try {
|
|
1257
|
+
let elementId = null;
|
|
1258
|
+
|
|
1259
|
+
// Find element using the best locator strategy
|
|
1260
|
+
switch (locator.strategy) {
|
|
1261
|
+
case 'id':
|
|
1262
|
+
const idResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1263
|
+
using: 'id',
|
|
1264
|
+
value: locator.selector
|
|
1265
|
+
});
|
|
1266
|
+
elementId = idResponse.data.value.ELEMENT;
|
|
1267
|
+
break;
|
|
1268
|
+
|
|
1269
|
+
case 'accessibility_id':
|
|
1270
|
+
const accessResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1271
|
+
using: 'accessibility id',
|
|
1272
|
+
value: locator.selector
|
|
1273
|
+
});
|
|
1274
|
+
elementId = accessResponse.data.value.ELEMENT;
|
|
1275
|
+
break;
|
|
1276
|
+
|
|
1277
|
+
case 'xpath':
|
|
1278
|
+
const xpathResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1279
|
+
using: 'xpath',
|
|
1280
|
+
value: locator.selector
|
|
1281
|
+
});
|
|
1282
|
+
elementId = xpathResponse.data.value.ELEMENT;
|
|
1283
|
+
break;
|
|
1284
|
+
|
|
1285
|
+
case 'text':
|
|
1286
|
+
const textResponse = await axios.post(`${baseUrl}/session/${sessionId}/element`, {
|
|
1287
|
+
using: 'partial link text',
|
|
1288
|
+
value: locator.selector
|
|
1289
|
+
});
|
|
1290
|
+
elementId = textResponse.data.value.ELEMENT;
|
|
1291
|
+
break;
|
|
1292
|
+
|
|
1293
|
+
case 'class_name':
|
|
1294
|
+
const classResponse = await axios.post(`${baseUrl}/session/${sessionId}/elements`, {
|
|
1295
|
+
using: 'class name',
|
|
1296
|
+
value: locator.selector
|
|
1297
|
+
});
|
|
1298
|
+
const elements = classResponse.data.value;
|
|
1299
|
+
if (elements && elements.length > locator.index) {
|
|
1300
|
+
elementId = elements[locator.index].ELEMENT;
|
|
1301
|
+
}
|
|
1302
|
+
break;
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
if (!elementId) {
|
|
1306
|
+
throw new Error('Element not found with locator strategy');
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
// Execute action on found element
|
|
1310
|
+
switch (action) {
|
|
1311
|
+
case 'tap':
|
|
1312
|
+
case 'click':
|
|
1313
|
+
await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/click`);
|
|
1314
|
+
break;
|
|
1315
|
+
case 'sendKeys':
|
|
1316
|
+
await axios.post(`${baseUrl}/session/${sessionId}/element/${elementId}/value`, {
|
|
1317
|
+
value: [locator.text || '']
|
|
1318
|
+
});
|
|
1319
|
+
break;
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
// ๐ฏ Invalidate cache after successful Android locator action (screen likely changed)
|
|
1323
|
+
this.invalidateCache(device.name);
|
|
1324
|
+
|
|
1325
|
+
return true;
|
|
1326
|
+
|
|
1327
|
+
} catch (error) {
|
|
1328
|
+
this.logger.warn(`Android locator action failed: ${error.message}`);
|
|
1329
|
+
return false;
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
/**
|
|
1334
|
+
* ๐ Execute Coordinate Action (Fallback) - Optimized Performance
|
|
1335
|
+
*/
|
|
1336
|
+
async executeCoordinateAction(device, coordinates, action) {
|
|
1337
|
+
const port = device.port || 8100;
|
|
1338
|
+
const platform = device.platform || 'ios';
|
|
1339
|
+
|
|
1340
|
+
// ๐ฏ FAST COORDINATE PROCESSING: Only scale if obviously wrong
|
|
1341
|
+
let scaledX = coordinates.x;
|
|
1342
|
+
let scaledY = coordinates.y;
|
|
1343
|
+
|
|
1344
|
+
// Quick check: Only scale if coordinates are suspiciously small (< 50 pixels)
|
|
1345
|
+
if (scaledX < 50 || scaledY < 100) {
|
|
1346
|
+
// These are likely incorrectly scaled - apply 3x scaling
|
|
1347
|
+
scaledX = scaledX * 3;
|
|
1348
|
+
scaledY = scaledY * 3;
|
|
1349
|
+
this.logger.info(`๐ง Quick-scaled tiny coordinates (${coordinates.x}, ${coordinates.y}) โ (${scaledX}, ${scaledY})`);
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
// Fast bounds check - only clamp if outside reasonable limits
|
|
1353
|
+
if (scaledX > 500) scaledX = 450;
|
|
1354
|
+
if (scaledY > 1000) scaledY = 900;
|
|
1355
|
+
if (scaledX < 5) scaledX = 10;
|
|
1356
|
+
if (scaledY < 20) scaledY = 30;
|
|
1357
|
+
|
|
1358
|
+
try {
|
|
1359
|
+
if (platform.toLowerCase() === 'ios') {
|
|
1360
|
+
// REVOLUTIONARY: Use universal session manager for coordinate actions
|
|
1361
|
+
let sessionId;
|
|
1362
|
+
try {
|
|
1363
|
+
sessionId = await this.universalSessionManager.getSessionWithRetry(device.name, port);
|
|
1364
|
+
this.logger.debug(`โ
Using universal session for coordinates: ${sessionId.substring(0, 8)}... for ${device.name}`);
|
|
1365
|
+
} catch (sessionError) {
|
|
1366
|
+
this.logger.warn(`โ ๏ธ Universal session failed for coordinates: ${sessionError.message}`);
|
|
1367
|
+
throw new Error('No valid iOS session available for coordinate action');
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
// Use WDA tap endpoint for iOS
|
|
1371
|
+
await axios.post(`http://localhost:${port}/session/${sessionId}/wda/tap`, {
|
|
1372
|
+
x: scaledX,
|
|
1373
|
+
y: scaledY
|
|
1374
|
+
}, { timeout: 5000 });
|
|
1375
|
+
|
|
1376
|
+
} else {
|
|
1377
|
+
// REVOLUTIONARY: Use universal session manager for Android too
|
|
1378
|
+
let sessionId;
|
|
1379
|
+
try {
|
|
1380
|
+
sessionId = await this.universalSessionManager.getSessionWithRetry(device.name, port);
|
|
1381
|
+
} catch (sessionError) {
|
|
1382
|
+
throw new Error('No valid Android session available for coordinate action');
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
await axios.post(`http://localhost:${port}/wd/hub/session/${sessionId}/touch/click`, {
|
|
1386
|
+
element: { x: scaledX, y: scaledY }
|
|
1387
|
+
}, { timeout: 5000 });
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
this.logger.info(`โ
Coordinate action successful on ${device.name} at (${scaledX}, ${scaledY})`);
|
|
1391
|
+
|
|
1392
|
+
// ๐ฏ Invalidate cache after successful coordinate action (screen likely changed)
|
|
1393
|
+
this.invalidateCache(device.name);
|
|
1394
|
+
|
|
1395
|
+
return true;
|
|
1396
|
+
|
|
1397
|
+
} catch (error) {
|
|
1398
|
+
// If 404 error, try session recovery once
|
|
1399
|
+
if (error.response?.status === 404) {
|
|
1400
|
+
this.logger.warn(`โ ๏ธ Session validation failed for ${device.name}: ${error.message}`);
|
|
1401
|
+
|
|
1402
|
+
try {
|
|
1403
|
+
// Try to get a fresh session directly from WDA status
|
|
1404
|
+
const statusRes = await axios.get(`http://localhost:${port}/status`, { timeout: 3000 });
|
|
1405
|
+
const freshSessionId = statusRes.data?.sessionId;
|
|
1406
|
+
|
|
1407
|
+
if (freshSessionId) {
|
|
1408
|
+
this.logger.info(`๐ Found different session after validation failure: ${freshSessionId.substring(0, 8)}...`);
|
|
1409
|
+
|
|
1410
|
+
// Update universal session manager port mapping
|
|
1411
|
+
this.universalSessionManager.setDevicePort(device.name, port);
|
|
1412
|
+
|
|
1413
|
+
// Retry the coordinate action with fresh session
|
|
1414
|
+
if (platform.toLowerCase() === 'ios') {
|
|
1415
|
+
await axios.post(`http://localhost:${port}/session/${freshSessionId}/wda/tap`, {
|
|
1416
|
+
x: coordinates.x,
|
|
1417
|
+
y: coordinates.y
|
|
1418
|
+
}, { timeout: 5000 });
|
|
1419
|
+
} else {
|
|
1420
|
+
await axios.post(`http://localhost:${port}/wd/hub/session/${freshSessionId}/touch/click`, {
|
|
1421
|
+
element: { x: coordinates.x, y: coordinates.y }
|
|
1422
|
+
}, { timeout: 5000 });
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
this.logger.info(`โ
Coordinate action successful on ${device.name} at (${coordinates.x}, ${coordinates.y}) after session recovery`);
|
|
1426
|
+
|
|
1427
|
+
// ๐ฏ Invalidate cache after successful coordinate recovery action (screen likely changed)
|
|
1428
|
+
this.invalidateCache(device.name);
|
|
1429
|
+
|
|
1430
|
+
return true;
|
|
1431
|
+
}
|
|
1432
|
+
} catch (recoveryError) {
|
|
1433
|
+
this.logger.error(`Failed to recover session for ${device.name}: ${recoveryError.message}`);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
this.logger.error(`Coordinate action failed: ${error.message}`);
|
|
1438
|
+
return false;
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
/**
|
|
1443
|
+
* ๐ Get Performance Metrics
|
|
1444
|
+
*/
|
|
1445
|
+
getPerformanceMetrics() {
|
|
1446
|
+
const total = this.performanceMetrics.cacheHits + this.performanceMetrics.cacheMisses;
|
|
1447
|
+
const cacheHitRate = total > 0 ? (this.performanceMetrics.cacheHits / total * 100).toFixed(1) : 0;
|
|
1448
|
+
|
|
1449
|
+
return {
|
|
1450
|
+
...this.performanceMetrics,
|
|
1451
|
+
cacheHitRate: `${cacheHitRate}%`,
|
|
1452
|
+
totalRequests: total,
|
|
1453
|
+
accuracyRate: `${((this.performanceMetrics.accurateMatches / (this.performanceMetrics.accurateMatches + this.performanceMetrics.fallbackToCoordinates)) * 100).toFixed(1)}%`
|
|
1454
|
+
};
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
/**
|
|
1458
|
+
* ๐ Convert elements array to parseable page source format
|
|
1459
|
+
*/
|
|
1460
|
+
convertElementsToPageSource(elements, port, sessionId) {
|
|
1461
|
+
// This is a fallback method to create a basic parseable structure
|
|
1462
|
+
// when proper page source endpoints fail
|
|
1463
|
+
let xmlContent = '<?xml version="1.0" encoding="UTF-8"?>\n<hierarchy>\n';
|
|
1464
|
+
|
|
1465
|
+
elements.forEach((element, index) => {
|
|
1466
|
+
try {
|
|
1467
|
+
// Get element details if possible
|
|
1468
|
+
const elementId = element.ELEMENT;
|
|
1469
|
+
// For now, just create a basic structure - this would need enhancement
|
|
1470
|
+
// to fetch actual element properties via separate API calls
|
|
1471
|
+
xmlContent += ` <XCUIElementTypeButton name="" label="" x="0" y="0" width="100" height="50" />\n`;
|
|
1472
|
+
} catch (error) {
|
|
1473
|
+
// Skip problematic elements
|
|
1474
|
+
}
|
|
1475
|
+
});
|
|
1476
|
+
|
|
1477
|
+
xmlContent += '</hierarchy>';
|
|
1478
|
+
return xmlContent;
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
/**
|
|
1482
|
+
* ๐งน Clear Cache (for testing or troubleshooting)
|
|
1483
|
+
*/
|
|
1484
|
+
clearCache() {
|
|
1485
|
+
this.locatorCache.clear();
|
|
1486
|
+
this.cacheTimestamps.clear();
|
|
1487
|
+
this.logger.info('๐งน Locator cache cleared');
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
/**
|
|
1491
|
+
* ๐ Check if device has cached locators
|
|
1492
|
+
*/
|
|
1493
|
+
hasCachedLocators(deviceName) {
|
|
1494
|
+
return this.locatorCache.has(deviceName) && this.locatorCache.get(deviceName).length > 0;
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
/**
|
|
1498
|
+
* ๐ฏ Invalidate Cache for Specific Device (after screen changes)
|
|
1499
|
+
*/
|
|
1500
|
+
invalidateCache(deviceName) {
|
|
1501
|
+
const hadCache = this.locatorCache.has(deviceName);
|
|
1502
|
+
const cacheSize = hadCache ? this.locatorCache.get(deviceName).length : 0;
|
|
1503
|
+
|
|
1504
|
+
this.locatorCache.delete(deviceName);
|
|
1505
|
+
this.cacheTimestamps.delete(deviceName);
|
|
1506
|
+
|
|
1507
|
+
if (hadCache) {
|
|
1508
|
+
this.logger.info(`๐งน Locator cache invalidated for ${deviceName} (had ${cacheSize} locators)`);
|
|
1509
|
+
} else {
|
|
1510
|
+
this.logger.debug(`๐งน Locator cache invalidated for ${deviceName} (was empty)`);
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
// Also remove from refreshing set to allow immediate refresh
|
|
1514
|
+
this.refreshingDevices.delete(deviceName);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
/**
|
|
1518
|
+
* ๐ Get Cache Age in milliseconds
|
|
1519
|
+
* Returns null if no cache exists, or age in ms since last refresh
|
|
1520
|
+
*/
|
|
1521
|
+
getCacheAge(deviceName) {
|
|
1522
|
+
const timestamp = this.cacheTimestamps.get(deviceName);
|
|
1523
|
+
if (!timestamp) {
|
|
1524
|
+
return null; // No cache exists
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
return Date.now() - timestamp;
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
/**
|
|
1531
|
+
* ๐ฏ Invalidate All Device Caches (after major screen changes)
|
|
1532
|
+
*/
|
|
1533
|
+
invalidateAllCaches() {
|
|
1534
|
+
const deviceCount = this.locatorCache.size;
|
|
1535
|
+
this.locatorCache.clear();
|
|
1536
|
+
this.cacheTimestamps.clear();
|
|
1537
|
+
this.logger.info(`๐งน All locator caches invalidated (${deviceCount} devices)`);
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
module.exports = IntelligentLocatorService;
|