@devassure/tpp-engine 0.0.0-stage → 1.8.0

package/src/web/web-executor-data-fetch.js

@@ -0,0 +1,353 @@
+const { getScreenshotPath } = require("../utils/screenshot-utils");
+const path = require('path');
+const fs = require('fs');
+
+const fetchWebState = async (page) => {
+
+    let filePath = '';
+    try {
+        allElementsData = await page.evaluate(() => {
+
+        const isInteractive = (element) => {
+            const interactiveTags = new Set([
+                'a', 'button', 'input', 'label', 'menu', 'select', 'textarea',
+            ]);
+            const interactiveRoles = new Set([
+                'button', 'checkbox', 'combobox', 'link', 'list', 'listbox', 'listitem', 'menu', 'menuitem', 
+                'option', 'progressbar', 'radio', 'search', 'searchbox','switch', 'tab', 'textbox', 'toolbar',
+            ]);
+        
+            const style = window.getComputedStyle(element);
+            if (style.display === 'none' || style.visibility === 'hidden') return false;
+
+            //TODO check size conditions
+            if (element.offsetWidth === 0 || element.offsetHeight === 0) return false;
+        
+            return element.hasAttribute('tabIndex')
+            || interactiveTags.has(element.tagName.toLowerCase())
+            || interactiveRoles.has(element.getAttribute('role')?.toLowerCase())
+            || element.hasAttribute('onclick')
+            || hasEventListener(element)
+            || hasAttatchedEvents(element);
+        };
+
+        function hasEventListener(element) {
+            // Check if getEventListeners is available (Chrome DevTools only)
+            if (typeof getEventListeners === 'function') {
+                const events = getEventListeners(element);
+                if (Object.keys(events).length > 0) {
+                    return true;
+                }
+            }
+
+            // Check for inline event handler (e.g., onclick)
+            if (element.onclick) {
+                return true;
+            }
+
+            return false;
+        };
+
+        function hasAttatchedEvents(element) {
+
+            function hasEventListeners(element) {
+                const eventTypes = ['click', 'mousedown', 'mouseup', 'touchstart', 'touchend', 'focus', 'blur', 'keydown', 'keyup'];
+                for (const eventType of eventTypes) {
+                    const listeners = element[`on${eventType}`];
+                    if (listeners) {
+                        return true;
+                    }
+                }
+                return false;
+            }
+
+            return element.onclick !== null ||
+            element.getAttribute('onclick') !== null ||
+            element.hasAttribute('ng-click') ||
+            element.hasAttribute('@click') ||
+            element.hasAttribute('v-on:click') || hasEventListeners(element);
+        }
+
+        //TODO! Adding this causes noise as multiple nested elements are included.
+        //Check how to sanitise before adding this condition.
+        function hasCursorPointer(element) {
+            const computedStyle = window.getComputedStyle(element);
+            return computedStyle?.cursor === 'pointer';
+        };
+
+        function isElementVisible(el) {
+            const style = window.getComputedStyle(el);
+            const tagName = el.tagName.toLowerCase();
+            // Check for hidden styles and dimensions
+            return el.checkVisibility() &&
+                   style.display !== 'none' && 
+                   style.visibility !== 'hidden' &&
+                   (tagName === 'select' || tagName === 'input' || parseFloat(style.opacity) > 0.1)
+        }
+        
+        function isElementInViewport(el) {
+            const rect = el.getBoundingClientRect();
+            return (
+                rect.top >= 0 &&
+                rect.left >= 0 &&
+                rect.top <= window.innerHeight &&
+                rect.left <= window.innerWidth
+            );
+        }
+        
+        function isElementNotCovered(el, visibilityThreshold = 0.5, sampleDensity = 4) {
+            const rect = el.getBoundingClientRect();
+            const stepX = Math.max(2, rect.width / sampleDensity);
+            const stepY = Math.max(2, rect.height / sampleDensity);
+            
+            let visiblePoints = 0;
+            let totalPoints = 0;
+        
+            for (let y = rect.top; y <= rect.bottom; y += stepY) {
+                for (let x = rect.left; x <= rect.right; x += stepX) {
+                    const elementsAtPoint = document.elementsFromPoint(x, y);
+                    
+                    if (elementsAtPoint.length === 0) continue;
+        
+                    const topElement = elementsAtPoint[0];
+        
+                    if (topElement !== el && !isInteractive(topElement)) {
+                        return true; // If the top element is non-interactive and not 'el', it's covered
+                    }
+        
+                    // Check if the first interactive element is this element
+                    const firstInteractive = elementsAtPoint.find(isInteractive);
+                    if (firstInteractive === el) {
+                        visiblePoints++;
+                    }
+                    
+                    totalPoints++;
+                }
+            }
+        
+            return totalPoints > 0 && (visiblePoints / totalPoints) >= visibilityThreshold;
+        }
+
+        const getAccuratePosition = (el) => {
+            const rect = el.getBoundingClientRect();
+            let x = Math.round(rect.left + rect.width / 2);
+            let y = Math.round(rect.top + rect.height / 2);
+                
+            // Handle inline elements (e.g., word-wrapped links)
+            if (el.tagName.toLowerCase() === 'a') {
+                const rects = el.getClientRects();
+                if (rects.length > 0) {
+                    x = Math.round(rects[0].left + rects[0].width / 2); // First line’s middle position
+                    y = Math.round(rects[0].top + rects[0].height / 2); // First line’s middle position
+                }
+            }
+        
+            return { x, y, width: rect.width, height: rect.height, top: rect.top, left: rect.left, right: rect.right, bottom: rect.bottom };
+        };
+
+        // Filter and sort elements by visual importance
+        const filterInteractiveElements = (allElements) =>
+            Array.from(allElements).filter(el => (isInteractive(el) && isElementInViewport(el) && isElementVisible(el) && isElementNotCovered(el)));
+
+            let allElementsData = [];
+            const allElements = document.querySelectorAll('*');
+            const interactiveElements = filterInteractiveElements(allElements);
+            for(let index=0; index < interactiveElements.length; index++) {
+                let el = interactiveElements[index];
+                const rect = el.getBoundingClientRect();
+                const tagName = el.tagName.toLowerCase();
+                const role = el.getAttribute('role') || undefined;
+                let placeHolderProp = el.getAttribute('placeholder') ? {placeholder: el.getAttribute('placeholder')} : {};
+                let labelProp = el.getAttribute('label') ? {label: el.getAttribute('label')} : {};
+                let ariaLabelProp = el.getAttribute('aria-label') ? {ariaLabel: el.getAttribute('aria-label')} : {};
+                let inputType = el.getAttribute('type') ? {buttonType: el.getAttribute('type')} : {};
+
+                // Color coding system
+                const colors = ['#0066cc', '#00aa00', '#ff9900', '#6600cc', '#cc0000']
+                let color = colors[index % colors.length];
+            
+                // Add highlight border
+                el.setAttribute('tpp-data-border', el.style.border);
+                el.setAttribute('tpp-data-border-radius', el.style.borderRadius);
+                el.style.border = `2px solid ${color}`;
+                el.style.borderRadius = '3px';
+
+                // Create index badge
+                const label = document.createElement('div');
+
+                const labelWidth = 30;
+                const labelHeight = 20;
+
+                // Determine if the element is "large enough" for inner placement
+                const isLargeElement = rect.width > 100 && rect.height > 50;
+
+                if (isLargeElement) {
+                    // Place the label inside the element (inner-right)
+                    label.style.left = `${rect.right - labelWidth}px`;
+                    label.style.top = `${rect.top}px`;
+                } else {
+                    // Place it outside but keep it within the viewport
+                    const adjustedLeft = Math.min(rect.right, window.innerWidth - labelWidth);
+                    label.style.left = `${adjustedLeft}px`;
+
+                    // Ensure it doesn’t overflow the bottom of the screen
+                    const adjustedTop = Math.min(rect.top, window.innerHeight - labelHeight);
+                    label.style.top = `${adjustedTop}px`;
+                }
+
+                label.textContent = index;
+                label.style.position = 'absolute';
+                label.style.color = '#ffffff';
+                label.style.fontSize = '12px';
+                label.style.fontWeight = 'bold';
+                label.style.zIndex = '10000';
+                label.style.backgroundColor = color;
+                label.style.padding = '2px 8px';
+                label.style.borderRadius = '4px';
+                label.setAttribute('tpp-data-index-label', `${index}`);
+                document.body.appendChild(label);
+                
+                let elementData = {
+                    index,
+                    type: tagName,
+                    role,
+                    text: el.textContent?.slice(0, 50) || '',
+                    ...getAccuratePosition(el),
+                    ...placeHolderProp,
+                    ...labelProp,
+                    ...ariaLabelProp,
+                    ...inputType,
+                    value: el.value,
+                }
+
+                // Add all aria attributes 
+                const ariaAttributes = Array.from(el.attributes)
+                    .filter(attr => attr.name.includes('aria'))
+                    .reduce((acc, attr) => {
+                        acc[attr.name] = attr.value;
+                        return acc;
+                    }, {});
+                elementData = { ...elementData, ...ariaAttributes };
+
+                // Store element metadata
+                allElementsData.push(elementData);
+            }
+        return allElementsData;
+    });
+
+    filePath = getScreenshotPath(`web_executor_agent_snap_${Date.now()}`);
+    await page.screenshot({ animations: 'disabled', path: filePath, timeout: 5000, });
+    
+ } catch (ex) {
+    throw new Error(`Failed to fetch web state: ${ex.message}`);
+ } finally {
+    //Remove styles
+    await page.evaluate(() => {
+        document.querySelectorAll('[tpp-data-border]').forEach(el => {
+            el.style.border = el.getAttribute('tpp-data-border') || '';
+            el.style.borderRadius = el.getAttribute('tpp-data-border-radius') || '';
+            el.removeAttribute('tpp-data-border');
+            el.removeAttribute('tpp-data-border-radius');
+        });
+
+        document.querySelectorAll('[tpp-data-index-label]').forEach(el => el.remove());
+    })
+}
+
+    return {
+        screenshot: filePath ? path.resolve(filePath) : '',
+        elementsData: allElementsData,
+    }
+
+};
+
+async function getLocatorForScreenshot(page, tppEvent) {  
+    if (!tppEvent.target) tppEvent.target = tppEvent.locator
+    let mainLocator = tppEvent.target.replace('${rowId}', tppEvent.params?.rowId || 1).replace('${colId}', tppEvent.params?.colId || 1);
+    let elementHandle;
+    try {
+        elementHandle = await page.$(mainLocator);
+    } catch (e) {
+        try {
+        elementHandle = await page.$(`xpath=${mainLocator}`);
+        } catch (e) {}
+    }  
+
+    if (elementHandle) return elementHandle;
+
+    for (let backup of tppEvent?.backupLocators || []) {
+        let backupLocator = backup.locator.replace('${rowId}', tppEvent.params?.rowId || 1).replace('${colId}', tppEvent.params?.colId || 1);
+        try {
+        elementHandle = await page.$(backupLocator);
+        } catch (e) {
+
+        try {
+            elementHandle = await page.$(`xpath=${backupLocator}`);
+        } catch (e) {}
+        }
+        if (elementHandle) {
+        return elementHandle;
+        }
+    }
+    return null;
+}
+
+async function takeElementScreenshot(page, locator, targetPath, id, logger) {
+    try {
+        const screenshotPath = path.join(targetPath, 'autoheal_snaps', `${id}.png`);
+        const dir = path.dirname(screenshotPath);
+        if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+        }
+        const elementHandle = await getLocatorForScreenshot(page, locator);
+        const attrs = await page.evaluate((element) => {
+            const eventElement = window.getTppEventElement
+                ? window.getTppEventElement(element, 'click', {})
+                : null;
+            return eventElement.attrs || "";
+        }, elementHandle);
+        if (!elementHandle) {
+        return null;
+        }
+        const boundingBox = await elementHandle.boundingBox();
+        if (!boundingBox) return;
+        await page.evaluate((box) => {
+        const overlay = document.createElement('div');
+        overlay.id = 'tpp-data-screenshot-overlay';
+        overlay.style.cssText = `
+            position: fixed;
+            left: ${box.x}px;
+            top: ${box.y}px;
+            width: ${box.width}px;
+            height: ${box.height}px;
+            border: 3px dashed #ff1493;
+            background: rgba(255, 20, 147, 0.1);
+            pointer-events: none;
+            z-index: 9999;
+            box-sizing: border-box;
+        `;
+        document.body.appendChild(overlay);
+        }, boundingBox);
+        // scroll the eleemnt handle into view
+        await elementHandle.scrollIntoViewIfNeeded();
+        const cdpSession = await page.context().newCDPSession(page);
+        await cdpSession.send('Page.enable');
+        await cdpSession.send('Runtime.enable');
+        const screenshot = await cdpSession.send('Page.captureScreenshot', {
+        format: 'png',
+        });
+        await page.evaluate(() => {
+        const overlay = document.getElementById('tpp-data-screenshot-overlay');
+        if(overlay) overlay.remove();
+        });
+        fs.writeFileSync(screenshotPath, screenshot.data, 'base64');
+        logger.log(`[AUTOHEAL_DATA] ${JSON.stringify({ screenshotPath, element: locator.label, testLabel: logger.testLabel, attrs, })}`);
+        await elementHandle.dispose();
+        return screenshotPath;
+    } catch (err) {    
+        return null;
+    }
+}
+
+exports.fetchWebState = fetchWebState;
+exports.takeElementScreenshot = takeElementScreenshot;