mx-cloud 0.0.9 → 0.0.10

package/build/browserSide/scraper.js

@@ -359,10 +359,55 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
    */
     window.scrapeList = function (_a) {
         return __awaiter(this, arguments, void 0, function* ({ listSelector, fields, limit = 10 }) {
-            // Enhanced query function to handle iframe, frame and shadow DOM
+            var _b;
+            // XPath evaluation functions
+            const evaluateXPath = (rootElement, xpath) => {
+                try {
+                    const ownerDoc = rootElement.nodeType === Node.DOCUMENT_NODE
+                        ? rootElement
+                        : rootElement.ownerDocument;
+                    if (!ownerDoc)
+                        return null;
+                    const result = ownerDoc.evaluate(xpath, rootElement, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+                    return result.singleNodeValue;
+                }
+                catch (error) {
+                    console.warn('XPath evaluation failed:', xpath, error);
+                    return null;
+                }
+            };
+            const evaluateXPathAll = (rootElement, xpath) => {
+                try {
+                    const ownerDoc = rootElement.nodeType === Node.DOCUMENT_NODE
+                        ? rootElement
+                        : rootElement.ownerDocument;
+                    if (!ownerDoc)
+                        return [];
+                    const result = ownerDoc.evaluate(xpath, rootElement, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                    const elements = [];
+                    for (let i = 0; i < result.snapshotLength; i++) {
+                        const node = result.snapshotItem(i);
+                        if (node && node.nodeType === Node.ELEMENT_NODE) {
+                            elements.push(node);
+                        }
+                    }
+                    return elements;
+                }
+                catch (error) {
+                    console.warn('XPath evaluation failed:', xpath, error);
+                    return [];
+                }
+            };
+            // Enhanced query function to handle iframe, frame, shadow DOM, and XPath
             const queryElement = (rootElement, selector) => {
                 if (!selector.includes('>>') && !selector.includes(':>>')) {
-                    return rootElement.querySelector(selector);
+                    // Check if it's an XPath selector
+                    if (selector.startsWith('//') || selector.startsWith('/') || selector.startsWith('./')) {
+                        return evaluateXPath(rootElement, selector);
+                    }
+                    else {
+                        return rootElement.querySelector(selector);
+                    }
                 }
                 const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
                 let currentElement = rootElement;
@@ -373,7 +418,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') {
                         try {
                             const frameDoc = currentElement.contentDocument || currentElement.contentWindow.document;
-                            currentElement = frameDoc.querySelector(parts[i]);
+                            if (!frameDoc)
+                                return null;
+                            if (parts[i].startsWith('//') || parts[i].startsWith('/') || parts[i].startsWith('./')) {
+                                currentElement = evaluateXPath(frameDoc, parts[i]);
+                            }
+                            else {
+                                currentElement = frameDoc.querySelector(parts[i]);
+                            }
                             continue;
                         }
                         catch (e) {
@@ -381,18 +433,36 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                             return null;
                         }
                     }
-                    // Try regular DOM first
-                    let nextElement = currentElement.querySelector(parts[i]);
+                    let nextElement = null;
+                    if ('querySelector' in currentElement) {
+                        // Handle XPath vs CSS selector
+                        if (parts[i].startsWith('//') || parts[i].startsWith('/') || parts[i].startsWith('./')) {
+                            nextElement = evaluateXPath(currentElement, parts[i]);
+                        }
+                        else {
+                            nextElement = currentElement.querySelector(parts[i]);
+                        }
+                    }
                     // Try shadow DOM if not found
-                    if (!nextElement && currentElement.shadowRoot) {
-                        nextElement = currentElement.shadowRoot.querySelector(parts[i]);
+                    if (!nextElement && 'shadowRoot' in currentElement && currentElement.shadowRoot) {
+                        if (parts[i].startsWith('//') || parts[i].startsWith('/') || parts[i].startsWith('./')) {
+                            nextElement = evaluateXPath(currentElement.shadowRoot, parts[i]);
+                        }
+                        else {
+                            nextElement = currentElement.shadowRoot.querySelector(parts[i]);
+                        }
                     }
                     // Check children's shadow roots if still not found
-                    if (!nextElement) {
+                    if (!nextElement && 'children' in currentElement) {
                         const children = Array.from(currentElement.children || []);
                         for (const child of children) {
                             if (child.shadowRoot) {
-                                nextElement = child.shadowRoot.querySelector(parts[i]);
+                                if (parts[i].startsWith('//') || parts[i].startsWith('/') || parts[i].startsWith('./')) {
+                                    nextElement = evaluateXPath(child.shadowRoot, parts[i]);
+                                }
+                                else {
+                                    nextElement = child.shadowRoot.querySelector(parts[i]);
+                                }
                                 if (nextElement)
                                     break;
                             }
@@ -402,10 +472,16 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 }
                 return currentElement;
             };
-            // Enhanced query all function for both contexts
+            // Enhanced query all function for XPath and CSS selectors
             const queryElementAll = (rootElement, selector) => {
                 if (!selector.includes('>>') && !selector.includes(':>>')) {
-                    return rootElement.querySelectorAll(selector);
+                    // Check if it's an XPath selector
+                    if (selector.startsWith('//') || selector.startsWith('/') || selector.startsWith('./')) {
+                        return evaluateXPathAll(rootElement, selector);
+                    }
+                    else {
+                        return Array.from(rootElement.querySelectorAll(selector));
+                    }
                 }
                 const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
                 let currentElements = [rootElement];
@@ -416,7 +492,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         if (element.tagName === 'IFRAME' || element.tagName === 'FRAME') {
                             try {
                                 const frameDoc = element.contentDocument || element.contentWindow.document;
-                                nextElements.push(...frameDoc.querySelectorAll(part));
+                                if (frameDoc) {
+                                    if (part.startsWith('//') || part.startsWith('/') || part.startsWith('./')) {
+                                        nextElements.push(...evaluateXPathAll(frameDoc, part));
+                                    }
+                                    else {
+                                        nextElements.push(...Array.from(frameDoc.querySelectorAll(part)));
+                                    }
+                                }
                             }
                             catch (e) {
                                 console.warn(`Cannot access ${element.tagName.toLowerCase()} content:`, e);
@@ -426,17 +509,32 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         else {
                             // Regular DOM elements
                             if (element.querySelectorAll) {
-                                nextElements.push(...element.querySelectorAll(part));
+                                if (part.startsWith('//') || part.startsWith('/') || part.startsWith('./')) {
+                                    nextElements.push(...evaluateXPathAll(element, part));
+                                }
+                                else {
+                                    nextElements.push(...Array.from(element.querySelectorAll(part)));
+                                }
                             }
                             // Shadow DOM elements
                             if (element.shadowRoot) {
-                                nextElements.push(...element.shadowRoot.querySelectorAll(part));
+                                if (part.startsWith('//') || part.startsWith('/') || part.startsWith('./')) {
+                                    nextElements.push(...evaluateXPathAll(element.shadowRoot, part));
+                                }
+                                else {
+                                    nextElements.push(...Array.from(element.shadowRoot.querySelectorAll(part)));
+                                }
                             }
                             // Check children's shadow roots
                             const children = Array.from(element.children || []);
                             for (const child of children) {
                                 if (child.shadowRoot) {
-                                    nextElements.push(...child.shadowRoot.querySelectorAll(part));
+                                    if (part.startsWith('//') || part.startsWith('/') || part.startsWith('./')) {
+                                        nextElements.push(...evaluateXPathAll(child.shadowRoot, part));
+                                    }
+                                    else {
+                                        nextElements.push(...Array.from(child.shadowRoot.querySelectorAll(part)));
+                                    }
                                 }
                             }
                         }
@@ -446,8 +544,8 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 return currentElements;
             };
             // Enhanced value extraction with context awareness
-            function extractValue(element, attribute) {
-                var _a, _b;
+            const extractValue = (element, attribute) => {
+                var _a, _b, _c, _d, _e, _f;
                 if (!element)
                     return null;
                 // Get context-aware base URL
@@ -460,36 +558,58 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     }
                 }
                 if (attribute === 'innerText') {
-                    return element.innerText.trim();
+                    // First try standard innerText/textContent
+                    let textContent = ((_c = element.innerText) === null || _c === void 0 ? void 0 : _c.trim()) || ((_d = element.textContent) === null || _d === void 0 ? void 0 : _d.trim());
+                    // If empty, check for common data attributes that might contain the text
+                    if (!textContent) {
+                        const dataAttributes = [
+                            'data-600',
+                            'data-text',
+                            'data-label',
+                            'data-value',
+                            'data-content',
+                        ];
+                        for (const attr of dataAttributes) {
+                            const dataValue = element.getAttribute(attr);
+                            if (dataValue && dataValue.trim()) {
+                                textContent = dataValue.trim();
+                                break;
+                            }
+                        }
+                    }
+                    return textContent || null;
                 }
                 else if (attribute === 'innerHTML') {
-                    return element.innerHTML.trim();
+                    return ((_e = element.innerHTML) === null || _e === void 0 ? void 0 : _e.trim()) || null;
                 }
-                else if (attribute === 'src' || attribute === 'href') {
-                    if (attribute === 'href' && element.tagName !== 'A') {
-                        const parentElement = element.parentElement;
-                        if (parentElement && parentElement.tagName === 'A') {
-                            const parentHref = parentElement.getAttribute('href');
-                            if (parentHref) {
-                                try {
-                                    return new URL(parentHref, baseURL).href;
-                                }
-                                catch (e) {
-                                    return parentHref;
-                                }
-                            }
-                        }
+                else if (attribute === 'href') {
+                    // For href, we need to find the anchor tag if the current element isn't one
+                    let anchorElement = element;
+                    // If current element is not an anchor, look for parent anchor
+                    if (element.tagName !== 'A') {
+                        anchorElement = element.closest('a') || ((_f = element.parentElement) === null || _f === void 0 ? void 0 : _f.closest('a')) || element;
+                    }
+                    const hrefValue = anchorElement.getAttribute('href');
+                    if (!hrefValue || hrefValue.trim() === '') {
+                        return null;
+                    }
+                    try {
+                        return new URL(hrefValue, baseURL).href;
+                    }
+                    catch (e) {
+                        console.warn('Error creating URL from', hrefValue, e);
+                        return hrefValue;
                     }
+                }
+                else if (attribute === 'src') {
                     const attrValue = element.getAttribute(attribute);
                     const dataAttr = attrValue || element.getAttribute('data-' + attribute);
                     if (!dataAttr || dataAttr.trim() === '') {
-                        if (attribute === 'src') {
-                            const style = window.getComputedStyle(element);
-                            const bgImage = style.backgroundImage;
-                            if (bgImage && bgImage !== 'none') {
-                                const matches = bgImage.match(/url\(['"]?([^'")]+)['"]?\)/);
-                                return matches ? new URL(matches[1], baseURL).href : null;
-                            }
+                        const style = window.getComputedStyle(element);
+                        const bgImage = style.backgroundImage;
+                        if (bgImage && bgImage !== 'none') {
+                            const matches = bgImage.match(/url\(['"]?([^'")]+)['"]?\)/);
+                            return matches ? new URL(matches[1], baseURL).href : null;
                         }
                         return null;
                     }
@@ -498,327 +618,101 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     }
                     catch (e) {
                         console.warn('Error creating URL from', dataAttr, e);
-                        return dataAttr; // Return the original value if URL construction fails
+                        return dataAttr;
                     }
                 }
                 return element.getAttribute(attribute);
-            }
-            // Enhanced table ancestor finding with context support
-            function findTableAncestor(element) {
-                let currentElement = element;
-                const MAX_DEPTH = 5;
-                let depth = 0;
-                while (currentElement && depth < MAX_DEPTH) {
-                    // Handle shadow DOM
-                    if (currentElement.getRootNode() instanceof ShadowRoot) {
-                        currentElement = currentElement.getRootNode().host;
-                        continue;
-                    }
-                    if (currentElement.tagName === 'TD') {
-                        return { type: 'TD', element: currentElement };
-                    }
-                    else if (currentElement.tagName === 'TR') {
-                        return { type: 'TR', element: currentElement };
-                    }
-                    // Handle iframe and frame crossing
-                    if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') {
-                        try {
-                            currentElement = currentElement.contentDocument.body;
-                        }
-                        catch (e) {
-                            return null;
-                        }
-                    }
-                    else {
-                        currentElement = currentElement.parentElement;
-                    }
-                    depth++;
-                }
-                return null;
-            }
-            // Helper function to get cell index
-            function getCellIndex(td) {
-                if (td.getRootNode() instanceof ShadowRoot) {
-                    const shadowRoot = td.getRootNode();
-                    const allCells = Array.from(shadowRoot.querySelectorAll('td'));
-                    return allCells.indexOf(td);
-                }
-                let index = 0;
-                let sibling = td;
-                while (sibling = sibling.previousElementSibling) {
-                    index++;
-                }
-                return index;
-            }
-            // Helper function to check for TH elements
-            function hasThElement(row, tableFields) {
-                for (const [_, { selector }] of Object.entries(tableFields)) {
-                    const element = queryElement(row, selector);
-                    if (element) {
-                        let current = element;
-                        while (current && current !== row) {
-                            if (current.getRootNode() instanceof ShadowRoot) {
-                                current = current.getRootNode().host;
-                                continue;
-                            }
-                            if (current.tagName === 'TH')
-                                return true;
-                            if (current.tagName === 'IFRAME' || current.tagName === 'FRAME') {
-                                try {
-                                    current = current.contentDocument.body;
-                                }
-                                catch (e) {
-                                    break;
-                                }
-                            }
-                            else {
-                                current = current.parentElement;
-                            }
-                        }
-                    }
-                }
-                return false;
-            }
-            // Helper function to filter rows
-            function filterRowsBasedOnTag(rows, tableFields) {
-                for (const row of rows) {
-                    if (hasThElement(row, tableFields)) {
-                        return rows;
-                    }
-                }
-                // Include shadow DOM in TH search
-                return rows.filter(row => {
-                    const directTH = row.getElementsByTagName('TH').length === 0;
-                    const shadowTH = row.shadowRoot ?
-                        row.shadowRoot.querySelector('th') === null : true;
-                    return directTH && shadowTH;
-                });
-            }
-            // Class similarity comparison functions
-            function calculateClassSimilarity(classList1, classList2) {
-                const set1 = new Set(classList1);
-                const set2 = new Set(classList2);
-                const intersection = new Set([...set1].filter(x => set2.has(x)));
-                const union = new Set([...set1, ...set2]);
-                return intersection.size / union.size;
-            }
-            // Enhanced similar elements finding with context support
-            function findSimilarElements(baseElement, similarityThreshold = 0.7) {
-                const baseClasses = Array.from(baseElement.classList);
-                if (baseClasses.length === 0)
-                    return [];
-                const allElements = [];
-                // Get elements from main document
-                allElements.push(...document.getElementsByTagName(baseElement.tagName));
-                // Get elements from shadow DOM
-                if (baseElement.getRootNode() instanceof ShadowRoot) {
-                    const shadowHost = baseElement.getRootNode().host;
-                    allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName));
+            };
+            // Create indexed XPath for specific container instance
+            const createIndexedXPath = (childSelector, listSelector, containerIndex) => {
+                console.log(`Creating indexed XPath for container ${containerIndex}`);
+                console.log(`Child selector: ${childSelector}`);
+                console.log(`List selector: ${listSelector}`);
+                // Check if the child selector contains the list selector pattern
+                if (childSelector.includes(listSelector.replace('//', ''))) {
+                    // Replace the list selector part with indexed version
+                    const listPattern = listSelector.replace('//', '');
+                    const indexedListSelector = `(${listSelector})[${containerIndex}]`;
+                    const indexedSelector = childSelector.replace(`//${listPattern}`, indexedListSelector);
+                    console.log(`Generated indexed selector: ${indexedSelector}`);
+                    return indexedSelector;
                 }
-                // Get elements from iframes and frames
-                const frames = [
-                    ...Array.from(document.getElementsByTagName('iframe')),
-                    ...Array.from(document.getElementsByTagName('frame'))
-                ];
-                for (const frame of frames) {
-                    try {
-                        const frameDoc = frame.contentDocument || frame.contentWindow.document;
-                        allElements.push(...frameDoc.getElementsByTagName(baseElement.tagName));
-                    }
-                    catch (e) {
-                        console.warn(`Cannot access ${frame.tagName.toLowerCase()} content:`, e);
-                    }
+                else {
+                    // If pattern doesn't match, create a more generic indexed selector
+                    console.warn(`Pattern doesn't match, using fallback approach`);
+                    return `(${listSelector})[${containerIndex}]${childSelector.replace('//', '/')}`;
                 }
-                return allElements.filter(element => {
-                    if (element === baseElement)
-                        return false;
-                    const similarity = calculateClassSimilarity(baseClasses, Array.from(element.classList));
-                    return similarity >= similarityThreshold;
-                });
-            }
-            function tryFallbackSelector(rootElement, originalSelector) {
-                let element = queryElement(rootElement, originalSelector);
-                if (!element && originalSelector.includes('nth-child')) {
-                    const match = originalSelector.match(/nth-child\((\d+)\)/);
-                    if (match) {
-                        const position = parseInt(match[1], 10);
-                        for (let i = position - 1; i >= 1; i--) {
-                            const fallbackSelector = originalSelector.replace(/nth-child\(\d+\)/, `nth-child(${i})`);
-                            element = queryElement(rootElement, fallbackSelector);
-                            if (element)
-                                break;
-                        }
-                        if (!element) {
-                            const baseSelector = originalSelector.replace(/\:nth-child\(\d+\)/, '');
-                            element = queryElement(rootElement, baseSelector);
-                        }
-                    }
-                }
-                return element;
-            }
-            // Main scraping logic with context support
-            let containers = queryElementAll(document, listSelector);
-            containers = Array.from(containers);
-            if (containers.length === 0)
+            };
+            // Main scraping logic
+            console.log('🚀 Starting list data extraction');
+            console.log('List Selector:', listSelector);
+            console.log('Fields:', fields);
+            // Step 1: Get all container elements matching the list selector
+            const containers = queryElementAll(document, listSelector);
+            console.log(`📦 Found ${containers.length} list containers`);
+            if (containers.length === 0) {
+                console.warn('❌ No containers found for listSelector:', listSelector);
                 return [];
-            if (limit > 1 && containers.length === 1) {
-                const baseContainer = containers[0];
-                const similarContainers = findSimilarElements(baseContainer);
-                if (similarContainers.length > 0) {
-                    const newContainers = similarContainers.filter(container => !container.matches(listSelector));
-                    containers = [...containers, ...newContainers];
-                }
             }
-            const containerFields = containers.map(() => ({
-                tableFields: {},
-                nonTableFields: {}
-            }));
-            // Classify fields
-            containers.forEach((container, containerIndex) => {
+            // Step 2: Extract data from each container up to the limit
+            const extractedData = [];
+            const containersToProcess = Math.min(containers.length, limit);
+            console.log(`🔄 Processing ${containersToProcess} containers...`);
+            for (let containerIndex = 0; containerIndex < containersToProcess; containerIndex++) {
+                const container = containers[containerIndex];
+                const record = {};
+                console.log(`\n📋 Processing container ${containerIndex + 1}/${containersToProcess}`);
+                // Step 3: For each field, extract data from the current container
                 for (const [label, field] of Object.entries(fields)) {
-                    const sampleElement = queryElement(container, field.selector);
-                    if (sampleElement) {
-                        const ancestor = findTableAncestor(sampleElement);
-                        if (ancestor) {
-                            containerFields[containerIndex].tableFields[label] = Object.assign(Object.assign({}, field), { tableContext: ancestor.type, cellIndex: ancestor.type === 'TD' ? getCellIndex(ancestor.element) : -1 });
-                        }
-                        else {
-                            containerFields[containerIndex].nonTableFields[label] = field;
+                    console.log(`\n  🔍 Extracting field "${label}"`);
+                    console.log(`    Original selector: ${field.selector}`);
+                    console.log(`    Attribute: ${field.attribute}`);
+                    let element = null;
+                    // Handle XPath selectors with container indexing
+                    if (field.selector.startsWith('//')) {
+                        // Create indexed absolute XPath
+                        const indexedSelector = createIndexedXPath(field.selector, listSelector, containerIndex + 1);
+                        console.log(`    📍 Indexed selector: ${indexedSelector}`);
+                        element = evaluateXPath(document, indexedSelector);
+                        console.log(`    📍 Indexed XPath result: ${element ? 'FOUND' : 'NOT FOUND'}`);
+                        if (element) {
+                            console.log(`    📍 Found element text: "${(_b = element.textContent) === null || _b === void 0 ? void 0 : _b.trim()}"`);
                         }
                     }
                     else {
-                        containerFields[containerIndex].nonTableFields[label] = field;
+                        // Fallback for non-XPath selectors - search within container
+                        element = queryElement(container, field.selector);
                     }
-                }
-            });
-            const tableData = [];
-            const nonTableData = [];
-            // Process table data with support for iframes, frames, and shadow DOM
-            for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
-                const container = containers[containerIndex];
-                const { tableFields } = containerFields[containerIndex];
-                if (Object.keys(tableFields).length > 0) {
-                    const firstField = Object.values(tableFields)[0];
-                    const firstElement = queryElement(container, firstField.selector);
-                    let tableContext = firstElement;
-                    // Find table context including iframe, frame and shadow DOM
-                    while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) {
-                        if (tableContext.getRootNode() instanceof ShadowRoot) {
-                            tableContext = tableContext.getRootNode().host;
-                            continue;
-                        }
-                        if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') {
-                            try {
-                                tableContext = tableContext.contentDocument.body;
-                            }
-                            catch (e) {
-                                break;
-                            }
+                    // Step 4: Extract the value from the found element
+                    if (element) {
+                        const value = extractValue(element, field.attribute);
+                        if (value !== null && value !== '') {
+                            record[label] = value;
+                            console.log(`    ✅ Extracted "${label}": "${value}"`);
                         }
                         else {
-                            tableContext = tableContext.parentElement;
+                            console.warn(`    ⚠️ Empty value for "${label}"`);
+                            record[label] = '';
                         }
                     }
-                    if (tableContext) {
-                        // Get rows from all contexts
-                        const rows = [];
-                        // Get rows from regular DOM
-                        rows.push(...tableContext.getElementsByTagName('TR'));
-                        // Get rows from shadow DOM
-                        if (tableContext.shadowRoot) {
-                            rows.push(...tableContext.shadowRoot.getElementsByTagName('TR'));
-                        }
-                        // Get rows from iframes and frames
-                        if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') {
-                            try {
-                                const frameDoc = tableContext.contentDocument || tableContext.contentWindow.document;
-                                rows.push(...frameDoc.getElementsByTagName('TR'));
-                            }
-                            catch (e) {
-                                console.warn(`Cannot access ${tableContext.tagName.toLowerCase()} rows:`, e);
-                            }
-                        }
-                        const processedRows = filterRowsBasedOnTag(rows, tableFields);
-                        for (let rowIndex = 0; rowIndex < Math.min(processedRows.length, limit); rowIndex++) {
-                            const record = {};
-                            const currentRow = processedRows[rowIndex];
-                            for (const [label, { selector, attribute, cellIndex }] of Object.entries(tableFields)) {
-                                let element = null;
-                                if (cellIndex >= 0) {
-                                    // Get TD element considering both contexts
-                                    let td = currentRow.children[cellIndex];
-                                    // Check shadow DOM for td
-                                    if (!td && currentRow.shadowRoot) {
-                                        const shadowCells = currentRow.shadowRoot.children;
-                                        if (shadowCells && shadowCells.length > cellIndex) {
-                                            td = shadowCells[cellIndex];
-                                        }
-                                    }
-                                    if (td) {
-                                        element = queryElement(td, selector);
-                                        if (!element && selector.split(/(?:>>|:>>)/).pop().includes('td:nth-child')) {
-                                            element = td;
-                                        }
-                                        if (!element) {
-                                            const tagOnlySelector = selector.split('.')[0];
-                                            element = queryElement(td, tagOnlySelector);
-                                        }
-                                        if (!element) {
-                                            let currentElement = td;
-                                            while (currentElement && currentElement.children.length > 0) {
-                                                let foundContentChild = false;
-                                                for (const child of currentElement.children) {
-                                                    if (extractValue(child, attribute)) {
-                                                        currentElement = child;
-                                                        foundContentChild = true;
-                                                        break;
-                                                    }
-                                                }
-                                                if (!foundContentChild)
-                                                    break;
-                                            }
-                                            element = currentElement;
-                                        }
-                                    }
-                                }
-                                else {
-                                    element = queryElement(currentRow, selector);
-                                }
-                                if (element) {
-                                    record[label] = extractValue(element, attribute);
-                                }
-                            }
-                            if (Object.keys(record).length > 0) {
-                                tableData.push(record);
-                            }
-                        }
+                    else {
+                        console.warn(`    ❌ Element not found for "${label}"`);
+                        record[label] = '';
                     }
                 }
-            }
-            // Process non-table data with all contexts support
-            for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
-                if (nonTableData.length >= limit)
-                    break;
-                const container = containers[containerIndex];
-                const { nonTableFields } = containerFields[containerIndex];
-                if (Object.keys(nonTableFields).length > 0) {
-                    const record = {};
-                    for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
-                        // Get the last part of the selector after any context delimiter
-                        const relativeSelector = selector.split(/(?:>>|:>>)/).slice(-1)[0];
-                        const element = tryFallbackSelector(container, relativeSelector);
-                        if (element) {
-                            record[label] = extractValue(element, attribute);
-                        }
-                    }
-                    if (Object.keys(record).length > 0) {
-                        nonTableData.push(record);
-                    }
+                // Step 5: Add record if it has any non-empty values
+                if (Object.values(record).some(value => value !== '')) {
+                    extractedData.push(record);
+                    console.log(`  ✅ Added record ${containerIndex + 1}:`, record);
+                }
+                else {
+                    console.warn(`  ⚠️ Skipping empty record for container ${containerIndex + 1}`);
                 }
             }
-            // Merge and limit the results
-            const scrapedData = [...tableData, ...nonTableData];
-            return scrapedData;
+            console.log('\n🎉 Extraction complete!');
+            console.log(`📊 Total records extracted: ${extractedData.length}`);
+            console.log('📋 All records:', extractedData);
+            return extractedData;
         });
     };
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.9",
+  "version": "0.0.10",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",