maxun-core 0.0.17 → 0.0.19

package/build/browserSide/scraper.js

@@ -359,21 +359,80 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
    */
     window.scrapeList = function ({ listSelector, fields, limit = 10 }) {
         return __awaiter(this, void 0, void 0, function* () {
-            // Enhanced query function to handle iframe, frame and shadow DOM
+            // XPath evaluation functions
+            const evaluateXPath = (rootElement, xpath) => {
+                try {
+                    const ownerDoc = rootElement.nodeType === Node.DOCUMENT_NODE
+                        ? rootElement
+                        : rootElement.ownerDocument;
+                    if (!ownerDoc)
+                        return null;
+                    const result = ownerDoc.evaluate(xpath, rootElement, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+                    return result.singleNodeValue;
+                }
+                catch (error) {
+                    console.warn("XPath evaluation failed:", xpath, error);
+                    return null;
+                }
+            };
+            const evaluateXPathAll = (rootElement, xpath) => {
+                try {
+                    const ownerDoc = rootElement.nodeType === Node.DOCUMENT_NODE
+                        ? rootElement
+                        : rootElement.ownerDocument;
+                    if (!ownerDoc)
+                        return [];
+                    const result = ownerDoc.evaluate(xpath, rootElement, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                    const elements = [];
+                    for (let i = 0; i < result.snapshotLength; i++) {
+                        const node = result.snapshotItem(i);
+                        if (node && node.nodeType === Node.ELEMENT_NODE) {
+                            elements.push(node);
+                        }
+                    }
+                    return elements;
+                }
+                catch (error) {
+                    console.warn("XPath evaluation failed:", xpath, error);
+                    return [];
+                }
+            };
+            // Helper function to detect selector type
+            const isXPathSelector = (selector) => {
+                return (selector.startsWith("//") ||
+                    selector.startsWith("/") ||
+                    selector.startsWith("./"));
+            };
+            // Enhanced query function to handle iframe, frame, shadow DOM, CSS selectors, and XPath
             const queryElement = (rootElement, selector) => {
-                if (!selector.includes('>>') && !selector.includes(':>>')) {
-                    return rootElement.querySelector(selector);
+                if (!selector.includes(">>") && !selector.includes(":>>")) {
+                    // Check if it's an XPath selector
+                    if (isXPathSelector(selector)) {
+                        return evaluateXPath(rootElement, selector);
+                    }
+                    else {
+                        return rootElement.querySelector(selector);
+                    }
                 }
-                const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
+                const parts = selector.split(/(?:>>|:>>)/).map((part) => part.trim());
                 let currentElement = rootElement;
                 for (let i = 0; i < parts.length; i++) {
                     if (!currentElement)
                         return null;
                     // Handle iframe and frame traversal
-                    if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') {
+                    if (currentElement.tagName === "IFRAME" ||
+                        currentElement.tagName === "FRAME") {
                         try {
-                            const frameDoc = currentElement.contentDocument || currentElement.contentWindow.document;
-                            currentElement = frameDoc.querySelector(parts[i]);
+                            const frameDoc = currentElement.contentDocument ||
+                                currentElement.contentWindow.document;
+                            if (!frameDoc)
+                                return null;
+                            if (isXPathSelector(parts[i])) {
+                                currentElement = evaluateXPath(frameDoc, parts[i]);
+                            }
+                            else {
+                                currentElement = frameDoc.querySelector(parts[i]);
+                            }
                             continue;
                         }
                         catch (e) {
@@ -381,18 +440,38 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                             return null;
                         }
                     }
+                    let nextElement = null;
                     // Try regular DOM first
-                    let nextElement = currentElement.querySelector(parts[i]);
+                    if ("querySelector" in currentElement) {
+                        if (isXPathSelector(parts[i])) {
+                            nextElement = evaluateXPath(currentElement, parts[i]);
+                        }
+                        else {
+                            nextElement = currentElement.querySelector(parts[i]);
+                        }
+                    }
                     // Try shadow DOM if not found
-                    if (!nextElement && currentElement.shadowRoot) {
-                        nextElement = currentElement.shadowRoot.querySelector(parts[i]);
+                    if (!nextElement &&
+                        "shadowRoot" in currentElement &&
+                        currentElement.shadowRoot) {
+                        if (isXPathSelector(parts[i])) {
+                            nextElement = evaluateXPath(currentElement.shadowRoot, parts[i]);
+                        }
+                        else {
+                            nextElement = currentElement.shadowRoot.querySelector(parts[i]);
+                        }
                     }
                     // Check children's shadow roots if still not found
-                    if (!nextElement) {
+                    if (!nextElement && "children" in currentElement) {
                         const children = Array.from(currentElement.children || []);
                         for (const child of children) {
                             if (child.shadowRoot) {
-                                nextElement = child.shadowRoot.querySelector(parts[i]);
+                                if (isXPathSelector(parts[i])) {
+                                    nextElement = evaluateXPath(child.shadowRoot, parts[i]);
+                                }
+                                else {
+                                    nextElement = child.shadowRoot.querySelector(parts[i]);
+                                }
                                 if (nextElement)
                                     break;
                             }
@@ -404,19 +483,31 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             };
             // Enhanced query all function for both contexts
             const queryElementAll = (rootElement, selector) => {
-                if (!selector.includes('>>') && !selector.includes(':>>')) {
-                    return rootElement.querySelectorAll(selector);
+                if (!selector.includes(">>") && !selector.includes(":>>")) {
+                    if (isXPathSelector(selector)) {
+                        return evaluateXPathAll(rootElement, selector);
+                    }
+                    else {
+                        return Array.from(rootElement.querySelectorAll(selector));
+                    }
                 }
-                const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
+                const parts = selector.split(/(?:>>|:>>)/).map((part) => part.trim());
                 let currentElements = [rootElement];
                 for (const part of parts) {
                     const nextElements = [];
                     for (const element of currentElements) {
                         // Handle iframe and frame traversal
-                        if (element.tagName === 'IFRAME' || element.tagName === 'FRAME') {
+                        if (element.tagName === "IFRAME" || element.tagName === "FRAME") {
                             try {
                                 const frameDoc = element.contentDocument || element.contentWindow.document;
-                                nextElements.push(...frameDoc.querySelectorAll(part));
+                                if (frameDoc) {
+                                    if (isXPathSelector(part)) {
+                                        nextElements.push(...evaluateXPathAll(frameDoc, part));
+                                    }
+                                    else {
+                                        nextElements.push(...Array.from(frameDoc.querySelectorAll(part)));
+                                    }
+                                }
                             }
                             catch (e) {
                                 console.warn(`Cannot access ${element.tagName.toLowerCase()} content:`, e);
@@ -426,17 +517,32 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         else {
                             // Regular DOM elements
                             if (element.querySelectorAll) {
-                                nextElements.push(...element.querySelectorAll(part));
+                                if (isXPathSelector(part)) {
+                                    nextElements.push(...evaluateXPathAll(element, part));
+                                }
+                                else {
+                                    nextElements.push(...Array.from(element.querySelectorAll(part)));
+                                }
                             }
                             // Shadow DOM elements
                             if (element.shadowRoot) {
-                                nextElements.push(...element.shadowRoot.querySelectorAll(part));
+                                if (isXPathSelector(part)) {
+                                    nextElements.push(...evaluateXPathAll(element.shadowRoot, part));
+                                }
+                                else {
+                                    nextElements.push(...Array.from(element.shadowRoot.querySelectorAll(part)));
+                                }
                             }
                             // Check children's shadow roots
                             const children = Array.from(element.children || []);
                             for (const child of children) {
                                 if (child.shadowRoot) {
-                                    nextElements.push(...child.shadowRoot.querySelectorAll(part));
+                                    if (isXPathSelector(part)) {
+                                        nextElements.push(...evaluateXPathAll(child.shadowRoot, part));
+                                    }
+                                    else {
+                                        nextElements.push(...Array.from(child.shadowRoot.querySelectorAll(part)));
+                                    }
                                 }
                             }
                         }
@@ -446,8 +552,8 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 return currentElements;
             };
             // Enhanced value extraction with context awareness
-            function extractValue(element, attribute) {
-                var _a, _b;
+            const extractValue = (element, attribute) => {
+                var _a, _b, _c, _d, _e;
                 if (!element)
                     return null;
                 // Get context-aware base URL
@@ -459,17 +565,36 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         return shadowContent.trim();
                     }
                 }
-                if (attribute === 'innerText') {
-                    return element.innerText.trim();
+                if (attribute === "innerText") {
+                    // First try standard innerText/textContent
+                    let textContent = ((_c = element.innerText) === null || _c === void 0 ? void 0 : _c.trim()) || ((_d = element.textContent) === null || _d === void 0 ? void 0 : _d.trim());
+                    // If empty, check for common data attributes that might contain the text
+                    if (!textContent) {
+                        const dataAttributes = [
+                            "data-600",
+                            "data-text",
+                            "data-label",
+                            "data-value",
+                            "data-content",
+                        ];
+                        for (const attr of dataAttributes) {
+                            const dataValue = element.getAttribute(attr);
+                            if (dataValue && dataValue.trim()) {
+                                textContent = dataValue.trim();
+                                break;
+                            }
+                        }
+                    }
+                    return textContent || null;
                 }
-                else if (attribute === 'innerHTML') {
-                    return element.innerHTML.trim();
+                else if (attribute === "innerHTML") {
+                    return ((_e = element.innerHTML) === null || _e === void 0 ? void 0 : _e.trim()) || null;
                 }
-                else if (attribute === 'src' || attribute === 'href') {
-                    if (attribute === 'href' && element.tagName !== 'A') {
+                else if (attribute === "src" || attribute === "href") {
+                    if (attribute === "href" && element.tagName !== "A") {
                         const parentElement = element.parentElement;
-                        if (parentElement && parentElement.tagName === 'A') {
-                            const parentHref = parentElement.getAttribute('href');
+                        if (parentElement && parentElement.tagName === "A") {
+                            const parentHref = parentElement.getAttribute("href");
                             if (parentHref) {
                                 try {
                                     return new URL(parentHref, baseURL).href;
@@ -481,12 +606,12 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         }
                     }
                     const attrValue = element.getAttribute(attribute);
-                    const dataAttr = attrValue || element.getAttribute('data-' + attribute);
-                    if (!dataAttr || dataAttr.trim() === '') {
-                        if (attribute === 'src') {
+                    const dataAttr = attrValue || element.getAttribute("data-" + attribute);
+                    if (!dataAttr || dataAttr.trim() === "") {
+                        if (attribute === "src") {
                             const style = window.getComputedStyle(element);
                             const bgImage = style.backgroundImage;
-                            if (bgImage && bgImage !== 'none') {
+                            if (bgImage && bgImage !== "none") {
                                 const matches = bgImage.match(/url\(['"]?([^'")]+)['"]?\)/);
                                 return matches ? new URL(matches[1], baseURL).href : null;
                             }
@@ -497,14 +622,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         return new URL(dataAttr, baseURL).href;
                     }
                     catch (e) {
-                        console.warn('Error creating URL from', dataAttr, e);
-                        return dataAttr; // Return the original value if URL construction fails
+                        console.warn("Error creating URL from", dataAttr, e);
+                        return dataAttr;
                     }
                 }
                 return element.getAttribute(attribute);
-            }
+            };
             // Enhanced table ancestor finding with context support
-            function findTableAncestor(element) {
+            const findTableAncestor = (element) => {
                 let currentElement = element;
                 const MAX_DEPTH = 5;
                 let depth = 0;
@@ -514,14 +639,15 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         currentElement = currentElement.getRootNode().host;
                         continue;
                     }
-                    if (currentElement.tagName === 'TD') {
-                        return { type: 'TD', element: currentElement };
+                    if (currentElement.tagName === "TD") {
+                        return { type: "TD", element: currentElement };
                     }
-                    else if (currentElement.tagName === 'TR') {
-                        return { type: 'TR', element: currentElement };
+                    else if (currentElement.tagName === "TR") {
+                        return { type: "TR", element: currentElement };
                     }
                     // Handle iframe and frame crossing
-                    if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') {
+                    if (currentElement.tagName === "IFRAME" ||
+                        currentElement.tagName === "FRAME") {
                         try {
                             currentElement = currentElement.contentDocument.body;
                         }
@@ -535,23 +661,23 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     depth++;
                 }
                 return null;
-            }
+            };
             // Helper function to get cell index
-            function getCellIndex(td) {
+            const getCellIndex = (td) => {
                 if (td.getRootNode() instanceof ShadowRoot) {
                     const shadowRoot = td.getRootNode();
-                    const allCells = Array.from(shadowRoot.querySelectorAll('td'));
+                    const allCells = Array.from(shadowRoot.querySelectorAll("td"));
                     return allCells.indexOf(td);
                 }
                 let index = 0;
                 let sibling = td;
-                while (sibling = sibling.previousElementSibling) {
+                while ((sibling = sibling.previousElementSibling)) {
                     index++;
                 }
                 return index;
-            }
+            };
             // Helper function to check for TH elements
-            function hasThElement(row, tableFields) {
+            const hasThElement = (row, tableFields) => {
                 for (const [_, { selector }] of Object.entries(tableFields)) {
                     const element = queryElement(row, selector);
                     if (element) {
@@ -561,9 +687,9 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                                 current = current.getRootNode().host;
                                 continue;
                             }
-                            if (current.tagName === 'TH')
+                            if (current.tagName === "TH")
                                 return true;
-                            if (current.tagName === 'IFRAME' || current.tagName === 'FRAME') {
+                            if (current.tagName === "IFRAME" || current.tagName === "FRAME") {
                                 try {
                                     current = current.contentDocument.body;
                                 }
@@ -578,32 +704,32 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     }
                 }
                 return false;
-            }
+            };
             // Helper function to filter rows
-            function filterRowsBasedOnTag(rows, tableFields) {
+            const filterRowsBasedOnTag = (rows, tableFields) => {
                 for (const row of rows) {
                     if (hasThElement(row, tableFields)) {
                         return rows;
                     }
                 }
-                // Include shadow DOM in TH search
-                return rows.filter(row => {
-                    const directTH = row.getElementsByTagName('TH').length === 0;
-                    const shadowTH = row.shadowRoot ?
-                        row.shadowRoot.querySelector('th') === null : true;
+                return rows.filter((row) => {
+                    const directTH = row.getElementsByTagName("TH").length === 0;
+                    const shadowTH = row.shadowRoot
+                        ? row.shadowRoot.querySelector("th") === null
+                        : true;
                     return directTH && shadowTH;
                 });
-            }
+            };
             // Class similarity comparison functions
-            function calculateClassSimilarity(classList1, classList2) {
+            const calculateClassSimilarity = (classList1, classList2) => {
                 const set1 = new Set(classList1);
                 const set2 = new Set(classList2);
-                const intersection = new Set([...set1].filter(x => set2.has(x)));
+                const intersection = new Set([...set1].filter((x) => set2.has(x)));
                 const union = new Set([...set1, ...set2]);
                 return intersection.size / union.size;
-            }
+            };
             // Enhanced similar elements finding with context support
-            function findSimilarElements(baseElement, similarityThreshold = 0.7) {
+            const findSimilarElements = (baseElement, similarityThreshold = 0.7) => {
                 const baseClasses = Array.from(baseElement.classList);
                 if (baseClasses.length === 0)
                     return [];
@@ -617,8 +743,8 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 }
                 // Get elements from iframes and frames
                 const frames = [
-                    ...Array.from(document.getElementsByTagName('iframe')),
-                    ...Array.from(document.getElementsByTagName('frame'))
+                    ...Array.from(document.getElementsByTagName("iframe")),
+                    ...Array.from(document.getElementsByTagName("frame")),
                 ];
                 for (const frame of frames) {
                     try {
@@ -629,16 +755,16 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         console.warn(`Cannot access ${frame.tagName.toLowerCase()} content:`, e);
                     }
                 }
-                return allElements.filter(element => {
+                return allElements.filter((element) => {
                     if (element === baseElement)
                         return false;
                     const similarity = calculateClassSimilarity(baseClasses, Array.from(element.classList));
                     return similarity >= similarityThreshold;
                 });
-            }
-            function tryFallbackSelector(rootElement, originalSelector) {
+            };
+            const tryFallbackSelector = (rootElement, originalSelector) => {
                 let element = queryElement(rootElement, originalSelector);
-                if (!element && originalSelector.includes('nth-child')) {
+                if (!element && originalSelector.includes("nth-child")) {
                     const match = originalSelector.match(/nth-child\((\d+)\)/);
                     if (match) {
                         const position = parseInt(match[1], 10);
@@ -649,38 +775,100 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                                 break;
                         }
                         if (!element) {
-                            const baseSelector = originalSelector.replace(/\:nth-child\(\d+\)/, '');
+                            const baseSelector = originalSelector.replace(/\:nth-child\(\d+\)/, "");
                             element = queryElement(rootElement, baseSelector);
                         }
                     }
                 }
                 return element;
-            }
-            // Main scraping logic with context support
+            };
+            // Create indexed XPath for specific container instance
+            const createIndexedXPath = (childSelector, listSelector, containerIndex) => {
+                // Check if the child selector contains the list selector pattern
+                if (childSelector.includes(listSelector.replace("//", ""))) {
+                    // Replace the list selector part with indexed version
+                    const listPattern = listSelector.replace("//", "");
+                    const indexedListSelector = `(${listSelector})[${containerIndex}]`;
+                    const indexedSelector = childSelector.replace(`//${listPattern}`, indexedListSelector);
+                    return indexedSelector;
+                }
+                else {
+                    // If pattern doesn't match, create a more generic indexed selector
+                    return `(${listSelector})[${containerIndex}]${childSelector.replace("//", "/")}`;
+                }
+            };
+            // Main scraping logic with unified support for both CSS and XPath
+            console.log("🚀 Starting unified list data extraction");
+            console.log("List Selector:", listSelector);
+            console.log("Fields:", fields);
             let containers = queryElementAll(document, listSelector);
             containers = Array.from(containers);
-            if (containers.length === 0)
+            if (containers.length === 0) {
+                console.warn("❌ No containers found for listSelector:", listSelector);
                 return [];
-            if (limit > 1 && containers.length === 1) {
+            }
+            console.log(`📦 Found ${containers.length} list containers`);
+            // For CSS selectors, try to find similar containers if needed
+            if (!isXPathSelector(listSelector) &&
+                limit > 1 &&
+                containers.length === 1) {
                 const baseContainer = containers[0];
                 const similarContainers = findSimilarElements(baseContainer);
                 if (similarContainers.length > 0) {
-                    const newContainers = similarContainers.filter(container => !container.matches(listSelector));
+                    const newContainers = similarContainers.filter((container) => !container.matches(listSelector));
                     containers = [...containers, ...newContainers];
                 }
             }
             const containerFields = containers.map(() => ({
                 tableFields: {},
-                nonTableFields: {}
+                nonTableFields: {},
             }));
-            // Classify fields
+            // For XPath selectors, use the new approach
+            if (isXPathSelector(listSelector)) {
+                const extractedData = [];
+                const containersToProcess = Math.min(containers.length, limit);
+                for (let containerIndex = 0; containerIndex < containersToProcess; containerIndex++) {
+                    const record = {};
+                    for (const [label, field] of Object.entries(fields)) {
+                        let element = null;
+                        if (isXPathSelector(field.selector)) {
+                            // Create indexed absolute XPath
+                            const indexedSelector = createIndexedXPath(field.selector, listSelector, containerIndex + 1);
+                            element = evaluateXPath(document, indexedSelector);
+                        }
+                        else {
+                            // Fallback for CSS selectors within XPath containers
+                            const container = containers[containerIndex];
+                            element = queryElement(container, field.selector);
+                        }
+                        if (element) {
+                            const value = extractValue(element, field.attribute);
+                            if (value !== null && value !== "") {
+                                record[label] = value;
+                            }
+                            else {
+                                record[label] = "";
+                            }
+                        }
+                        else {
+                            record[label] = "";
+                        }
+                    }
+                    if (Object.values(record).some((value) => value !== "")) {
+                        extractedData.push(record);
+                    }
+                }
+                console.log(`📊 Total records extracted: ${extractedData.length}`);
+                return extractedData;
+            }
+            // For CSS selectors, use the original table-aware approach
             containers.forEach((container, containerIndex) => {
                 for (const [label, field] of Object.entries(fields)) {
                     const sampleElement = queryElement(container, field.selector);
                     if (sampleElement) {
                         const ancestor = findTableAncestor(sampleElement);
                         if (ancestor) {
-                            containerFields[containerIndex].tableFields[label] = Object.assign(Object.assign({}, field), { tableContext: ancestor.type, cellIndex: ancestor.type === 'TD' ? getCellIndex(ancestor.element) : -1 });
+                            containerFields[containerIndex].tableFields[label] = Object.assign(Object.assign({}, field), { tableContext: ancestor.type, cellIndex: ancestor.type === "TD" ? getCellIndex(ancestor.element) : -1 });
                         }
                         else {
                             containerFields[containerIndex].nonTableFields[label] = field;
@@ -702,12 +890,15 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     const firstElement = queryElement(container, firstField.selector);
                     let tableContext = firstElement;
                     // Find table context including iframe, frame and shadow DOM
-                    while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) {
+                    while (tableContext &&
+                        tableContext.tagName !== "TABLE" &&
+                        tableContext !== container) {
                         if (tableContext.getRootNode() instanceof ShadowRoot) {
                             tableContext = tableContext.getRootNode().host;
                             continue;
                         }
-                        if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') {
+                        if (tableContext.tagName === "IFRAME" ||
+                            tableContext.tagName === "FRAME") {
                             try {
                                 tableContext = tableContext.contentDocument.body;
                             }
@@ -723,16 +914,18 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         // Get rows from all contexts
                         const rows = [];
                         // Get rows from regular DOM
-                        rows.push(...tableContext.getElementsByTagName('TR'));
+                        rows.push(...tableContext.getElementsByTagName("TR"));
                         // Get rows from shadow DOM
                         if (tableContext.shadowRoot) {
-                            rows.push(...tableContext.shadowRoot.getElementsByTagName('TR'));
+                            rows.push(...tableContext.shadowRoot.getElementsByTagName("TR"));
                         }
                         // Get rows from iframes and frames
-                        if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') {
+                        if (tableContext.tagName === "IFRAME" ||
+                            tableContext.tagName === "FRAME") {
                             try {
-                                const frameDoc = tableContext.contentDocument || tableContext.contentWindow.document;
-                                rows.push(...frameDoc.getElementsByTagName('TR'));
+                                const frameDoc = tableContext.contentDocument ||
+                                    tableContext.contentWindow.document;
+                                rows.push(...frameDoc.getElementsByTagName("TR"));
                             }
                             catch (e) {
                                 console.warn(`Cannot access ${tableContext.tagName.toLowerCase()} rows:`, e);
@@ -742,7 +935,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         for (let rowIndex = 0; rowIndex < Math.min(processedRows.length, limit); rowIndex++) {
                             const record = {};
                             const currentRow = processedRows[rowIndex];
-                            for (const [label, { selector, attribute, cellIndex }] of Object.entries(tableFields)) {
+                            for (const [label, { selector, attribute, cellIndex },] of Object.entries(tableFields)) {
                                 let element = null;
                                 if (cellIndex >= 0) {
                                     // Get TD element considering both contexts
@@ -756,16 +949,21 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                                     }
                                     if (td) {
                                         element = queryElement(td, selector);
-                                        if (!element && selector.split(/(?:>>|:>>)/).pop().includes('td:nth-child')) {
+                                        if (!element &&
+                                            selector
+                                                .split(/(?:>>|:>>)/)
+                                                .pop()
+                                                .includes("td:nth-child")) {
                                             element = td;
                                         }
                                         if (!element) {
-                                            const tagOnlySelector = selector.split('.')[0];
+                                            const tagOnlySelector = selector.split(".")[0];
                                             element = queryElement(td, tagOnlySelector);
                                         }
                                         if (!element) {
                                             let currentElement = td;
-                                            while (currentElement && currentElement.children.length > 0) {
+                                            while (currentElement &&
+                                                currentElement.children.length > 0) {
                                                 let foundContentChild = false;
                                                 for (const child of currentElement.children) {
                                                     if (extractValue(child, attribute)) {
@@ -818,6 +1016,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             }
             // Merge and limit the results
             const scrapedData = [...tableData, ...nonTableData];
+            console.log(`📊 Total records extracted: ${scrapedData.length}`);
             return scrapedData;
         });
     };

package/build/interpret.d.ts

@@ -41,6 +41,7 @@ interface InterpreterOptions {
         activeId: (id: number) => void;
         debugMessage: (msg: string) => void;
         setActionType: (type: string) => void;
+        incrementScrapeListIndex: () => void;
     }>;
 }
 /**

package/build/interpret.js

@@ -385,7 +385,7 @@ class Interpreter extends events_1.EventEmitter {
                     yield this.options.serializableCallback([mergedResult]);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
-                    var _f;
+                    var _f, _g;
                     if ((_f = this.options.debugChannel) === null || _f === void 0 ? void 0 : _f.setActionType) {
                         this.options.debugChannel.setActionType('scrapeList');
                     }
@@ -394,6 +394,9 @@ class Interpreter extends events_1.EventEmitter {
                         return;
                     }
                     yield this.ensureScriptsLoaded(page);
+                    if ((_g = this.options.debugChannel) === null || _g === void 0 ? void 0 : _g.incrementScrapeListIndex) {
+                        this.options.debugChannel.incrementScrapeListIndex();
+                    }
                     if (!config.pagination) {
                         const scrapeResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
                         yield this.options.serializableCallback(scrapeResults);
@@ -404,8 +407,8 @@ class Interpreter extends events_1.EventEmitter {
                     }
                 }),
                 scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
-                    var _g;
-                    if ((_g = this.options.debugChannel) === null || _g === void 0 ? void 0 : _g.setActionType) {
+                    var _h;
+                    if ((_h = this.options.debugChannel) === null || _h === void 0 ? void 0 : _h.setActionType) {
                         this.options.debugChannel.setActionType('scrapeListAuto');
                     }
                     yield this.ensureScriptsLoaded(page);
@@ -415,8 +418,8 @@ class Interpreter extends events_1.EventEmitter {
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scroll: (pages) => __awaiter(this, void 0, void 0, function* () {
-                    var _h;
-                    if ((_h = this.options.debugChannel) === null || _h === void 0 ? void 0 : _h.setActionType) {
+                    var _j;
+                    if ((_j = this.options.debugChannel) === null || _j === void 0 ? void 0 : _j.setActionType) {
                         this.options.debugChannel.setActionType('scroll');
                     }
                     yield page.evaluate((pagesInternal) => __awaiter(this, void 0, void 0, function* () {
@@ -427,8 +430,8 @@ class Interpreter extends events_1.EventEmitter {
                     }), pages !== null && pages !== void 0 ? pages : 1);
                 }),
                 script: (code) => __awaiter(this, void 0, void 0, function* () {
-                    var _j;
-                    if ((_j = this.options.debugChannel) === null || _j === void 0 ? void 0 : _j.setActionType) {
+                    var _k;
+                    if ((_k = this.options.debugChannel) === null || _k === void 0 ? void 0 : _k.setActionType) {
                         this.options.debugChannel.setActionType('script');
                     }
                     const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
@@ -530,6 +533,7 @@ class Interpreter extends events_1.EventEmitter {
                 });
                 allResults = allResults.concat(newResults);
                 debugLog("Results collected:", allResults.length);
+                yield this.options.serializableCallback(allResults);
             });
             const checkLimit = () => {
                 if (config.limit && allResults.length >= config.limit) {
@@ -674,10 +678,47 @@ class Interpreter extends events_1.EventEmitter {
                             }
                             let retryCount = 0;
                             let paginationSuccess = false;
-                            // Capture basic content signature before click
+                            // Capture basic content signature before click - with XPath support
                             const captureContentSignature = () => __awaiter(this, void 0, void 0, function* () {
-                                return yield page.evaluate((selector) => {
-                                    const items = document.querySelectorAll(selector);
+                                return yield page.evaluate((listSelector) => {
+                                    const isXPath = (selector) => {
+                                        return selector.startsWith('//') || selector.startsWith('./') || selector.includes('::');
+                                    };
+                                    let items = [];
+                                    if (isXPath(listSelector)) {
+                                        try {
+                                            // Use XPath to find elements
+                                            const xpathResult = document.evaluate(listSelector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                                            items = [];
+                                            for (let i = 0; i < xpathResult.snapshotLength; i++) {
+                                                const node = xpathResult.snapshotItem(i);
+                                                if (node && node.nodeType === Node.ELEMENT_NODE) {
+                                                    items.push(node);
+                                                }
+                                            }
+                                        }
+                                        catch (xpathError) {
+                                            console.warn('XPath evaluation failed, trying CSS selector as fallback:', xpathError);
+                                            // Fallback to CSS selector
+                                            try {
+                                                items = document.querySelectorAll(listSelector);
+                                            }
+                                            catch (cssError) {
+                                                console.warn('CSS selector fallback also failed:', cssError);
+                                                items = [];
+                                            }
+                                        }
+                                    }
+                                    else {
+                                        try {
+                                            // Use CSS selector
+                                            items = document.querySelectorAll(listSelector);
+                                        }
+                                        catch (cssError) {
+                                            console.warn('CSS selector failed:', cssError);
+                                            items = [];
+                                        }
+                                    }
                                     return {
                                         url: window.location.href,
                                         itemCount: items.length,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.17",
+  "version": "0.0.19",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",