maxun-core 0.0.7 → 0.0.9

package/build/browserSide/scraper.js

@@ -159,6 +159,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
      * @returns {Array.<Object.<string, string>>}
      */
     window.scrapeSchema = function (lists) {
+        // Utility functions remain the same
         function omap(object, f, kf = (x) => x) {
             return Object.fromEntries(Object.entries(object)
                 .map(([k, v]) => [kf(k), f(v)]));
@@ -167,15 +168,121 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             return Object.fromEntries(Object.entries(object)
                 .filter(([k, v]) => f(k, v)));
         }
+        function findAllElements(config) {
+            var _a;
+            // Regular DOM query if no special delimiters
+            if (!config.selector.includes('>>') && !config.selector.includes(':>>')) {
+                return Array.from(document.querySelectorAll(config.selector));
+            }
+            // First handle iframe traversal if present
+            if (config.selector.includes(':>>')) {
+                const parts = config.selector.split(':>>').map(s => s.trim());
+                let currentElements = [document];
+                // Traverse through each part of the selector
+                for (let i = 0; i < parts.length; i++) {
+                    const part = parts[i];
+                    const nextElements = [];
+                    const isLast = i === parts.length - 1;
+                    for (const element of currentElements) {
+                        try {
+                            // For document or iframe document
+                            const doc = element.contentDocument || element || ((_a = element.contentWindow) === null || _a === void 0 ? void 0 : _a.document);
+                            if (!doc)
+                                continue;
+                            // Query elements in current context
+                            const found = Array.from(doc.querySelectorAll(part));
+                            if (isLast) {
+                                // If it's the last part, keep all matching elements
+                                nextElements.push(...found);
+                            }
+                            else {
+                                // If not last, only keep iframes for next iteration
+                                const iframes = found.filter(el => el.tagName === 'IFRAME');
+                                nextElements.push(...iframes);
+                            }
+                        }
+                        catch (error) {
+                            console.warn('Cannot access iframe content:', error, {
+                                part,
+                                element,
+                                index: i
+                            });
+                        }
+                    }
+                    if (nextElements.length === 0) {
+                        console.warn('No elements found for part:', part, 'at depth:', i);
+                        return [];
+                    }
+                    currentElements = nextElements;
+                }
+                return currentElements;
+            }
+            // Handle shadow DOM traversal
+            if (config.selector.includes('>>')) {
+                const parts = config.selector.split('>>').map(s => s.trim());
+                let currentElements = [document];
+                for (const part of parts) {
+                    const nextElements = [];
+                    for (const element of currentElements) {
+                        // Try regular DOM first
+                        const found = Array.from(element.querySelectorAll(part));
+                        // Then check shadow roots
+                        for (const foundEl of found) {
+                            if (foundEl.shadowRoot) {
+                                nextElements.push(foundEl.shadowRoot);
+                            }
+                            else {
+                                nextElements.push(foundEl);
+                            }
+                        }
+                    }
+                    currentElements = nextElements;
+                }
+                return currentElements.filter(el => !(el instanceof ShadowRoot));
+            }
+            return [];
+        }
+        // Modified to handle iframe context for URL resolution
+        function getElementValue(element, attribute) {
+            var _a, _b, _c, _d, _e;
+            if (!element)
+                return null;
+            // Get the base URL for resolving relative URLs
+            const baseURL = ((_b = (_a = element.ownerDocument) === null || _a === void 0 ? void 0 : _a.location) === null || _b === void 0 ? void 0 : _b.href) || window.location.origin;
+            switch (attribute) {
+                case 'href': {
+                    const relativeHref = element.getAttribute('href');
+                    return relativeHref ? new URL(relativeHref, baseURL).href : null;
+                }
+                case 'src': {
+                    const relativeSrc = element.getAttribute('src');
+                    return relativeSrc ? new URL(relativeSrc, baseURL).href : null;
+                }
+                case 'innerText':
+                    return (_c = element.innerText) === null || _c === void 0 ? void 0 : _c.trim();
+                case 'textContent':
+                    return (_d = element.textContent) === null || _d === void 0 ? void 0 : _d.trim();
+                default:
+                    return element.getAttribute(attribute) || ((_e = element.innerText) === null || _e === void 0 ? void 0 : _e.trim());
+            }
+        }
+        // Rest of the functions remain largely the same
         function getSeedKey(listObj) {
-            const maxLength = Math.max(...Object.values(omap(listObj, (x) => document.querySelectorAll(x.selector).length)));
-            return Object.keys(ofilter(listObj, (_, v) => document.querySelectorAll(v.selector).length === maxLength))[0];
+            const maxLength = Math.max(...Object.values(omap(listObj, (x) => findAllElements(x).length)));
+            return Object.keys(ofilter(listObj, (_, v) => findAllElements(v).length === maxLength))[0];
         }
+        // Find minimal bounding elements
         function getMBEs(elements) {
             return elements.map((element) => {
                 let candidate = element;
                 const isUniqueChild = (e) => elements
-                    .filter((elem) => { var _a; return (_a = e.parentNode) === null || _a === void 0 ? void 0 : _a.contains(elem); })
+                    .filter((elem) => {
+                    var _a;
+                    // Handle both iframe and shadow DOM boundaries
+                    const sameContext = elem.getRootNode() === e.getRootNode() &&
+                        elem.ownerDocument === e.ownerDocument;
+                    return sameContext && ((_a = e.parentNode) === null || _a === void 0 ? void 0 : _a.contains(elem));
+                })
                     .length === 1;
                 while (candidate && isUniqueChild(candidate)) {
                     candidate = candidate.parentNode;
@@ -184,28 +291,35 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             });
         }
         const seedName = getSeedKey(lists);
-        const seedElements = Array.from(document.querySelectorAll(lists[seedName].selector));
+        const seedElements = findAllElements(lists[seedName]);
         const MBEs = getMBEs(seedElements);
-        return MBEs.map((mbe) => omap(lists, ({ selector, attribute }, key) => {
-            const elem = Array.from(document.querySelectorAll(selector)).find((elem) => mbe.contains(elem));
-            if (!elem)
-                return undefined;
-            switch (attribute) {
-                case 'href':
-                    const relativeHref = elem.getAttribute('href');
-                    return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
-                case 'src':
-                    const relativeSrc = elem.getAttribute('src');
-                    return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
-                case 'innerText':
-                    return elem.innerText;
-                case 'textContent':
-                    return elem.textContent;
-                default:
-                    return elem.innerText;
-            }
-        }, (key) => key // Use the original key in the output
-        )) || [];
+        const mbeResults = MBEs.map((mbe) => omap(lists, (config) => {
+            const elem = findAllElements(config)
+                .find((elem) => mbe.contains(elem));
+            return elem ? getElementValue(elem, config.attribute) : undefined;
+        }, (key) => key)) || [];
+        // If MBE approach didn't find all elements, try independent scraping
+        if (mbeResults.some(result => Object.values(result).some(v => v === undefined))) {
+            // Fall back to independent scraping
+            const results = [];
+            const foundElements = new Map();
+            // Find all elements for each selector
+            Object.entries(lists).forEach(([key, config]) => {
+                const elements = findAllElements(config);
+                foundElements.set(key, elements);
+            });
+            // Create result objects for each found element
+            foundElements.forEach((elements, key) => {
+                elements.forEach((element, index) => {
+                    if (!results[index]) {
+                        results[index] = {};
+                    }
+                    results[index][key] = getElementValue(element, lists[key].attribute);
+                });
+            });
+            return results.filter(result => Object.keys(result).length > 0);
+        }
+        return mbeResults;
     };
     /**
    * Scrapes multiple lists of similar items based on a template item.
@@ -218,67 +332,410 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
    */
     window.scrapeList = function ({ listSelector, fields, limit = 10 }) {
         return __awaiter(this, void 0, void 0, function* () {
-            const scrapedData = [];
-            while (scrapedData.length < limit) {
-                let parentElements = Array.from(document.querySelectorAll(listSelector));
-                // If we only got one element or none, try a more generic approach
-                if (limit > 1 && parentElements.length <= 1) {
-                    const [containerSelector, _] = listSelector.split('>').map(s => s.trim());
-                    const container = document.querySelector(containerSelector);
-                    if (container) {
-                        const allChildren = Array.from(container.children);
-                        const firstMatch = document.querySelector(listSelector);
-                        if (firstMatch) {
-                            // Get classes from the first matching element
-                            const firstMatchClasses = Array.from(firstMatch.classList);
-                            // Find similar elements by matching most of their classes
-                            parentElements = allChildren.filter(element => {
-                                const elementClasses = Array.from(element.classList);
-                                // Element should share at least 70% of classes with the first match
-                                const commonClasses = firstMatchClasses.filter(cls => elementClasses.includes(cls));
-                                return commonClasses.length >= Math.floor(firstMatchClasses.length * 0.7);
-                            });
+            // Enhanced query function to handle both iframe and shadow DOM
+            const queryElement = (rootElement, selector) => {
+                if (!selector.includes('>>') && !selector.includes(':>>')) {
+                    return rootElement.querySelector(selector);
+                }
+                const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
+                let currentElement = rootElement;
+                for (let i = 0; i < parts.length; i++) {
+                    if (!currentElement)
+                        return null;
+                    // Handle iframe traversal
+                    if (currentElement.tagName === 'IFRAME') {
+                        try {
+                            const iframeDoc = currentElement.contentDocument || currentElement.contentWindow.document;
+                            currentElement = iframeDoc.querySelector(parts[i]);
+                            continue;
+                        }
+                        catch (e) {
+                            console.warn('Cannot access iframe content:', e);
+                            return null;
+                        }
+                    }
+                    // Try regular DOM first
+                    let nextElement = currentElement.querySelector(parts[i]);
+                    // Try shadow DOM if not found
+                    if (!nextElement && currentElement.shadowRoot) {
+                        nextElement = currentElement.shadowRoot.querySelector(parts[i]);
+                    }
+                    // Check children's shadow roots if still not found
+                    if (!nextElement) {
+                        const children = Array.from(currentElement.children || []);
+                        for (const child of children) {
+                            if (child.shadowRoot) {
+                                nextElement = child.shadowRoot.querySelector(parts[i]);
+                                if (nextElement)
+                                    break;
+                            }
                         }
                     }
+                    currentElement = nextElement;
                 }
-                // Iterate through each parent element
-                for (const parent of parentElements) {
-                    if (scrapedData.length >= limit)
-                        break;
-                    const record = {};
-                    // For each field, select the corresponding element within the parent
-                    for (const [label, { selector, attribute }] of Object.entries(fields)) {
-                        const fieldElement = parent.querySelector(selector);
-                        if (fieldElement) {
-                            if (attribute === 'innerText') {
-                                record[label] = fieldElement.innerText.trim();
+                return currentElement;
+            };
+            // Enhanced query all function for both contexts
+            const queryElementAll = (rootElement, selector) => {
+                if (!selector.includes('>>') && !selector.includes(':>>')) {
+                    return rootElement.querySelectorAll(selector);
+                }
+                const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
+                let currentElements = [rootElement];
+                for (const part of parts) {
+                    const nextElements = [];
+                    for (const element of currentElements) {
+                        // Handle iframe traversal
+                        if (element.tagName === 'IFRAME') {
+                            try {
+                                const iframeDoc = element.contentDocument || element.contentWindow.document;
+                                nextElements.push(...iframeDoc.querySelectorAll(part));
+                            }
+                            catch (e) {
+                                console.warn('Cannot access iframe content:', e);
+                                continue;
+                            }
+                        }
+                        else {
+                            // Regular DOM elements
+                            if (element.querySelectorAll) {
+                                nextElements.push(...element.querySelectorAll(part));
                             }
-                            else if (attribute === 'innerHTML') {
-                                record[label] = fieldElement.innerHTML.trim();
+                            // Shadow DOM elements
+                            if (element.shadowRoot) {
+                                nextElements.push(...element.shadowRoot.querySelectorAll(part));
                             }
-                            else if (attribute === 'src') {
-                                // Handle relative 'src' URLs
-                                const src = fieldElement.getAttribute('src');
-                                record[label] = src ? new URL(src, window.location.origin).href : null;
+                            // Check children's shadow roots
+                            const children = Array.from(element.children || []);
+                            for (const child of children) {
+                                if (child.shadowRoot) {
+                                    nextElements.push(...child.shadowRoot.querySelectorAll(part));
+                                }
+                            }
+                        }
+                    }
+                    currentElements = nextElements;
+                }
+                return currentElements;
+            };
+            // Enhanced value extraction with context awareness
+            function extractValue(element, attribute) {
+                var _a, _b;
+                if (!element)
+                    return null;
+                // Get context-aware base URL
+                const baseURL = ((_b = (_a = element.ownerDocument) === null || _a === void 0 ? void 0 : _a.location) === null || _b === void 0 ? void 0 : _b.href) || window.location.origin;
+                // Check shadow root first
+                if (element.shadowRoot) {
+                    const shadowContent = element.shadowRoot.textContent;
+                    if (shadowContent === null || shadowContent === void 0 ? void 0 : shadowContent.trim()) {
+                        return shadowContent.trim();
+                    }
+                }
+                if (attribute === 'innerText') {
+                    return element.innerText.trim();
+                }
+                else if (attribute === 'innerHTML') {
+                    return element.innerHTML.trim();
+                }
+                else if (attribute === 'src' || attribute === 'href') {
+                    const attrValue = element.getAttribute(attribute);
+                    return attrValue ? new URL(attrValue, baseURL).href : null;
+                }
+                return element.getAttribute(attribute);
+            }
+            // Enhanced table ancestor finding with context support
+            function findTableAncestor(element) {
+                let currentElement = element;
+                const MAX_DEPTH = 5;
+                let depth = 0;
+                while (currentElement && depth < MAX_DEPTH) {
+                    // Handle shadow DOM
+                    if (currentElement.getRootNode() instanceof ShadowRoot) {
+                        currentElement = currentElement.getRootNode().host;
+                        continue;
+                    }
+                    if (currentElement.tagName === 'TD') {
+                        return { type: 'TD', element: currentElement };
+                    }
+                    else if (currentElement.tagName === 'TR') {
+                        return { type: 'TR', element: currentElement };
+                    }
+                    // Handle iframe crossing
+                    if (currentElement.tagName === 'IFRAME') {
+                        try {
+                            currentElement = currentElement.contentDocument.body;
+                        }
+                        catch (e) {
+                            return null;
+                        }
+                    }
+                    else {
+                        currentElement = currentElement.parentElement;
+                    }
+                    depth++;
+                }
+                return null;
+            }
+            // Helper function to get cell index
+            function getCellIndex(td) {
+                if (td.getRootNode() instanceof ShadowRoot) {
+                    const shadowRoot = td.getRootNode();
+                    const allCells = Array.from(shadowRoot.querySelectorAll('td'));
+                    return allCells.indexOf(td);
+                }
+                let index = 0;
+                let sibling = td;
+                while (sibling = sibling.previousElementSibling) {
+                    index++;
+                }
+                return index;
+            }
+            // Helper function to check for TH elements
+            function hasThElement(row, tableFields) {
+                for (const [_, { selector }] of Object.entries(tableFields)) {
+                    const element = queryElement(row, selector);
+                    if (element) {
+                        let current = element;
+                        while (current && current !== row) {
+                            if (current.getRootNode() instanceof ShadowRoot) {
+                                current = current.getRootNode().host;
+                                continue;
                             }
-                            else if (attribute === 'href') {
-                                // Handle relative 'href' URLs
-                                const href = fieldElement.getAttribute('href');
-                                record[label] = href ? new URL(href, window.location.origin).href : null;
+                            if (current.tagName === 'TH')
+                                return true;
+                            if (current.tagName === 'IFRAME') {
+                                try {
+                                    current = current.contentDocument.body;
+                                }
+                                catch (e) {
+                                    break;
+                                }
                             }
                             else {
-                                record[label] = fieldElement.getAttribute(attribute);
+                                current = current.parentElement;
                             }
                         }
                     }
-                    scrapedData.push(record);
                 }
-                // If we've processed all available elements and still haven't reached the limit,
-                // break to avoid infinite loop
-                if (parentElements.length === 0 || scrapedData.length >= parentElements.length) {
+                return false;
+            }
+            // Helper function to filter rows
+            function filterRowsBasedOnTag(rows, tableFields) {
+                for (const row of rows) {
+                    if (hasThElement(row, tableFields)) {
+                        return rows;
+                    }
+                }
+                // Include shadow DOM in TH search
+                return rows.filter(row => {
+                    const directTH = row.getElementsByTagName('TH').length === 0;
+                    const shadowTH = row.shadowRoot ?
+                        row.shadowRoot.querySelector('th') === null : true;
+                    return directTH && shadowTH;
+                });
+            }
+            // Class similarity comparison functions
+            function calculateClassSimilarity(classList1, classList2) {
+                const set1 = new Set(classList1);
+                const set2 = new Set(classList2);
+                const intersection = new Set([...set1].filter(x => set2.has(x)));
+                const union = new Set([...set1, ...set2]);
+                return intersection.size / union.size;
+            }
+            // Enhanced similar elements finding with context support
+            function findSimilarElements(baseElement, similarityThreshold = 0.7) {
+                const baseClasses = Array.from(baseElement.classList);
+                if (baseClasses.length === 0)
+                    return [];
+                const allElements = [];
+                // Get elements from main document
+                allElements.push(...document.getElementsByTagName(baseElement.tagName));
+                // Get elements from shadow DOM
+                if (baseElement.getRootNode() instanceof ShadowRoot) {
+                    const shadowHost = baseElement.getRootNode().host;
+                    allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName));
+                }
+                // Get elements from iframes
+                const iframes = document.getElementsByTagName('iframe');
+                for (const iframe of iframes) {
+                    try {
+                        const iframeDoc = iframe.contentDocument || iframe.contentWindow.document;
+                        allElements.push(...iframeDoc.getElementsByTagName(baseElement.tagName));
+                    }
+                    catch (e) {
+                        console.warn('Cannot access iframe content:', e);
+                    }
+                }
+                return allElements.filter(element => {
+                    if (element === baseElement)
+                        return false;
+                    const similarity = calculateClassSimilarity(baseClasses, Array.from(element.classList));
+                    return similarity >= similarityThreshold;
+                });
+            }
+            // Main scraping logic with context support
+            let containers = queryElementAll(document, listSelector);
+            containers = Array.from(containers);
+            if (containers.length === 0)
+                return [];
+            if (limit > 1 && containers.length === 1) {
+                const baseContainer = containers[0];
+                const similarContainers = findSimilarElements(baseContainer);
+                if (similarContainers.length > 0) {
+                    const newContainers = similarContainers.filter(container => !container.matches(listSelector));
+                    containers = [...containers, ...newContainers];
+                }
+            }
+            const containerFields = containers.map(() => ({
+                tableFields: {},
+                nonTableFields: {}
+            }));
+            // Classify fields
+            containers.forEach((container, containerIndex) => {
+                for (const [label, field] of Object.entries(fields)) {
+                    const sampleElement = queryElement(container, field.selector);
+                    if (sampleElement) {
+                        const ancestor = findTableAncestor(sampleElement);
+                        if (ancestor) {
+                            containerFields[containerIndex].tableFields[label] = Object.assign(Object.assign({}, field), { tableContext: ancestor.type, cellIndex: ancestor.type === 'TD' ? getCellIndex(ancestor.element) : -1 });
+                        }
+                        else {
+                            containerFields[containerIndex].nonTableFields[label] = field;
+                        }
+                    }
+                    else {
+                        containerFields[containerIndex].nonTableFields[label] = field;
+                    }
+                }
+            });
+            const tableData = [];
+            const nonTableData = [];
+            // Process table data with both iframe and shadow DOM support
+            for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
+                const container = containers[containerIndex];
+                const { tableFields } = containerFields[containerIndex];
+                if (Object.keys(tableFields).length > 0) {
+                    const firstField = Object.values(tableFields)[0];
+                    const firstElement = queryElement(container, firstField.selector);
+                    let tableContext = firstElement;
+                    // Find table context including both iframe and shadow DOM
+                    while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) {
+                        if (tableContext.getRootNode() instanceof ShadowRoot) {
+                            tableContext = tableContext.getRootNode().host;
+                            continue;
+                        }
+                        if (tableContext.tagName === 'IFRAME') {
+                            try {
+                                tableContext = tableContext.contentDocument.body;
+                            }
+                            catch (e) {
+                                break;
+                            }
+                        }
+                        else {
+                            tableContext = tableContext.parentElement;
+                        }
+                    }
+                    if (tableContext) {
+                        // Get rows from all contexts
+                        const rows = [];
+                        // Get rows from regular DOM
+                        rows.push(...tableContext.getElementsByTagName('TR'));
+                        // Get rows from shadow DOM
+                        if (tableContext.shadowRoot) {
+                            rows.push(...tableContext.shadowRoot.getElementsByTagName('TR'));
+                        }
+                        // Get rows from iframes
+                        if (tableContext.tagName === 'IFRAME') {
+                            try {
+                                const iframeDoc = tableContext.contentDocument || tableContext.contentWindow.document;
+                                rows.push(...iframeDoc.getElementsByTagName('TR'));
+                            }
+                            catch (e) {
+                                console.warn('Cannot access iframe rows:', e);
+                            }
+                        }
+                        const processedRows = filterRowsBasedOnTag(rows, tableFields);
+                        for (let rowIndex = 0; rowIndex < Math.min(processedRows.length, limit); rowIndex++) {
+                            const record = {};
+                            const currentRow = processedRows[rowIndex];
+                            for (const [label, { selector, attribute, cellIndex }] of Object.entries(tableFields)) {
+                                let element = null;
+                                if (cellIndex >= 0) {
+                                    // Get TD element considering both contexts
+                                    let td = currentRow.children[cellIndex];
+                                    // Check shadow DOM for td
+                                    if (!td && currentRow.shadowRoot) {
+                                        const shadowCells = currentRow.shadowRoot.children;
+                                        if (shadowCells && shadowCells.length > cellIndex) {
+                                            td = shadowCells[cellIndex];
+                                        }
+                                    }
+                                    if (td) {
+                                        element = queryElement(td, selector);
+                                        if (!element && selector.split(/(?:>>|:>>)/).pop().includes('td:nth-child')) {
+                                            element = td;
+                                        }
+                                        if (!element) {
+                                            const tagOnlySelector = selector.split('.')[0];
+                                            element = queryElement(td, tagOnlySelector);
+                                        }
+                                        if (!element) {
+                                            let currentElement = td;
+                                            while (currentElement && currentElement.children.length > 0) {
+                                                let foundContentChild = false;
+                                                for (const child of currentElement.children) {
+                                                    if (extractValue(child, attribute)) {
+                                                        currentElement = child;
+                                                        foundContentChild = true;
+                                                        break;
+                                                    }
+                                                }
+                                                if (!foundContentChild)
+                                                    break;
+                                            }
+                                            element = currentElement;
+                                        }
+                                    }
+                                }
+                                else {
+                                    element = queryElement(currentRow, selector);
+                                }
+                                if (element) {
+                                    record[label] = extractValue(element, attribute);
+                                }
+                            }
+                            if (Object.keys(record).length > 0) {
+                                tableData.push(record);
+                            }
+                        }
+                    }
+                }
+            }
+            // Process non-table data with both contexts support
+            for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
+                if (nonTableData.length >= limit)
                     break;
+                const container = containers[containerIndex];
+                const { nonTableFields } = containerFields[containerIndex];
+                if (Object.keys(nonTableFields).length > 0) {
+                    const record = {};
+                    for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
+                        // Get the last part of the selector after any context delimiter
+                        const relativeSelector = selector.split(/(?:>>|:>>)/).slice(-1)[0];
+                        const element = queryElement(container, relativeSelector);
+                        if (element) {
+                            record[label] = extractValue(element, attribute);
+                        }
+                    }
+                    if (Object.keys(record).length > 0) {
+                        nonTableData.push(record);
+                    }
                 }
             }
+            // Merge and limit the results
+            const scrapedData = [...tableData, ...nonTableData];
             return scrapedData;
         });
     };

package/build/interpret.d.ts

@@ -86,6 +86,8 @@ export default class Interpreter extends EventEmitter {
     private carryOutSteps;
     private handlePagination;
     private getMatchingActionId;
+    private removeShadowSelectors;
+    private removeSpecialSelectors;
     private runLoop;
     private ensureScriptsLoaded;
     /**

package/build/interpret.js

@@ -461,6 +461,8 @@ class Interpreter extends events_1.EventEmitter {
             let previousHeight = 0;
             // track unique items per page to avoid re-scraping
             let scrapedItems = new Set();
+            let visitedUrls = [];
+            let availableSelectors = config.pagination.selector.split(',');
             while (true) {
                 switch (config.pagination.type) {
                     case 'scrollDown':
@@ -486,6 +488,7 @@ class Interpreter extends events_1.EventEmitter {
                         previousHeight = currentTopHeight;
                         break;
                     case 'clickNext':
+                        console.log("Page URL:", page.url());
                         const pageResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
                         // console.log("Page results:", pageResults);
                         // Filter out already scraped items
@@ -497,30 +500,128 @@ class Interpreter extends events_1.EventEmitter {
                             return true;
                         });
                         allResults = allResults.concat(newResults);
+                        console.log("Results so far:", allResults.length);
                         if (config.limit && allResults.length >= config.limit) {
                             return allResults.slice(0, config.limit);
                         }
-                        const nextButton = yield page.$(config.pagination.selector);
+                        let checkButton = null;
+                        let workingSelector = null;
+                        for (let i = 0; i < availableSelectors.length; i++) {
+                            const selector = availableSelectors[i];
+                            try {
+                                // Wait for selector with a short timeout
+                                checkButton = yield page.waitForSelector(selector, { state: 'attached' });
+                                if (checkButton) {
+                                    workingSelector = selector;
+                                    break;
+                                }
+                            }
+                            catch (error) {
+                                console.log(`Selector failed: ${selector}`);
+                            }
+                        }
+                        if (!workingSelector) {
+                            return allResults;
+                        }
+                        // const nextButton = await page.$(config.pagination.selector);
+                        const nextButton = yield page.$(workingSelector);
                         if (!nextButton) {
                             return allResults; // No more pages to scrape
                         }
-                        yield Promise.all([
-                            nextButton.dispatchEvent('click'),
-                            page.waitForNavigation({ waitUntil: 'networkidle' })
-                        ]);
+                        const selectorIndex = availableSelectors.indexOf(workingSelector);
+                        availableSelectors = availableSelectors.slice(selectorIndex);
+                        // await Promise.all([
+                        //   nextButton.dispatchEvent('click'),
+                        //   page.waitForNavigation({ waitUntil: 'networkidle' })
+                        // ]);
+                        const previousUrl = page.url();
+                        visitedUrls.push(previousUrl);
+                        try {
+                            // Try both click methods simultaneously
+                            yield Promise.race([
+                                Promise.all([
+                                    page.waitForNavigation({ waitUntil: 'networkidle', timeout: 30000 }),
+                                    nextButton.click()
+                                ]),
+                                Promise.all([
+                                    page.waitForNavigation({ waitUntil: 'networkidle', timeout: 30000 }),
+                                    nextButton.dispatchEvent('click')
+                                ])
+                            ]);
+                        }
+                        catch (error) {
+                            // Verify if navigation actually succeeded
+                            const currentUrl = page.url();
+                            if (currentUrl === previousUrl) {
+                                console.log("Previous URL same as current URL. Navigation failed.");
+                            }
+                        }
+                        const currentUrl = page.url();
+                        if (visitedUrls.includes(currentUrl)) {
+                            console.log(`Detected navigation to a previously visited URL: ${currentUrl}`);
+                            // Extract the current page number from the URL
+                            const match = currentUrl.match(/\d+/);
+                            if (match) {
+                                const currentNumber = match[0];
+                                // Use visitedUrls.length + 1 as the next page number
+                                const nextNumber = visitedUrls.length + 1;
+                                // Create new URL by replacing the current number with the next number
+                                const nextUrl = currentUrl.replace(currentNumber, nextNumber.toString());
+                                console.log(`Navigating to constructed URL: ${nextUrl}`);
+                                // Navigate to the next page
+                                yield Promise.all([
+                                    page.waitForNavigation({ waitUntil: 'networkidle' }),
+                                    page.goto(nextUrl)
+                                ]);
+                            }
+                        }
+                        // Give the page a moment to stabilize after navigation
                         yield page.waitForTimeout(1000);
                         break;
                     case 'clickLoadMore':
                         while (true) {
-                            const loadMoreButton = yield page.$(config.pagination.selector);
+                            let checkButton = null;
+                            let workingSelector = null;
+                            for (let i = 0; i < availableSelectors.length; i++) {
+                                const selector = availableSelectors[i];
+                                try {
+                                    // Wait for selector with a short timeout
+                                    checkButton = yield page.waitForSelector(selector, { state: 'attached' });
+                                    if (checkButton) {
+                                        workingSelector = selector;
+                                        break;
+                                    }
+                                }
+                                catch (error) {
+                                    console.log(`Selector failed: ${selector}`);
+                                }
+                            }
+                            if (!workingSelector) {
+                                // No more working selectors available, so scrape the remaining items
+                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                                allResults = allResults.concat(finalResults);
+                                return allResults;
+                            }
+                            const loadMoreButton = yield page.$(workingSelector);
                             if (!loadMoreButton) {
                                 // No more "Load More" button, so scrape the remaining items
                                 const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
                                 allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
+                            const selectorIndex = availableSelectors.indexOf(workingSelector);
+                            availableSelectors = availableSelectors.slice(selectorIndex);
                             // Click the 'Load More' button to load additional items
-                            yield loadMoreButton.dispatchEvent('click');
+                            // await loadMoreButton.dispatchEvent('click');
+                            try {
+                                yield Promise.race([
+                                    loadMoreButton.click(),
+                                    loadMoreButton.dispatchEvent('click')
+                                ]);
+                            }
+                            catch (error) {
+                                console.log('Both click attempts failed');
+                            }
                             yield page.waitForTimeout(2000); // Wait for new items to load
                             // After clicking 'Load More', scroll down to load more items
                             yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
@@ -568,10 +669,32 @@ class Interpreter extends events_1.EventEmitter {
             }
         }
     }
+    removeShadowSelectors(workflow) {
+        for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
+            const step = workflow[actionId];
+            // Check if step has where and selectors
+            if (step.where && Array.isArray(step.where.selectors)) {
+                // Filter out selectors that contain ">>"
+                step.where.selectors = step.where.selectors.filter(selector => !selector.includes('>>'));
+            }
+        }
+        return workflow;
+    }
+    removeSpecialSelectors(workflow) {
+        for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
+            const step = workflow[actionId];
+            if (step.where && Array.isArray(step.where.selectors)) {
+                // Filter out if selector has EITHER ":>>" OR ">>"
+                step.where.selectors = step.where.selectors.filter(selector => !(selector.includes(':>>') || selector.includes('>>')));
+            }
+        }
+        return workflow;
+    }
     runLoop(p, workflow) {
         var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {
-            const workflowCopy = JSON.parse(JSON.stringify(workflow));
+            let workflowCopy = JSON.parse(JSON.stringify(workflow));
+            workflowCopy = this.removeSpecialSelectors(workflowCopy);
             // apply ad-blocker to the current page
             try {
                 yield this.applyAdBlocker(p);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",