maxun-core 0.0.6 → 0.0.8

package/build/browserSide/scraper.js

@@ -159,6 +159,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
      * @returns {Array.<Object.<string, string>>}
      */
     window.scrapeSchema = function (lists) {
+        // Utility functions remain the same
         function omap(object, f, kf = (x) => x) {
             return Object.fromEntries(Object.entries(object)
                 .map(([k, v]) => [kf(k), f(v)]));
@@ -167,15 +168,121 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             return Object.fromEntries(Object.entries(object)
                 .filter(([k, v]) => f(k, v)));
         }
+        function findAllElements(config) {
+            var _a;
+            // Regular DOM query if no special delimiters
+            if (!config.selector.includes('>>') && !config.selector.includes(':>>')) {
+                return Array.from(document.querySelectorAll(config.selector));
+            }
+            // First handle iframe traversal if present
+            if (config.selector.includes(':>>')) {
+                const parts = config.selector.split(':>>').map(s => s.trim());
+                let currentElements = [document];
+                // Traverse through each part of the selector
+                for (let i = 0; i < parts.length; i++) {
+                    const part = parts[i];
+                    const nextElements = [];
+                    const isLast = i === parts.length - 1;
+                    for (const element of currentElements) {
+                        try {
+                            // For document or iframe document
+                            const doc = element.contentDocument || element || ((_a = element.contentWindow) === null || _a === void 0 ? void 0 : _a.document);
+                            if (!doc)
+                                continue;
+                            // Query elements in current context
+                            const found = Array.from(doc.querySelectorAll(part));
+                            if (isLast) {
+                                // If it's the last part, keep all matching elements
+                                nextElements.push(...found);
+                            }
+                            else {
+                                // If not last, only keep iframes for next iteration
+                                const iframes = found.filter(el => el.tagName === 'IFRAME');
+                                nextElements.push(...iframes);
+                            }
+                        }
+                        catch (error) {
+                            console.warn('Cannot access iframe content:', error, {
+                                part,
+                                element,
+                                index: i
+                            });
+                        }
+                    }
+                    if (nextElements.length === 0) {
+                        console.warn('No elements found for part:', part, 'at depth:', i);
+                        return [];
+                    }
+                    currentElements = nextElements;
+                }
+                return currentElements;
+            }
+            // Handle shadow DOM traversal
+            if (config.selector.includes('>>')) {
+                const parts = config.selector.split('>>').map(s => s.trim());
+                let currentElements = [document];
+                for (const part of parts) {
+                    const nextElements = [];
+                    for (const element of currentElements) {
+                        // Try regular DOM first
+                        const found = Array.from(element.querySelectorAll(part));
+                        // Then check shadow roots
+                        for (const foundEl of found) {
+                            if (foundEl.shadowRoot) {
+                                nextElements.push(foundEl.shadowRoot);
+                            }
+                            else {
+                                nextElements.push(foundEl);
+                            }
+                        }
+                    }
+                    currentElements = nextElements;
+                }
+                return currentElements.filter(el => !(el instanceof ShadowRoot));
+            }
+            return [];
+        }
+        // Modified to handle iframe context for URL resolution
+        function getElementValue(element, attribute) {
+            var _a, _b, _c, _d, _e;
+            if (!element)
+                return null;
+            // Get the base URL for resolving relative URLs
+            const baseURL = ((_b = (_a = element.ownerDocument) === null || _a === void 0 ? void 0 : _a.location) === null || _b === void 0 ? void 0 : _b.href) || window.location.origin;
+            switch (attribute) {
+                case 'href': {
+                    const relativeHref = element.getAttribute('href');
+                    return relativeHref ? new URL(relativeHref, baseURL).href : null;
+                }
+                case 'src': {
+                    const relativeSrc = element.getAttribute('src');
+                    return relativeSrc ? new URL(relativeSrc, baseURL).href : null;
+                }
+                case 'innerText':
+                    return (_c = element.innerText) === null || _c === void 0 ? void 0 : _c.trim();
+                case 'textContent':
+                    return (_d = element.textContent) === null || _d === void 0 ? void 0 : _d.trim();
+                default:
+                    return element.getAttribute(attribute) || ((_e = element.innerText) === null || _e === void 0 ? void 0 : _e.trim());
+            }
+        }
+        // Rest of the functions remain largely the same
         function getSeedKey(listObj) {
-            const maxLength = Math.max(...Object.values(omap(listObj, (x) => document.querySelectorAll(x.selector).length)));
-            return Object.keys(ofilter(listObj, (_, v) => document.querySelectorAll(v.selector).length === maxLength))[0];
+            const maxLength = Math.max(...Object.values(omap(listObj, (x) => findAllElements(x).length)));
+            return Object.keys(ofilter(listObj, (_, v) => findAllElements(v).length === maxLength))[0];
         }
+        // Find minimal bounding elements
         function getMBEs(elements) {
             return elements.map((element) => {
                 let candidate = element;
                 const isUniqueChild = (e) => elements
-                    .filter((elem) => { var _a; return (_a = e.parentNode) === null || _a === void 0 ? void 0 : _a.contains(elem); })
+                    .filter((elem) => {
+                    var _a;
+                    // Handle both iframe and shadow DOM boundaries
+                    const sameContext = elem.getRootNode() === e.getRootNode() &&
+                        elem.ownerDocument === e.ownerDocument;
+                    return sameContext && ((_a = e.parentNode) === null || _a === void 0 ? void 0 : _a.contains(elem));
+                })
                     .length === 1;
                 while (candidate && isUniqueChild(candidate)) {
                     candidate = candidate.parentNode;
@@ -184,28 +291,35 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             });
         }
         const seedName = getSeedKey(lists);
-        const seedElements = Array.from(document.querySelectorAll(lists[seedName].selector));
+        const seedElements = findAllElements(lists[seedName]);
         const MBEs = getMBEs(seedElements);
-        return MBEs.map((mbe) => omap(lists, ({ selector, attribute }, key) => {
-            const elem = Array.from(document.querySelectorAll(selector)).find((elem) => mbe.contains(elem));
-            if (!elem)
-                return undefined;
-            switch (attribute) {
-                case 'href':
-                    const relativeHref = elem.getAttribute('href');
-                    return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
-                case 'src':
-                    const relativeSrc = elem.getAttribute('src');
-                    return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
-                case 'innerText':
-                    return elem.innerText;
-                case 'textContent':
-                    return elem.textContent;
-                default:
-                    return elem.innerText;
-            }
-        }, (key) => key // Use the original key in the output
-        )) || [];
+        const mbeResults = MBEs.map((mbe) => omap(lists, (config) => {
+            const elem = findAllElements(config)
+                .find((elem) => mbe.contains(elem));
+            return elem ? getElementValue(elem, config.attribute) : undefined;
+        }, (key) => key)) || [];
+        // If MBE approach didn't find all elements, try independent scraping
+        if (mbeResults.some(result => Object.values(result).some(v => v === undefined))) {
+            // Fall back to independent scraping
+            const results = [];
+            const foundElements = new Map();
+            // Find all elements for each selector
+            Object.entries(lists).forEach(([key, config]) => {
+                const elements = findAllElements(config);
+                foundElements.set(key, elements);
+            });
+            // Create result objects for each found element
+            foundElements.forEach((elements, key) => {
+                elements.forEach((element, index) => {
+                    if (!results[index]) {
+                        results[index] = {};
+                    }
+                    results[index][key] = getElementValue(element, lists[key].attribute);
+                });
+            });
+            return results.filter(result => Object.keys(result).length > 0);
+        }
+        return mbeResults;
     };
     /**
    * Scrapes multiple lists of similar items based on a template item.
@@ -218,43 +332,410 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
    */
     window.scrapeList = function ({ listSelector, fields, limit = 10 }) {
         return __awaiter(this, void 0, void 0, function* () {
-            const scrapedData = [];
-            while (scrapedData.length < limit) {
-                // Get all parent elements matching the listSelector
-                const parentElements = Array.from(document.querySelectorAll(listSelector));
-                // Iterate through each parent element
-                for (const parent of parentElements) {
-                    if (scrapedData.length >= limit)
-                        break;
-                    const record = {};
-                    // For each field, select the corresponding element within the parent
-                    for (const [label, { selector, attribute }] of Object.entries(fields)) {
-                        const fieldElement = parent.querySelector(selector);
-                        if (fieldElement) {
-                            if (attribute === 'innerText') {
-                                record[label] = fieldElement.innerText.trim();
+            // Enhanced query function to handle both iframe and shadow DOM
+            const queryElement = (rootElement, selector) => {
+                if (!selector.includes('>>') && !selector.includes(':>>')) {
+                    return rootElement.querySelector(selector);
+                }
+                const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
+                let currentElement = rootElement;
+                for (let i = 0; i < parts.length; i++) {
+                    if (!currentElement)
+                        return null;
+                    // Handle iframe traversal
+                    if (currentElement.tagName === 'IFRAME') {
+                        try {
+                            const iframeDoc = currentElement.contentDocument || currentElement.contentWindow.document;
+                            currentElement = iframeDoc.querySelector(parts[i]);
+                            continue;
+                        }
+                        catch (e) {
+                            console.warn('Cannot access iframe content:', e);
+                            return null;
+                        }
+                    }
+                    // Try regular DOM first
+                    let nextElement = currentElement.querySelector(parts[i]);
+                    // Try shadow DOM if not found
+                    if (!nextElement && currentElement.shadowRoot) {
+                        nextElement = currentElement.shadowRoot.querySelector(parts[i]);
+                    }
+                    // Check children's shadow roots if still not found
+                    if (!nextElement) {
+                        const children = Array.from(currentElement.children || []);
+                        for (const child of children) {
+                            if (child.shadowRoot) {
+                                nextElement = child.shadowRoot.querySelector(parts[i]);
+                                if (nextElement)
+                                    break;
                             }
-                            else if (attribute === 'innerHTML') {
-                                record[label] = fieldElement.innerHTML.trim();
+                        }
+                    }
+                    currentElement = nextElement;
+                }
+                return currentElement;
+            };
+            // Enhanced query all function for both contexts
+            const queryElementAll = (rootElement, selector) => {
+                if (!selector.includes('>>') && !selector.includes(':>>')) {
+                    return rootElement.querySelectorAll(selector);
+                }
+                const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
+                let currentElements = [rootElement];
+                for (const part of parts) {
+                    const nextElements = [];
+                    for (const element of currentElements) {
+                        // Handle iframe traversal
+                        if (element.tagName === 'IFRAME') {
+                            try {
+                                const iframeDoc = element.contentDocument || element.contentWindow.document;
+                                nextElements.push(...iframeDoc.querySelectorAll(part));
+                            }
+                            catch (e) {
+                                console.warn('Cannot access iframe content:', e);
+                                continue;
+                            }
+                        }
+                        else {
+                            // Regular DOM elements
+                            if (element.querySelectorAll) {
+                                nextElements.push(...element.querySelectorAll(part));
+                            }
+                            // Shadow DOM elements
+                            if (element.shadowRoot) {
+                                nextElements.push(...element.shadowRoot.querySelectorAll(part));
                             }
-                            else if (attribute === 'src') {
-                                // Handle relative 'src' URLs
-                                const src = fieldElement.getAttribute('src');
-                                record[label] = src ? new URL(src, window.location.origin).href : null;
+                            // Check children's shadow roots
+                            const children = Array.from(element.children || []);
+                            for (const child of children) {
+                                if (child.shadowRoot) {
+                                    nextElements.push(...child.shadowRoot.querySelectorAll(part));
+                                }
+                            }
+                        }
+                    }
+                    currentElements = nextElements;
+                }
+                return currentElements;
+            };
+            // Enhanced value extraction with context awareness
+            function extractValue(element, attribute) {
+                var _a, _b;
+                if (!element)
+                    return null;
+                // Get context-aware base URL
+                const baseURL = ((_b = (_a = element.ownerDocument) === null || _a === void 0 ? void 0 : _a.location) === null || _b === void 0 ? void 0 : _b.href) || window.location.origin;
+                // Check shadow root first
+                if (element.shadowRoot) {
+                    const shadowContent = element.shadowRoot.textContent;
+                    if (shadowContent === null || shadowContent === void 0 ? void 0 : shadowContent.trim()) {
+                        return shadowContent.trim();
+                    }
+                }
+                if (attribute === 'innerText') {
+                    return element.innerText.trim();
+                }
+                else if (attribute === 'innerHTML') {
+                    return element.innerHTML.trim();
+                }
+                else if (attribute === 'src' || attribute === 'href') {
+                    const attrValue = element.getAttribute(attribute);
+                    return attrValue ? new URL(attrValue, baseURL).href : null;
+                }
+                return element.getAttribute(attribute);
+            }
+            // Enhanced table ancestor finding with context support
+            function findTableAncestor(element) {
+                let currentElement = element;
+                const MAX_DEPTH = 5;
+                let depth = 0;
+                while (currentElement && depth < MAX_DEPTH) {
+                    // Handle shadow DOM
+                    if (currentElement.getRootNode() instanceof ShadowRoot) {
+                        currentElement = currentElement.getRootNode().host;
+                        continue;
+                    }
+                    if (currentElement.tagName === 'TD') {
+                        return { type: 'TD', element: currentElement };
+                    }
+                    else if (currentElement.tagName === 'TR') {
+                        return { type: 'TR', element: currentElement };
+                    }
+                    // Handle iframe crossing
+                    if (currentElement.tagName === 'IFRAME') {
+                        try {
+                            currentElement = currentElement.contentDocument.body;
+                        }
+                        catch (e) {
+                            return null;
+                        }
+                    }
+                    else {
+                        currentElement = currentElement.parentElement;
+                    }
+                    depth++;
+                }
+                return null;
+            }
+            // Helper function to get cell index
+            function getCellIndex(td) {
+                if (td.getRootNode() instanceof ShadowRoot) {
+                    const shadowRoot = td.getRootNode();
+                    const allCells = Array.from(shadowRoot.querySelectorAll('td'));
+                    return allCells.indexOf(td);
+                }
+                let index = 0;
+                let sibling = td;
+                while (sibling = sibling.previousElementSibling) {
+                    index++;
+                }
+                return index;
+            }
+            // Helper function to check for TH elements
+            function hasThElement(row, tableFields) {
+                for (const [_, { selector }] of Object.entries(tableFields)) {
+                    const element = queryElement(row, selector);
+                    if (element) {
+                        let current = element;
+                        while (current && current !== row) {
+                            if (current.getRootNode() instanceof ShadowRoot) {
+                                current = current.getRootNode().host;
+                                continue;
                             }
-                            else if (attribute === 'href') {
-                                // Handle relative 'href' URLs
-                                const href = fieldElement.getAttribute('href');
-                                record[label] = href ? new URL(href, window.location.origin).href : null;
+                            if (current.tagName === 'TH')
+                                return true;
+                            if (current.tagName === 'IFRAME') {
+                                try {
+                                    current = current.contentDocument.body;
+                                }
+                                catch (e) {
+                                    break;
+                                }
                             }
                             else {
-                                record[label] = fieldElement.getAttribute(attribute);
+                                current = current.parentElement;
+                            }
+                        }
+                    }
+                }
+                return false;
+            }
+            // Helper function to filter rows
+            function filterRowsBasedOnTag(rows, tableFields) {
+                for (const row of rows) {
+                    if (hasThElement(row, tableFields)) {
+                        return rows;
+                    }
+                }
+                // Include shadow DOM in TH search
+                return rows.filter(row => {
+                    const directTH = row.getElementsByTagName('TH').length === 0;
+                    const shadowTH = row.shadowRoot ?
+                        row.shadowRoot.querySelector('th') === null : true;
+                    return directTH && shadowTH;
+                });
+            }
+            // Class similarity comparison functions
+            function calculateClassSimilarity(classList1, classList2) {
+                const set1 = new Set(classList1);
+                const set2 = new Set(classList2);
+                const intersection = new Set([...set1].filter(x => set2.has(x)));
+                const union = new Set([...set1, ...set2]);
+                return intersection.size / union.size;
+            }
+            // Enhanced similar elements finding with context support
+            function findSimilarElements(baseElement, similarityThreshold = 0.7) {
+                const baseClasses = Array.from(baseElement.classList);
+                if (baseClasses.length === 0)
+                    return [];
+                const allElements = [];
+                // Get elements from main document
+                allElements.push(...document.getElementsByTagName(baseElement.tagName));
+                // Get elements from shadow DOM
+                if (baseElement.getRootNode() instanceof ShadowRoot) {
+                    const shadowHost = baseElement.getRootNode().host;
+                    allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName));
+                }
+                // Get elements from iframes
+                const iframes = document.getElementsByTagName('iframe');
+                for (const iframe of iframes) {
+                    try {
+                        const iframeDoc = iframe.contentDocument || iframe.contentWindow.document;
+                        allElements.push(...iframeDoc.getElementsByTagName(baseElement.tagName));
+                    }
+                    catch (e) {
+                        console.warn('Cannot access iframe content:', e);
+                    }
+                }
+                return allElements.filter(element => {
+                    if (element === baseElement)
+                        return false;
+                    const similarity = calculateClassSimilarity(baseClasses, Array.from(element.classList));
+                    return similarity >= similarityThreshold;
+                });
+            }
+            // Main scraping logic with context support
+            let containers = queryElementAll(document, listSelector);
+            containers = Array.from(containers);
+            if (containers.length === 0)
+                return [];
+            if (limit > 1 && containers.length === 1) {
+                const baseContainer = containers[0];
+                const similarContainers = findSimilarElements(baseContainer);
+                if (similarContainers.length > 0) {
+                    const newContainers = similarContainers.filter(container => !container.matches(listSelector));
+                    containers = [...containers, ...newContainers];
+                }
+            }
+            const containerFields = containers.map(() => ({
+                tableFields: {},
+                nonTableFields: {}
+            }));
+            // Classify fields
+            containers.forEach((container, containerIndex) => {
+                for (const [label, field] of Object.entries(fields)) {
+                    const sampleElement = queryElement(container, field.selector);
+                    if (sampleElement) {
+                        const ancestor = findTableAncestor(sampleElement);
+                        if (ancestor) {
+                            containerFields[containerIndex].tableFields[label] = Object.assign(Object.assign({}, field), { tableContext: ancestor.type, cellIndex: ancestor.type === 'TD' ? getCellIndex(ancestor.element) : -1 });
+                        }
+                        else {
+                            containerFields[containerIndex].nonTableFields[label] = field;
+                        }
+                    }
+                    else {
+                        containerFields[containerIndex].nonTableFields[label] = field;
+                    }
+                }
+            });
+            const tableData = [];
+            const nonTableData = [];
+            // Process table data with both iframe and shadow DOM support
+            for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
+                const container = containers[containerIndex];
+                const { tableFields } = containerFields[containerIndex];
+                if (Object.keys(tableFields).length > 0) {
+                    const firstField = Object.values(tableFields)[0];
+                    const firstElement = queryElement(container, firstField.selector);
+                    let tableContext = firstElement;
+                    // Find table context including both iframe and shadow DOM
+                    while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) {
+                        if (tableContext.getRootNode() instanceof ShadowRoot) {
+                            tableContext = tableContext.getRootNode().host;
+                            continue;
+                        }
+                        if (tableContext.tagName === 'IFRAME') {
+                            try {
+                                tableContext = tableContext.contentDocument.body;
+                            }
+                            catch (e) {
+                                break;
                             }
                         }
+                        else {
+                            tableContext = tableContext.parentElement;
+                        }
+                    }
+                    if (tableContext) {
+                        // Get rows from all contexts
+                        const rows = [];
+                        // Get rows from regular DOM
+                        rows.push(...tableContext.getElementsByTagName('TR'));
+                        // Get rows from shadow DOM
+                        if (tableContext.shadowRoot) {
+                            rows.push(...tableContext.shadowRoot.getElementsByTagName('TR'));
+                        }
+                        // Get rows from iframes
+                        if (tableContext.tagName === 'IFRAME') {
+                            try {
+                                const iframeDoc = tableContext.contentDocument || tableContext.contentWindow.document;
+                                rows.push(...iframeDoc.getElementsByTagName('TR'));
+                            }
+                            catch (e) {
+                                console.warn('Cannot access iframe rows:', e);
+                            }
+                        }
+                        const processedRows = filterRowsBasedOnTag(rows, tableFields);
+                        for (let rowIndex = 0; rowIndex < Math.min(processedRows.length, limit); rowIndex++) {
+                            const record = {};
+                            const currentRow = processedRows[rowIndex];
+                            for (const [label, { selector, attribute, cellIndex }] of Object.entries(tableFields)) {
+                                let element = null;
+                                if (cellIndex >= 0) {
+                                    // Get TD element considering both contexts
+                                    let td = currentRow.children[cellIndex];
+                                    // Check shadow DOM for td
+                                    if (!td && currentRow.shadowRoot) {
+                                        const shadowCells = currentRow.shadowRoot.children;
+                                        if (shadowCells && shadowCells.length > cellIndex) {
+                                            td = shadowCells[cellIndex];
+                                        }
+                                    }
+                                    if (td) {
+                                        element = queryElement(td, selector);
+                                        if (!element && selector.split(/(?:>>|:>>)/).pop().includes('td:nth-child')) {
+                                            element = td;
+                                        }
+                                        if (!element) {
+                                            const tagOnlySelector = selector.split('.')[0];
+                                            element = queryElement(td, tagOnlySelector);
+                                        }
+                                        if (!element) {
+                                            let currentElement = td;
+                                            while (currentElement && currentElement.children.length > 0) {
+                                                let foundContentChild = false;
+                                                for (const child of currentElement.children) {
+                                                    if (extractValue(child, attribute)) {
+                                                        currentElement = child;
+                                                        foundContentChild = true;
+                                                        break;
+                                                    }
+                                                }
+                                                if (!foundContentChild)
+                                                    break;
+                                            }
+                                            element = currentElement;
+                                        }
+                                    }
+                                }
+                                else {
+                                    element = queryElement(currentRow, selector);
+                                }
+                                if (element) {
+                                    record[label] = extractValue(element, attribute);
+                                }
+                            }
+                            if (Object.keys(record).length > 0) {
+                                tableData.push(record);
+                            }
+                        }
+                    }
+                }
+            }
+            // Process non-table data with both contexts support
+            for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
+                if (nonTableData.length >= limit)
+                    break;
+                const container = containers[containerIndex];
+                const { nonTableFields } = containerFields[containerIndex];
+                if (Object.keys(nonTableFields).length > 0) {
+                    const record = {};
+                    for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
+                        // Get the last part of the selector after any context delimiter
+                        const relativeSelector = selector.split(/(?:>>|:>>)/).slice(-1)[0];
+                        const element = queryElement(container, relativeSelector);
+                        if (element) {
+                            record[label] = extractValue(element, attribute);
+                        }
+                    }
+                    if (Object.keys(record).length > 0) {
+                        nonTableData.push(record);
                     }
-                    scrapedData.push(record);
                 }
             }
+            // Merge and limit the results
+            const scrapedData = [...tableData, ...nonTableData];
             return scrapedData;
         });
     };

package/build/interpret.d.ts

@@ -86,6 +86,8 @@ export default class Interpreter extends EventEmitter {
     private carryOutSteps;
     private handlePagination;
     private getMatchingActionId;
+    private removeShadowSelectors;
+    private removeSpecialSelectors;
     private runLoop;
     private ensureScriptsLoaded;
     /**

package/build/interpret.js

@@ -84,14 +84,24 @@ class Interpreter extends events_1.EventEmitter {
     applyAdBlocker(page) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.blocker) {
-                yield this.blocker.enableBlockingInPage(page);
+                try {
+                    yield this.blocker.enableBlockingInPage(page);
+                }
+                catch (err) {
+                    this.log(`Ad-blocker operation failed:`, logger_1.Level.ERROR);
+                }
             }
         });
     }
     disableAdBlocker(page) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.blocker) {
-                yield this.blocker.disableBlockingInPage(page);
+                try {
+                    yield this.blocker.disableBlockingInPage(page);
+                }
+                catch (err) {
+                    this.log(`Ad-blocker operation failed:`, logger_1.Level.ERROR);
+                }
             }
         });
     }
@@ -156,8 +166,8 @@ class Interpreter extends events_1.EventEmitter {
             // const actionable = async (selector: string): Promise<boolean> => {
             //   try {
             //     const proms = [
-            //       page.isEnabled(selector, { timeout: 5000 }),
-            //       page.isVisible(selector, { timeout: 5000 }),
+            //       page.isEnabled(selector, { timeout: 10000 }),
+            //       page.isVisible(selector, { timeout: 10000 }),
             //     ];
             //     return await Promise.all(proms).then((bools) => bools.every((x) => x));
             //   } catch (e) {
@@ -176,6 +186,15 @@ class Interpreter extends events_1.EventEmitter {
             //     return [];
             //   }),
             // ).then((x) => x.flat());
+            const presentSelectors = yield Promise.all(selectors.map((selector) => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield page.waitForSelector(selector, { state: 'attached' });
+                    return [selector];
+                }
+                catch (e) {
+                    return [];
+                }
+            }))).then((x) => x.flat());
             const action = workflowCopy[workflowCopy.length - 1];
             // console.log("Next action:", action)
             let url = page.url();
@@ -186,7 +205,7 @@ class Interpreter extends events_1.EventEmitter {
                 url,
                 cookies: (yield page.context().cookies([page.url()]))
                     .reduce((p, cookie) => (Object.assign(Object.assign({}, p), { [cookie.name]: cookie.value })), {}),
-                selectors,
+                selectors: presentSelectors,
             };
         });
     }
@@ -420,7 +439,12 @@ class Interpreter extends events_1.EventEmitter {
                             yield executeAction(invokee, methodName, step.args);
                         }
                         catch (error) {
-                            yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                            try {
+                                yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                            }
+                            catch (error) {
+                                continue;
+                            }
                         }
                     }
                     else {
@@ -544,12 +568,39 @@ class Interpreter extends events_1.EventEmitter {
             }
         }
     }
+    removeShadowSelectors(workflow) {
+        for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
+            const step = workflow[actionId];
+            // Check if step has where and selectors
+            if (step.where && Array.isArray(step.where.selectors)) {
+                // Filter out selectors that contain ">>"
+                step.where.selectors = step.where.selectors.filter(selector => !selector.includes('>>'));
+            }
+        }
+        return workflow;
+    }
+    removeSpecialSelectors(workflow) {
+        for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
+            const step = workflow[actionId];
+            if (step.where && Array.isArray(step.where.selectors)) {
+                // Filter out if selector has EITHER ":>>" OR ">>"
+                step.where.selectors = step.where.selectors.filter(selector => !(selector.includes(':>>') || selector.includes('>>')));
+            }
+        }
+        return workflow;
+    }
     runLoop(p, workflow) {
         var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {
-            const workflowCopy = JSON.parse(JSON.stringify(workflow));
+            let workflowCopy = JSON.parse(JSON.stringify(workflow));
+            workflowCopy = this.removeSpecialSelectors(workflowCopy);
             // apply ad-blocker to the current page
-            yield this.applyAdBlocker(p);
+            try {
+                yield this.applyAdBlocker(p);
+            }
+            catch (error) {
+                this.log(`Failed to apply ad-blocker: ${error.message}`, logger_1.Level.ERROR);
+            }
             const usedActions = [];
             let selectors = [];
             let lastAction = null;
@@ -660,6 +711,7 @@ class Interpreter extends events_1.EventEmitter {
         return __awaiter(this, void 0, void 0, function* () {
             this.log('Starting the workflow.', logger_1.Level.LOG);
             const context = page.context();
+            page.setDefaultNavigationTimeout(100000);
             // Check proxy settings from context options
             const contextOptions = context._options;
             const hasProxy = !!(contextOptions === null || contextOptions === void 0 ? void 0 : contextOptions.proxy);

package/build/utils/concurrency.d.ts

@@ -3,43 +3,43 @@
  */
 export default class Concurrency {
     /**
-       * Maximum number of workers running in parallel. If set to `null`, there is no limit.
-       */
+    * Maximum number of workers running in parallel. If set to `null`, there is no limit.
+    */
     maxConcurrency: number;
     /**
-       * Number of currently active workers.
-       */
+    * Number of currently active workers.
+    */
     activeWorkers: number;
     /**
-       * Queue of jobs waiting to be completed.
-       */
+    * Queue of jobs waiting to be completed.
+    */
     private jobQueue;
     /**
-       * "Resolve" callbacks of the waitForCompletion() promises.
-       */
+    * "Resolve" callbacks of the waitForCompletion() promises.
+    */
     private waiting;
     /**
-       * Constructs a new instance of concurrency manager.
-       * @param {number} maxConcurrency Maximum number of workers running in parallel.
-       */
+    * Constructs a new instance of concurrency manager.
+    * @param {number} maxConcurrency Maximum number of workers running in parallel.
+    */
     constructor(maxConcurrency: number);
     /**
-       * Takes a waiting job out of the queue and runs it.
-       */
+    * Takes a waiting job out of the queue and runs it.
+    */
     private runNextJob;
     /**
-       * Pass a job (a time-demanding async function) to the concurrency manager. \
-       * The time of the job's execution depends on the concurrency manager itself
-       * (given a generous enough `maxConcurrency` value, it might be immediate,
-       * but this is not guaranteed).
-       * @param worker Async function to be executed (job to be processed).
-       */
+    * Pass a job (a time-demanding async function) to the concurrency manager. \
+    * The time of the job's execution depends on the concurrency manager itself
+    * (given a generous enough `maxConcurrency` value, it might be immediate,
+    * but this is not guaranteed).
+    * @param worker Async function to be executed (job to be processed).
+    */
     addJob(job: () => Promise<any>): void;
     /**
-       * Waits until there is no running nor waiting job. \
-       * If the concurrency manager is idle at the time of calling this function,
-       * it waits until at least one job is completed (can be "presubscribed").
-       * @returns Promise, resolved after there is no running/waiting worker.
-       */
+    * Waits until there is no running nor waiting job. \
+    * If the concurrency manager is idle at the time of calling this function,
+    * it waits until at least one job is completed (can be "presubscribed").
+    * @returns Promise, resolved after there is no running/waiting worker.
+    */
     waitForCompletion(): Promise<void>;
 }

package/build/utils/concurrency.js

@@ -5,31 +5,31 @@ Object.defineProperty(exports, "__esModule", { value: true });
  */
 class Concurrency {
     /**
-       * Constructs a new instance of concurrency manager.
-       * @param {number} maxConcurrency Maximum number of workers running in parallel.
-       */
+    * Constructs a new instance of concurrency manager.
+    * @param {number} maxConcurrency Maximum number of workers running in parallel.
+    */
     constructor(maxConcurrency) {
         /**
-           * Maximum number of workers running in parallel. If set to `null`, there is no limit.
-           */
+        * Maximum number of workers running in parallel. If set to `null`, there is no limit.
+        */
         this.maxConcurrency = 1;
         /**
-           * Number of currently active workers.
-           */
+        * Number of currently active workers.
+        */
         this.activeWorkers = 0;
         /**
-           * Queue of jobs waiting to be completed.
-           */
+        * Queue of jobs waiting to be completed.
+        */
         this.jobQueue = [];
         /**
-           * "Resolve" callbacks of the waitForCompletion() promises.
-           */
+        * "Resolve" callbacks of the waitForCompletion() promises.
+        */
         this.waiting = [];
         this.maxConcurrency = maxConcurrency;
     }
     /**
-       * Takes a waiting job out of the queue and runs it.
-       */
+    * Takes a waiting job out of the queue and runs it.
+    */
     runNextJob() {
         const job = this.jobQueue.pop();
         if (job) {
@@ -49,12 +49,12 @@ class Concurrency {
         }
     }
     /**
-       * Pass a job (a time-demanding async function) to the concurrency manager. \
-       * The time of the job's execution depends on the concurrency manager itself
-       * (given a generous enough `maxConcurrency` value, it might be immediate,
-       * but this is not guaranteed).
-       * @param worker Async function to be executed (job to be processed).
-       */
+    * Pass a job (a time-demanding async function) to the concurrency manager. \
+    * The time of the job's execution depends on the concurrency manager itself
+    * (given a generous enough `maxConcurrency` value, it might be immediate,
+    * but this is not guaranteed).
+    * @param worker Async function to be executed (job to be processed).
+    */
     addJob(job) {
         // console.debug("Adding a worker!");
         this.jobQueue.push(job);
@@ -67,11 +67,11 @@ class Concurrency {
         }
     }
     /**
-       * Waits until there is no running nor waiting job. \
-       * If the concurrency manager is idle at the time of calling this function,
-       * it waits until at least one job is completed (can be "presubscribed").
-       * @returns Promise, resolved after there is no running/waiting worker.
-       */
+    * Waits until there is no running nor waiting job. \
+    * If the concurrency manager is idle at the time of calling this function,
+    * it waits until at least one job is completed (can be "presubscribed").
+    * @returns Promise, resolved after there is no running/waiting worker.
+    */
     waitForCompletion() {
         return new Promise((res) => {
             this.waiting.push(res);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",