maxun-core 0.0.12 → 0.0.13

package/build/browserSide/scraper.js

@@ -174,7 +174,6 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             if (!config.selector.includes('>>') && !config.selector.includes(':>>')) {
                 return Array.from(document.querySelectorAll(config.selector));
             }
-            // First handle iframe traversal if present
             if (config.selector.includes(':>>')) {
                 const parts = config.selector.split(':>>').map(s => s.trim());
                 let currentElements = [document];
@@ -185,24 +184,42 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     const isLast = i === parts.length - 1;
                     for (const element of currentElements) {
                         try {
-                            // For document or iframe document
                             const doc = element.contentDocument || element || ((_a = element.contentWindow) === null || _a === void 0 ? void 0 : _a.document);
                             if (!doc)
                                 continue;
-                            // Query elements in current context
+                            if (part.startsWith('frame[name=') || part.startsWith('iframe[name=')) {
+                                const nameMatch = part.match(/\[name=['"]([^'"]+)['"]\]/);
+                                if (nameMatch && nameMatch[1]) {
+                                    const frameName = nameMatch[1];
+                                    let foundFrames = [];
+                                    if (doc.getElementsByName && typeof doc.getElementsByName === 'function') {
+                                        foundFrames = Array.from(doc.getElementsByName(frameName))
+                                            .filter(el => el.tagName === 'FRAME' || el.tagName === 'IFRAME');
+                                    }
+                                    if (foundFrames.length === 0) {
+                                        const framesBySelector = Array.from(doc.querySelectorAll(`frame[name="${frameName}"], iframe[name="${frameName}"]`));
+                                        foundFrames = framesBySelector;
+                                    }
+                                    if (isLast) {
+                                        nextElements.push(...foundFrames);
+                                    }
+                                    else {
+                                        nextElements.push(...foundFrames);
+                                    }
+                                    continue;
+                                }
+                            }
                             const found = Array.from(doc.querySelectorAll(part));
                             if (isLast) {
-                                // If it's the last part, keep all matching elements
                                 nextElements.push(...found);
                             }
                             else {
-                                // If not last, only keep iframes for next iteration
-                                const iframes = found.filter(el => el.tagName === 'IFRAME');
-                                nextElements.push(...iframes);
+                                const frames = found.filter(el => el.tagName === 'IFRAME' || el.tagName === 'FRAME');
+                                nextElements.push(...frames);
                             }
                         }
                         catch (error) {
-                            console.warn('Cannot access iframe content:', error, {
+                            console.warn('Cannot access iframe/frame content:', error, {
                                 part,
                                 element,
                                 index: i
@@ -242,13 +259,19 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             }
             return [];
         }
-        // Modified to handle iframe context for URL resolution
         function getElementValue(element, attribute) {
-            var _a, _b, _c, _d, _e;
+            var _a, _b, _c, _d, _e, _f;
             if (!element)
                 return null;
-            // Get the base URL for resolving relative URLs
-            const baseURL = ((_b = (_a = element.ownerDocument) === null || _a === void 0 ? void 0 : _a.location) === null || _b === void 0 ? void 0 : _b.href) || window.location.origin;
+            let baseURL;
+            try {
+                baseURL = ((_b = (_a = element.ownerDocument) === null || _a === void 0 ? void 0 : _a.location) === null || _b === void 0 ? void 0 : _b.href) ||
+                    ((_c = element.ownerDocument) === null || _c === void 0 ? void 0 : _c.baseURI) ||
+                    window.location.origin;
+            }
+            catch (e) {
+                baseURL = window.location.origin;
+            }
             switch (attribute) {
                 case 'href': {
                     const relativeHref = element.getAttribute('href');
@@ -259,11 +282,15 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     return relativeSrc ? new URL(relativeSrc, baseURL).href : null;
                 }
                 case 'innerText':
-                    return (_c = element.innerText) === null || _c === void 0 ? void 0 : _c.trim();
+                    return (_d = element.innerText) === null || _d === void 0 ? void 0 : _d.trim();
                 case 'textContent':
-                    return (_d = element.textContent) === null || _d === void 0 ? void 0 : _d.trim();
+                    return (_e = element.textContent) === null || _e === void 0 ? void 0 : _e.trim();
+                case 'innerHTML':
+                    return element.innerHTML;
+                case 'outerHTML':
+                    return element.outerHTML;
                 default:
-                    return element.getAttribute(attribute) || ((_e = element.innerText) === null || _e === void 0 ? void 0 : _e.trim());
+                    return element.getAttribute(attribute) || ((_f = element.innerText) === null || _f === void 0 ? void 0 : _f.trim());
             }
         }
         // Rest of the functions remain largely the same
@@ -332,7 +359,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
    */
     window.scrapeList = function ({ listSelector, fields, limit = 10 }) {
         return __awaiter(this, void 0, void 0, function* () {
-            // Enhanced query function to handle both iframe and shadow DOM
+            // Enhanced query function to handle iframe, frame and shadow DOM
             const queryElement = (rootElement, selector) => {
                 if (!selector.includes('>>') && !selector.includes(':>>')) {
                     return rootElement.querySelector(selector);
@@ -342,15 +369,15 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 for (let i = 0; i < parts.length; i++) {
                     if (!currentElement)
                         return null;
-                    // Handle iframe traversal
-                    if (currentElement.tagName === 'IFRAME') {
+                    // Handle iframe and frame traversal
+                    if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') {
                         try {
-                            const iframeDoc = currentElement.contentDocument || currentElement.contentWindow.document;
-                            currentElement = iframeDoc.querySelector(parts[i]);
+                            const frameDoc = currentElement.contentDocument || currentElement.contentWindow.document;
+                            currentElement = frameDoc.querySelector(parts[i]);
                             continue;
                         }
                         catch (e) {
-                            console.warn('Cannot access iframe content:', e);
+                            console.warn(`Cannot access ${currentElement.tagName.toLowerCase()} content:`, e);
                             return null;
                         }
                     }
@@ -385,14 +412,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 for (const part of parts) {
                     const nextElements = [];
                     for (const element of currentElements) {
-                        // Handle iframe traversal
-                        if (element.tagName === 'IFRAME') {
+                        // Handle iframe and frame traversal
+                        if (element.tagName === 'IFRAME' || element.tagName === 'FRAME') {
                             try {
-                                const iframeDoc = element.contentDocument || element.contentWindow.document;
-                                nextElements.push(...iframeDoc.querySelectorAll(part));
+                                const frameDoc = element.contentDocument || element.contentWindow.document;
+                                nextElements.push(...frameDoc.querySelectorAll(part));
                             }
                             catch (e) {
-                                console.warn('Cannot access iframe content:', e);
+                                console.warn(`Cannot access ${element.tagName.toLowerCase()} content:`, e);
                                 continue;
                             }
                         }
@@ -461,8 +488,8 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     else if (currentElement.tagName === 'TR') {
                         return { type: 'TR', element: currentElement };
                     }
-                    // Handle iframe crossing
-                    if (currentElement.tagName === 'IFRAME') {
+                    // Handle iframe and frame crossing
+                    if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') {
                         try {
                             currentElement = currentElement.contentDocument.body;
                         }
@@ -504,7 +531,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                             }
                             if (current.tagName === 'TH')
                                 return true;
-                            if (current.tagName === 'IFRAME') {
+                            if (current.tagName === 'IFRAME' || current.tagName === 'FRAME') {
                                 try {
                                     current = current.contentDocument.body;
                                 }
@@ -556,15 +583,18 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     const shadowHost = baseElement.getRootNode().host;
                     allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName));
                 }
-                // Get elements from iframes
-                const iframes = document.getElementsByTagName('iframe');
-                for (const iframe of iframes) {
+                // Get elements from iframes and frames
+                const frames = [
+                    ...Array.from(document.getElementsByTagName('iframe')),
+                    ...Array.from(document.getElementsByTagName('frame'))
+                ];
+                for (const frame of frames) {
                     try {
-                        const iframeDoc = iframe.contentDocument || iframe.contentWindow.document;
-                        allElements.push(...iframeDoc.getElementsByTagName(baseElement.tagName));
+                        const frameDoc = frame.contentDocument || frame.contentWindow.document;
+                        allElements.push(...frameDoc.getElementsByTagName(baseElement.tagName));
                     }
                     catch (e) {
-                        console.warn('Cannot access iframe content:', e);
+                        console.warn(`Cannot access ${frame.tagName.toLowerCase()} content:`, e);
                     }
                 }
                 return allElements.filter(element => {
@@ -611,7 +641,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
             });
             const tableData = [];
             const nonTableData = [];
-            // Process table data with both iframe and shadow DOM support
+            // Process table data with support for iframes, frames, and shadow DOM
             for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
                 const container = containers[containerIndex];
                 const { tableFields } = containerFields[containerIndex];
@@ -619,13 +649,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     const firstField = Object.values(tableFields)[0];
                     const firstElement = queryElement(container, firstField.selector);
                     let tableContext = firstElement;
-                    // Find table context including both iframe and shadow DOM
+                    // Find table context including iframe, frame and shadow DOM
                     while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) {
                         if (tableContext.getRootNode() instanceof ShadowRoot) {
                             tableContext = tableContext.getRootNode().host;
                             continue;
                         }
-                        if (tableContext.tagName === 'IFRAME') {
+                        if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') {
                             try {
                                 tableContext = tableContext.contentDocument.body;
                             }
@@ -646,14 +676,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                         if (tableContext.shadowRoot) {
                             rows.push(...tableContext.shadowRoot.getElementsByTagName('TR'));
                         }
-                        // Get rows from iframes
-                        if (tableContext.tagName === 'IFRAME') {
+                        // Get rows from iframes and frames
+                        if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') {
                             try {
-                                const iframeDoc = tableContext.contentDocument || tableContext.contentWindow.document;
-                                rows.push(...iframeDoc.getElementsByTagName('TR'));
+                                const frameDoc = tableContext.contentDocument || tableContext.contentWindow.document;
+                                rows.push(...frameDoc.getElementsByTagName('TR'));
                             }
                             catch (e) {
-                                console.warn('Cannot access iframe rows:', e);
+                                console.warn(`Cannot access ${tableContext.tagName.toLowerCase()} rows:`, e);
                             }
                         }
                         const processedRows = filterRowsBasedOnTag(rows, tableFields);
@@ -713,7 +743,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     }
                 }
             }
-            // Process non-table data with both contexts support
+            // Process non-table data with all contexts support
             for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
                 if (nonTableData.length >= limit)
                     break;

package/build/interpret.js

@@ -234,6 +234,11 @@ class Interpreter extends events_1.EventEmitter {
             parsedSuperset[key] = Array.isArray(superset[key])
                 ? (0, utils_1.arrayToObject)(superset[key])
                 : superset[key];
+            if ((key === 'url' || key === 'selectors') &&
+                Array.isArray(value) && Array.isArray(superset[key]) &&
+                value.length === 0 && superset[key].length === 0) {
+                return true;
+            }
             if (key === 'selectors' && Array.isArray(value) && Array.isArray(superset[key])) {
                 return value.some(selector => superset[key].includes(selector));
             }
@@ -495,29 +500,45 @@ class Interpreter extends events_1.EventEmitter {
                 return false;
             };
             // Enhanced button finder with retry mechanism
-            const findWorkingButton = (selectors, retryCount = 0) => __awaiter(this, void 0, void 0, function* () {
-                for (const selector of selectors) {
-                    try {
-                        const button = yield page.waitForSelector(selector, {
-                            state: 'attached',
-                            timeout: 10000 // Reduced timeout for faster checks
-                        });
-                        if (button) {
-                            debugLog('Found working selector:', selector);
-                            return { button, workingSelector: selector };
+            const findWorkingButton = (selectors) => __awaiter(this, void 0, void 0, function* () {
+                let updatedSelectors = [...selectors];
+                for (let i = 0; i < selectors.length; i++) {
+                    const selector = selectors[i];
+                    let retryCount = 0;
+                    let selectorSuccess = false;
+                    while (retryCount < MAX_RETRIES && !selectorSuccess) {
+                        try {
+                            const button = yield page.waitForSelector(selector, {
+                                state: 'attached',
+                                timeout: 10000
+                            });
+                            if (button) {
+                                debugLog('Found working selector:', selector);
+                                return {
+                                    button,
+                                    workingSelector: selector,
+                                    updatedSelectors
+                                };
+                            }
+                        }
+                        catch (error) {
+                            retryCount++;
+                            debugLog(`Selector "${selector}" failed: attempt ${retryCount}/${MAX_RETRIES}`);
+                            if (retryCount < MAX_RETRIES) {
+                                yield page.waitForTimeout(RETRY_DELAY);
+                            }
+                            else {
+                                debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
+                                updatedSelectors = updatedSelectors.filter(s => s !== selector);
+                            }
                         }
                     }
-                    catch (error) {
-                        debugLog(`Selector failed: ${selector}`);
-                    }
-                }
-                // Implement retry mechanism when no selectors work
-                if (selectors.length > 0 && retryCount < MAX_RETRIES) {
-                    debugLog(`Retry attempt ${retryCount + 1} of ${MAX_RETRIES}`);
-                    yield page.waitForTimeout(RETRY_DELAY);
-                    return findWorkingButton(selectors, retryCount + 1);
                 }
-                return { button: null, workingSelector: null };
+                return {
+                    button: null,
+                    workingSelector: null,
+                    updatedSelectors
+                };
             });
             const retryOperation = (operation, retryCount = 0) => __awaiter(this, void 0, void 0, function* () {
                 try {
@@ -569,7 +590,8 @@ class Interpreter extends events_1.EventEmitter {
                             yield scrapeCurrentPage();
                             if (checkLimit())
                                 return allResults;
-                            const { button, workingSelector } = yield findWorkingButton(availableSelectors);
+                            const { button, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
+                            availableSelectors = updatedSelectors;
                             if (!button || !workingSelector) {
                                 // Final retry for navigation when no selectors work
                                 const success = yield retryOperation(() => __awaiter(this, void 0, void 0, function* () {
@@ -586,7 +608,6 @@ class Interpreter extends events_1.EventEmitter {
                                     return allResults;
                                 break;
                             }
-                            availableSelectors = availableSelectors.slice(availableSelectors.indexOf(workingSelector));
                             let retryCount = 0;
                             let navigationSuccess = false;
                             while (retryCount < MAX_RETRIES && !navigationSuccess) {
@@ -651,17 +672,21 @@ class Interpreter extends events_1.EventEmitter {
                             break;
                         }
                         case 'clickLoadMore': {
+                            yield scrapeCurrentPage();
+                            if (checkLimit())
+                                return allResults;
+                            let loadMoreCounter = 0;
+                            let previousResultCount = allResults.length;
+                            let noNewItemsCounter = 0;
+                            const MAX_NO_NEW_ITEMS = 2;
                             while (true) {
-                                // Find working button with retry mechanism, consistent with clickNext
-                                const { button: loadMoreButton, workingSelector } = yield findWorkingButton(availableSelectors);
+                                // Find working button with retry mechanism
+                                const { button: loadMoreButton, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
+                                availableSelectors = updatedSelectors;
                                 if (!workingSelector || !loadMoreButton) {
                                     debugLog('No working Load More selector found after retries');
-                                    const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                    allResults = allResults.concat(finalResults);
                                     return allResults;
                                 }
-                                // Update available selectors to start from the working one
-                                availableSelectors = availableSelectors.slice(availableSelectors.indexOf(workingSelector));
                                 // Implement retry mechanism for clicking the button
                                 let retryCount = 0;
                                 let clickSuccess = false;
@@ -685,6 +710,8 @@ class Interpreter extends events_1.EventEmitter {
                                         }
                                         if (clickSuccess) {
                                             yield page.waitForTimeout(1000);
+                                            loadMoreCounter++;
+                                            debugLog(`Successfully clicked Load More button (${loadMoreCounter} times)`);
                                         }
                                     }
                                     catch (error) {
@@ -698,8 +725,6 @@ class Interpreter extends events_1.EventEmitter {
                                 }
                                 if (!clickSuccess) {
                                     debugLog(`Load More clicking failed after ${MAX_RETRIES} attempts`);
-                                    const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                    allResults = allResults.concat(finalResults);
                                     return allResults;
                                 }
                                 // Wait for content to load and check scroll height
@@ -707,19 +732,30 @@ class Interpreter extends events_1.EventEmitter {
                                 yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
                                 yield page.waitForTimeout(2000);
                                 const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
-                                if (currentHeight === previousHeight) {
+                                const heightChanged = currentHeight !== previousHeight;
+                                previousHeight = currentHeight;
+                                yield scrapeCurrentPage();
+                                const currentResultCount = allResults.length;
+                                const newItemsAdded = currentResultCount > previousResultCount;
+                                if (!newItemsAdded) {
+                                    noNewItemsCounter++;
+                                    debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`);
+                                    if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) {
+                                        debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`);
+                                        return allResults;
+                                    }
+                                }
+                                else {
+                                    noNewItemsCounter = 0;
+                                    previousResultCount = currentResultCount;
+                                }
+                                if (checkLimit())
+                                    return allResults;
+                                if (!heightChanged) {
                                     debugLog('No more items loaded after Load More');
-                                    const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                    allResults = allResults.concat(finalResults);
                                     return allResults;
                                 }
-                                previousHeight = currentHeight;
-                                if (config.limit && allResults.length >= config.limit) {
-                                    allResults = allResults.slice(0, config.limit);
-                                    break;
-                                }
                             }
-                            break;
                         }
                         default: {
                             yield scrapeCurrentPage();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",