maxun-core 0.0.13 → 0.0.15

package/build/browserSide/scraper.js

@@ -467,7 +467,25 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 }
                 else if (attribute === 'src' || attribute === 'href') {
                     const attrValue = element.getAttribute(attribute);
-                    return attrValue ? new URL(attrValue, baseURL).href : null;
+                    const dataAttr = attrValue || element.getAttribute('data-' + attribute);
+                    if (!dataAttr || dataAttr.trim() === '') {
+                        if (attribute === 'src') {
+                            const style = window.getComputedStyle(element);
+                            const bgImage = style.backgroundImage;
+                            if (bgImage && bgImage !== 'none') {
+                                const matches = bgImage.match(/url\(['"]?([^'")]+)['"]?\)/);
+                                return matches ? new URL(matches[1], baseURL).href : null;
+                            }
+                        }
+                        return null;
+                    }
+                    try {
+                        return new URL(dataAttr, baseURL).href;
+                    }
+                    catch (e) {
+                        console.warn('Error creating URL from', dataAttr, e);
+                        return dataAttr; // Return the original value if URL construction fails
+                    }
                 }
                 return element.getAttribute(attribute);
             }

package/build/interpret.js

@@ -477,6 +477,7 @@ class Interpreter extends events_1.EventEmitter {
             let visitedUrls = new Set();
             const MAX_RETRIES = 3;
             const RETRY_DELAY = 1000; // 1 second delay between retries
+            const MAX_UNCHANGED_RESULTS = 5;
             const debugLog = (message, ...args) => {
                 console.log(`[Page ${visitedUrls.size}] [URL: ${page.url()}] ${message}`, ...args);
             };
@@ -555,30 +556,55 @@ class Interpreter extends events_1.EventEmitter {
                 }
             });
             let availableSelectors = config.pagination.selector.split(',');
+            let unchangedResultCounter = 0;
             try {
                 while (true) {
-                    // Reduced timeout for faster performance
-                    yield page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { });
                     switch (config.pagination.type) {
                         case 'scrollDown': {
+                            let previousResultCount = allResults.length;
+                            yield scrapeCurrentPage();
+                            if (checkLimit()) {
+                                return allResults;
+                            }
                             yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
                             yield page.waitForTimeout(2000);
                             const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                            const currentResultCount = allResults.length;
+                            if (currentResultCount === previousResultCount) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
+                                    return allResults;
+                                }
+                            }
+                            else {
+                                unchangedResultCounter = 0;
+                            }
                             if (currentHeight === previousHeight) {
-                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
                             previousHeight = currentHeight;
                             break;
                         }
                         case 'scrollUp': {
+                            let previousResultCount = allResults.length;
+                            yield scrapeCurrentPage();
+                            if (checkLimit()) {
+                                return allResults;
+                            }
                             yield page.evaluate(() => window.scrollTo(0, 0));
                             yield page.waitForTimeout(2000);
                             const currentTopHeight = yield page.evaluate(() => document.documentElement.scrollTop);
+                            const currentResultCount = allResults.length;
+                            if (currentResultCount === previousResultCount) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
+                                    return allResults;
+                                }
+                            }
+                            else {
+                                unchangedResultCounter = 0;
+                            }
                             if (currentTopHeight === 0) {
-                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
                             previousHeight = currentTopHeight;
@@ -609,64 +635,93 @@ class Interpreter extends events_1.EventEmitter {
                                 break;
                             }
                             let retryCount = 0;
-                            let navigationSuccess = false;
-                            while (retryCount < MAX_RETRIES && !navigationSuccess) {
+                            let paginationSuccess = false;
+                            // Capture basic content signature before click
+                            const captureContentSignature = () => __awaiter(this, void 0, void 0, function* () {
+                                return yield page.evaluate((selector) => {
+                                    const items = document.querySelectorAll(selector);
+                                    return {
+                                        url: window.location.href,
+                                        itemCount: items.length,
+                                        firstItems: Array.from(items).slice(0, 3).map(el => el.textContent || '').join('|')
+                                    };
+                                }, config.listSelector);
+                            });
+                            const beforeSignature = yield captureContentSignature();
+                            debugLog(`Before click: ${beforeSignature.itemCount} items`);
+                            while (retryCount < MAX_RETRIES && !paginationSuccess) {
                                 try {
                                     try {
                                         yield Promise.all([
                                             page.waitForNavigation({
                                                 waitUntil: 'networkidle',
                                                 timeout: 15000
+                                            }).catch(e => {
+                                                throw e;
                                             }),
                                             button.click()
                                         ]);
-                                        navigationSuccess = true;
+                                        debugLog("Navigation successful after regular click");
+                                        paginationSuccess = true;
                                     }
-                                    catch (error) {
-                                        debugLog(`Regular click failed on attempt ${retryCount + 1}. Trying DispatchEvent`);
-                                        // If regular click fails, try dispatchEvent
-                                        if (page.url() === currentUrl) {
+                                    catch (navError) {
+                                        debugLog("Regular click with navigation failed, trying dispatch event with navigation");
+                                        try {
+                                            yield Promise.all([
+                                                page.waitForNavigation({
+                                                    waitUntil: 'networkidle',
+                                                    timeout: 15000
+                                                }).catch(e => {
+                                                    throw e;
+                                                }),
+                                                button.dispatchEvent('click')
+                                            ]);
+                                            debugLog("Navigation successful after dispatch event");
+                                            paginationSuccess = true;
+                                        }
+                                        catch (dispatchNavError) {
                                             try {
-                                                yield Promise.all([
-                                                    page.waitForNavigation({
-                                                        waitUntil: 'networkidle',
-                                                        timeout: 15000
-                                                    }),
-                                                    button.dispatchEvent('click')
-                                                ]);
-                                                navigationSuccess = true;
+                                                yield button.click();
+                                                yield page.waitForTimeout(2000);
                                             }
-                                            catch (dispatchError) {
-                                                debugLog(`DispatchEvent failed on attempt ${retryCount + 1}.`);
+                                            catch (clickError) {
+                                                yield button.dispatchEvent('click');
+                                                yield page.waitForTimeout(2000);
                                             }
                                         }
-                                        else {
-                                            navigationSuccess = true;
-                                        }
                                     }
-                                    const newUrl = page.url();
-                                    if (visitedUrls.has(newUrl)) {
-                                        debugLog(`Detected navigation to previously visited URL ${newUrl} on attempt ${retryCount + 1}`);
-                                        navigationSuccess = false;
-                                    }
-                                    if (navigationSuccess) {
-                                        yield page.waitForTimeout(1000);
+                                    yield page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => { });
+                                    if (!paginationSuccess) {
+                                        const newUrl = page.url();
+                                        const afterSignature = yield captureContentSignature();
+                                        if (newUrl !== currentUrl) {
+                                            debugLog(`URL changed to ${newUrl}`);
+                                            visitedUrls.add(newUrl);
+                                            paginationSuccess = true;
+                                        }
+                                        else if (afterSignature.firstItems !== beforeSignature.firstItems) {
+                                            debugLog("Content changed without URL change");
+                                            paginationSuccess = true;
+                                        }
+                                        else if (afterSignature.itemCount !== beforeSignature.itemCount) {
+                                            debugLog(`Item count changed from ${beforeSignature.itemCount} to ${afterSignature.itemCount}`);
+                                            paginationSuccess = true;
+                                        }
                                     }
                                 }
                                 catch (error) {
-                                    debugLog(`Navigation attempt ${retryCount + 1} failed completely.`);
-                                    navigationSuccess = false;
+                                    debugLog(`Pagination attempt ${retryCount + 1} failed: ${error.message}`);
                                 }
-                                if (!navigationSuccess) {
+                                if (!paginationSuccess) {
                                     retryCount++;
                                     if (retryCount < MAX_RETRIES) {
-                                        debugLog(`Retrying navigation - attempt ${retryCount + 1} of ${MAX_RETRIES}`);
+                                        debugLog(`Retrying pagination - attempt ${retryCount + 1} of ${MAX_RETRIES}`);
                                         yield page.waitForTimeout(RETRY_DELAY);
                                     }
                                 }
                             }
-                            if (!navigationSuccess) {
-                                debugLog(`Navigation failed after ${MAX_RETRIES} attempts`);
+                            if (!paginationSuccess) {
+                                debugLog(`Pagination failed after ${MAX_RETRIES} attempts`);
                                 return allResults;
                             }
                             break;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.13",
+  "version": "0.0.15",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",