maxun-core 0.0.8 → 0.0.10

package/build/interpret.js

@@ -404,6 +404,12 @@ class Interpreter extends events_1.EventEmitter {
             };
             const executeAction = (invokee, methodName, args) => __awaiter(this, void 0, void 0, function* () {
                 console.log("Executing action:", methodName, args);
+                if (methodName === 'press' || methodName === 'type') {
+                    // Extract only the first two arguments for these methods
+                    const limitedArgs = Array.isArray(args) ? args.slice(0, 2) : [args];
+                    yield invokee[methodName](...limitedArgs);
+                    return;
+                }
                 if (!args || Array.isArray(args)) {
                     yield invokee[methodName](...(args !== null && args !== void 0 ? args : []));
                 }
@@ -461,6 +467,12 @@ class Interpreter extends events_1.EventEmitter {
             let previousHeight = 0;
             // track unique items per page to avoid re-scraping
             let scrapedItems = new Set();
+            let visitedUrls = [];
+            // Debug logging helper
+            const debugLog = (message, ...args) => {
+                console.log(`[Page ${visitedUrls.length + 1}] ${message}`, ...args);
+            };
+            let availableSelectors = config.pagination.selector.split(',');
             while (true) {
                 switch (config.pagination.type) {
                     case 'scrollDown':
@@ -486,56 +498,154 @@ class Interpreter extends events_1.EventEmitter {
                         previousHeight = currentTopHeight;
                         break;
                     case 'clickNext':
+                        debugLog("Current URL:", page.url());
                         const pageResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                        // console.log("Page results:", pageResults);
                         // Filter out already scraped items
                         const newResults = pageResults.filter(item => {
                             const uniqueKey = JSON.stringify(item);
                             if (scrapedItems.has(uniqueKey))
-                                return false; // Ignore if already scraped
-                            scrapedItems.add(uniqueKey); // Mark as scraped
+                                return false;
+                            scrapedItems.add(uniqueKey);
                             return true;
                         });
                         allResults = allResults.concat(newResults);
+                        debugLog("Results collected so far:", allResults.length);
                         if (config.limit && allResults.length >= config.limit) {
                             return allResults.slice(0, config.limit);
                         }
-                        const nextButton = yield page.$(config.pagination.selector);
+                        yield page.waitForLoadState('networkidle', { timeout: 30000 });
+                        yield page.waitForTimeout(2000);
+                        let checkButton = null;
+                        let workingSelector = null;
+                        // Try each selector with explicit waiting
+                        for (const selector of availableSelectors) {
+                            try {
+                                checkButton = yield page.waitForSelector(selector, {
+                                    state: 'attached',
+                                    timeout: 30000
+                                });
+                                if (checkButton) {
+                                    workingSelector = selector;
+                                    debugLog('Found working selector:', selector);
+                                    break;
+                                }
+                            }
+                            catch (error) {
+                                debugLog(`Selector failed: ${selector} - ${error.message}`);
+                            }
+                        }
+                        if (!workingSelector) {
+                            debugLog('No working selector found after trying all options');
+                            return allResults;
+                        }
+                        const nextButton = yield page.$(workingSelector);
                         if (!nextButton) {
-                            return allResults; // No more pages to scrape
+                            debugLog('Next button not found');
+                            return allResults;
+                        }
+                        const selectorIndex = availableSelectors.indexOf(workingSelector);
+                        availableSelectors = availableSelectors.slice(selectorIndex);
+                        try {
+                            // Store current URL to check if navigation succeeded
+                            const previousUrl = page.url();
+                            visitedUrls.push(previousUrl);
+                            // Try both click methods in sequence
+                            try {
+                                yield Promise.all([
+                                    page.waitForNavigation({
+                                        waitUntil: 'networkidle',
+                                        timeout: 15000
+                                    }),
+                                    nextButton.click()
+                                ]);
+                            }
+                            catch (error) {
+                                // If we're still on the same URL, try dispatch event
+                                if (page.url() === previousUrl) {
+                                    yield Promise.all([
+                                        page.waitForNavigation({
+                                            waitUntil: 'networkidle',
+                                            timeout: 15000
+                                        }),
+                                        nextButton.dispatchEvent('click')
+                                    ]);
+                                }
+                            }
+                            yield page.waitForLoadState('domcontentloaded');
+                            yield page.waitForLoadState('networkidle', { timeout: 30000 });
+                            const currentUrl = page.url();
+                            if (visitedUrls.includes(currentUrl)) {
+                                debugLog(`Navigation failed/Detected navigation to previously visited URL: ${currentUrl}`);
+                                return allResults;
+                            }
+                            // Give the page a moment to stabilize after navigation
+                            yield page.waitForTimeout(1000);
+                        }
+                        catch (error) {
+                            debugLog(`Navigation failed completely: ${error.message}`);
+                            return allResults;
                         }
-                        yield Promise.all([
-                            nextButton.dispatchEvent('click'),
-                            page.waitForNavigation({ waitUntil: 'networkidle' })
-                        ]);
-                        yield page.waitForTimeout(1000);
                         break;
                     case 'clickLoadMore':
                         while (true) {
-                            const loadMoreButton = yield page.$(config.pagination.selector);
+                            let checkButton = null;
+                            let workingSelector = null;
+                            for (const selector of availableSelectors) {
+                                try {
+                                    checkButton = yield page.waitForSelector(selector, {
+                                        state: 'attached',
+                                        timeout: 30000
+                                    });
+                                    if (checkButton) {
+                                        workingSelector = selector;
+                                        debugLog('Found working selector:', selector);
+                                        break;
+                                    }
+                                }
+                                catch (error) {
+                                    debugLog(`Load More selector failed: ${selector}`);
+                                }
+                            }
+                            if (!workingSelector) {
+                                debugLog('No working Load More selector found');
+                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                                allResults = allResults.concat(finalResults);
+                                return allResults;
+                            }
+                            const loadMoreButton = yield page.$(workingSelector);
                             if (!loadMoreButton) {
-                                // No more "Load More" button, so scrape the remaining items
+                                debugLog('Load More button not found');
                                 const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
                                 allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
-                            // Click the 'Load More' button to load additional items
-                            yield loadMoreButton.dispatchEvent('click');
-                            yield page.waitForTimeout(2000); // Wait for new items to load
-                            // After clicking 'Load More', scroll down to load more items
+                            const selectorIndex = availableSelectors.indexOf(workingSelector);
+                            availableSelectors = availableSelectors.slice(selectorIndex);
+                            try {
+                                try {
+                                    yield loadMoreButton.click();
+                                }
+                                catch (error) {
+                                    yield loadMoreButton.dispatchEvent('click');
+                                }
+                            }
+                            catch (error) {
+                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                                allResults = allResults.concat(finalResults);
+                                return allResults;
+                            }
+                            yield page.waitForTimeout(2000);
                             yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
                             yield page.waitForTimeout(2000);
-                            // Check if more items are available
                             const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
                             if (currentHeight === previousHeight) {
-                                // No more items loaded, return the scraped results
+                                debugLog('No more items loaded after Load More');
                                 const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
                                 allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
                             previousHeight = currentHeight;
                             if (config.limit && allResults.length >= config.limit) {
-                                // If limit is set and reached, return the limited results
                                 allResults = allResults.slice(0, config.limit);
                                 break;
                             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",