maxun-core 0.0.14 → 0.0.15

package/build/interpret.js

@@ -477,6 +477,7 @@ class Interpreter extends events_1.EventEmitter {
             let visitedUrls = new Set();
             const MAX_RETRIES = 3;
             const RETRY_DELAY = 1000; // 1 second delay between retries
+            const MAX_UNCHANGED_RESULTS = 5;
             const debugLog = (message, ...args) => {
                 console.log(`[Page ${visitedUrls.size}] [URL: ${page.url()}] ${message}`, ...args);
             };
@@ -555,30 +556,55 @@ class Interpreter extends events_1.EventEmitter {
                 }
             });
             let availableSelectors = config.pagination.selector.split(',');
+            let unchangedResultCounter = 0;
             try {
                 while (true) {
-                    // Reduced timeout for faster performance
-                    yield page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { });
                     switch (config.pagination.type) {
                         case 'scrollDown': {
+                            let previousResultCount = allResults.length;
+                            yield scrapeCurrentPage();
+                            if (checkLimit()) {
+                                return allResults;
+                            }
                             yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
                             yield page.waitForTimeout(2000);
                             const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                            const currentResultCount = allResults.length;
+                            if (currentResultCount === previousResultCount) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
+                                    return allResults;
+                                }
+                            }
+                            else {
+                                unchangedResultCounter = 0;
+                            }
                             if (currentHeight === previousHeight) {
-                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
                             previousHeight = currentHeight;
                             break;
                         }
                         case 'scrollUp': {
+                            let previousResultCount = allResults.length;
+                            yield scrapeCurrentPage();
+                            if (checkLimit()) {
+                                return allResults;
+                            }
                             yield page.evaluate(() => window.scrollTo(0, 0));
                             yield page.waitForTimeout(2000);
                             const currentTopHeight = yield page.evaluate(() => document.documentElement.scrollTop);
+                            const currentResultCount = allResults.length;
+                            if (currentResultCount === previousResultCount) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
+                                    return allResults;
+                                }
+                            }
+                            else {
+                                unchangedResultCounter = 0;
+                            }
                             if (currentTopHeight === 0) {
-                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
                             previousHeight = currentTopHeight;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.14",
+  "version": "0.0.15",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",