maxun-core 0.0.30 → 0.0.31

package/build/interpret.js

@@ -1496,8 +1496,17 @@ class Interpreter extends events_1.EventEmitter {
                     scrapedItems.add(uniqueKey);
                     return true;
                 });
-                allResults = allResults.concat(newResults);
-                debugLog("Results collected:", allResults.length);
+                let itemsToAdd = newResults;
+                if (config.limit) {
+                    const remainingCapacity = config.limit - allResults.length;
+                    if (remainingCapacity <= 0) {
+                        itemsToAdd = [];
+                    }
+                    else if (newResults.length > remainingCapacity) {
+                        itemsToAdd = newResults.slice(0, remainingCapacity);
+                    }
+                }
+                allResults = allResults.concat(itemsToAdd);
                 this.serializableDataByType[actionType][actionName] = [...allResults];
                 yield this.options.serializableCallback({
                     scrapeList: this.serializableDataByType.scrapeList,
@@ -1635,11 +1644,38 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit()) {
                                 return allResults;
                             }
-                            yield page.evaluate(() => {
-                                const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
-                                window.scrollTo(0, scrollHeight);
-                            });
+                            const scrollIterations = 3;
+                            for (let i = 0; i < scrollIterations; i++) {
+                                yield page.evaluate(() => {
+                                    window.scrollBy(0, window.innerHeight * 0.8);
+                                });
+                                yield page.waitForTimeout(500);
+                            }
                             yield page.waitForTimeout(2000);
+                            try {
+                                yield page.evaluate((listSelector) => {
+                                    const isXPath = listSelector.startsWith('//') || listSelector.startsWith('/');
+                                    let lastElement = null;
+                                    if (isXPath) {
+                                        const result = document.evaluate(listSelector, document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+                                        if (result.snapshotLength > 0) {
+                                            lastElement = result.snapshotItem(result.snapshotLength - 1);
+                                        }
+                                    }
+                                    else {
+                                        const elements = document.querySelectorAll(listSelector);
+                                        if (elements.length > 0) {
+                                            lastElement = elements[elements.length - 1];
+                                        }
+                                    }
+                                    if (lastElement) {
+                                        lastElement.scrollIntoView({ behavior: 'smooth', block: 'end' });
+                                    }
+                                }, config.listSelector);
+                                yield page.waitForTimeout(1500);
+                            }
+                            catch (e) {
+                            }
                             const currentHeight = yield page.evaluate(() => {
                                 return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
                             });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.30",
+  "version": "0.0.31",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",