maxun-core 0.0.16 → 0.0.18

package/build/browserSide/scraper.js

@@ -636,6 +636,26 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     return similarity >= similarityThreshold;
                 });
             }
+            function tryFallbackSelector(rootElement, originalSelector) {
+                let element = queryElement(rootElement, originalSelector);
+                if (!element && originalSelector.includes('nth-child')) {
+                    const match = originalSelector.match(/nth-child\((\d+)\)/);
+                    if (match) {
+                        const position = parseInt(match[1], 10);
+                        for (let i = position - 1; i >= 1; i--) {
+                            const fallbackSelector = originalSelector.replace(/nth-child\(\d+\)/, `nth-child(${i})`);
+                            element = queryElement(rootElement, fallbackSelector);
+                            if (element)
+                                break;
+                        }
+                        if (!element) {
+                            const baseSelector = originalSelector.replace(/\:nth-child\(\d+\)/, '');
+                            element = queryElement(rootElement, baseSelector);
+                        }
+                    }
+                }
+                return element;
+            }
             // Main scraping logic with context support
             let containers = queryElementAll(document, listSelector);
             containers = Array.from(containers);
@@ -786,7 +806,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
                         // Get the last part of the selector after any context delimiter
                         const relativeSelector = selector.split(/(?:>>|:>>)/).slice(-1)[0];
-                        const element = queryElement(container, relativeSelector);
+                        const element = tryFallbackSelector(container, relativeSelector);
                         if (element) {
                             record[label] = extractValue(element, attribute);
                         }

package/build/interpret.d.ts

@@ -41,6 +41,7 @@ interface InterpreterOptions {
         activeId: (id: number) => void;
         debugMessage: (msg: string) => void;
         setActionType: (type: string) => void;
+        incrementScrapeListIndex: () => void;
     }>;
 }
 /**

package/build/interpret.js

@@ -385,7 +385,7 @@ class Interpreter extends events_1.EventEmitter {
                     yield this.options.serializableCallback([mergedResult]);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
-                    var _f;
+                    var _f, _g;
                     if ((_f = this.options.debugChannel) === null || _f === void 0 ? void 0 : _f.setActionType) {
                         this.options.debugChannel.setActionType('scrapeList');
                     }
@@ -394,6 +394,9 @@ class Interpreter extends events_1.EventEmitter {
                         return;
                     }
                     yield this.ensureScriptsLoaded(page);
+                    if ((_g = this.options.debugChannel) === null || _g === void 0 ? void 0 : _g.incrementScrapeListIndex) {
+                        this.options.debugChannel.incrementScrapeListIndex();
+                    }
                     if (!config.pagination) {
                         const scrapeResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
                         yield this.options.serializableCallback(scrapeResults);
@@ -404,8 +407,8 @@ class Interpreter extends events_1.EventEmitter {
                     }
                 }),
                 scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
-                    var _g;
-                    if ((_g = this.options.debugChannel) === null || _g === void 0 ? void 0 : _g.setActionType) {
+                    var _h;
+                    if ((_h = this.options.debugChannel) === null || _h === void 0 ? void 0 : _h.setActionType) {
                         this.options.debugChannel.setActionType('scrapeListAuto');
                     }
                     yield this.ensureScriptsLoaded(page);
@@ -415,8 +418,8 @@ class Interpreter extends events_1.EventEmitter {
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scroll: (pages) => __awaiter(this, void 0, void 0, function* () {
-                    var _h;
-                    if ((_h = this.options.debugChannel) === null || _h === void 0 ? void 0 : _h.setActionType) {
+                    var _j;
+                    if ((_j = this.options.debugChannel) === null || _j === void 0 ? void 0 : _j.setActionType) {
                         this.options.debugChannel.setActionType('scroll');
                     }
                     yield page.evaluate((pagesInternal) => __awaiter(this, void 0, void 0, function* () {
@@ -427,8 +430,8 @@ class Interpreter extends events_1.EventEmitter {
                     }), pages !== null && pages !== void 0 ? pages : 1);
                 }),
                 script: (code) => __awaiter(this, void 0, void 0, function* () {
-                    var _j;
-                    if ((_j = this.options.debugChannel) === null || _j === void 0 ? void 0 : _j.setActionType) {
+                    var _k;
+                    if ((_k = this.options.debugChannel) === null || _k === void 0 ? void 0 : _k.setActionType) {
                         this.options.debugChannel.setActionType('script');
                     }
                     const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
@@ -530,6 +533,7 @@ class Interpreter extends events_1.EventEmitter {
                 });
                 allResults = allResults.concat(newResults);
                 debugLog("Results collected:", allResults.length);
+                yield this.options.serializableCallback(allResults);
             });
             const checkLimit = () => {
                 if (config.limit && allResults.length >= config.limit) {
@@ -700,6 +704,7 @@ class Interpreter extends events_1.EventEmitter {
                                             button.click()
                                         ]);
                                         debugLog("Navigation successful after regular click");
+                                        yield page.waitForTimeout(2000);
                                         paginationSuccess = true;
                                     }
                                     catch (navError) {
@@ -715,6 +720,7 @@ class Interpreter extends events_1.EventEmitter {
                                                 button.dispatchEvent('click')
                                             ]);
                                             debugLog("Navigation successful after dispatch event");
+                                            yield page.waitForTimeout(2000);
                                             paginationSuccess = true;
                                         }
                                         catch (dispatchNavError) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.16",
+  "version": "0.0.18",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",