maxun-core 0.0.6 → 0.0.7

package/build/browserSide/scraper.js

@@ -220,8 +220,27 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
         return __awaiter(this, void 0, void 0, function* () {
             const scrapedData = [];
             while (scrapedData.length < limit) {
-                // Get all parent elements matching the listSelector
-                const parentElements = Array.from(document.querySelectorAll(listSelector));
+                let parentElements = Array.from(document.querySelectorAll(listSelector));
+                // If we only got one element or none, try a more generic approach
+                if (limit > 1 && parentElements.length <= 1) {
+                    const [containerSelector, _] = listSelector.split('>').map(s => s.trim());
+                    const container = document.querySelector(containerSelector);
+                    if (container) {
+                        const allChildren = Array.from(container.children);
+                        const firstMatch = document.querySelector(listSelector);
+                        if (firstMatch) {
+                            // Get classes from the first matching element
+                            const firstMatchClasses = Array.from(firstMatch.classList);
+                            // Find similar elements by matching most of their classes
+                            parentElements = allChildren.filter(element => {
+                                const elementClasses = Array.from(element.classList);
+                                // Element should share at least 70% of classes with the first match
+                                const commonClasses = firstMatchClasses.filter(cls => elementClasses.includes(cls));
+                                return commonClasses.length >= Math.floor(firstMatchClasses.length * 0.7);
+                            });
+                        }
+                    }
+                }
                 // Iterate through each parent element
                 for (const parent of parentElements) {
                     if (scrapedData.length >= limit)
@@ -254,6 +273,11 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     }
                     scrapedData.push(record);
                 }
+                // If we've processed all available elements and still haven't reached the limit,
+                // break to avoid infinite loop
+                if (parentElements.length === 0 || scrapedData.length >= parentElements.length) {
+                    break;
+                }
             }
             return scrapedData;
         });

package/build/interpret.js

@@ -84,14 +84,24 @@ class Interpreter extends events_1.EventEmitter {
     applyAdBlocker(page) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.blocker) {
-                yield this.blocker.enableBlockingInPage(page);
+                try {
+                    yield this.blocker.enableBlockingInPage(page);
+                }
+                catch (err) {
+                    this.log(`Ad-blocker operation failed:`, logger_1.Level.ERROR);
+                }
             }
         });
     }
     disableAdBlocker(page) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.blocker) {
-                yield this.blocker.disableBlockingInPage(page);
+                try {
+                    yield this.blocker.disableBlockingInPage(page);
+                }
+                catch (err) {
+                    this.log(`Ad-blocker operation failed:`, logger_1.Level.ERROR);
+                }
             }
         });
     }
@@ -156,8 +166,8 @@ class Interpreter extends events_1.EventEmitter {
             // const actionable = async (selector: string): Promise<boolean> => {
             //   try {
             //     const proms = [
-            //       page.isEnabled(selector, { timeout: 5000 }),
-            //       page.isVisible(selector, { timeout: 5000 }),
+            //       page.isEnabled(selector, { timeout: 10000 }),
+            //       page.isVisible(selector, { timeout: 10000 }),
             //     ];
             //     return await Promise.all(proms).then((bools) => bools.every((x) => x));
             //   } catch (e) {
@@ -176,6 +186,15 @@ class Interpreter extends events_1.EventEmitter {
             //     return [];
             //   }),
             // ).then((x) => x.flat());
+            const presentSelectors = yield Promise.all(selectors.map((selector) => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield page.waitForSelector(selector, { state: 'attached' });
+                    return [selector];
+                }
+                catch (e) {
+                    return [];
+                }
+            }))).then((x) => x.flat());
             const action = workflowCopy[workflowCopy.length - 1];
             // console.log("Next action:", action)
             let url = page.url();
@@ -186,7 +205,7 @@ class Interpreter extends events_1.EventEmitter {
                 url,
                 cookies: (yield page.context().cookies([page.url()]))
                     .reduce((p, cookie) => (Object.assign(Object.assign({}, p), { [cookie.name]: cookie.value })), {}),
-                selectors,
+                selectors: presentSelectors,
             };
         });
     }
@@ -420,7 +439,12 @@ class Interpreter extends events_1.EventEmitter {
                             yield executeAction(invokee, methodName, step.args);
                         }
                         catch (error) {
-                            yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                            try {
+                                yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                            }
+                            catch (error) {
+                                continue;
+                            }
                         }
                     }
                     else {
@@ -549,7 +573,12 @@ class Interpreter extends events_1.EventEmitter {
         return __awaiter(this, void 0, void 0, function* () {
             const workflowCopy = JSON.parse(JSON.stringify(workflow));
             // apply ad-blocker to the current page
-            yield this.applyAdBlocker(p);
+            try {
+                yield this.applyAdBlocker(p);
+            }
+            catch (error) {
+                this.log(`Failed to apply ad-blocker: ${error.message}`, logger_1.Level.ERROR);
+            }
             const usedActions = [];
             let selectors = [];
             let lastAction = null;
@@ -660,6 +689,7 @@ class Interpreter extends events_1.EventEmitter {
         return __awaiter(this, void 0, void 0, function* () {
             this.log('Starting the workflow.', logger_1.Level.LOG);
             const context = page.context();
+            page.setDefaultNavigationTimeout(100000);
             // Check proxy settings from context options
             const contextOptions = context._options;
             const hasProxy = !!(contextOptions === null || contextOptions === void 0 ? void 0 : contextOptions.proxy);

package/build/utils/concurrency.d.ts

@@ -3,43 +3,43 @@
  */
 export default class Concurrency {
     /**
-       * Maximum number of workers running in parallel. If set to `null`, there is no limit.
-       */
+    * Maximum number of workers running in parallel. If set to `null`, there is no limit.
+    */
     maxConcurrency: number;
     /**
-       * Number of currently active workers.
-       */
+    * Number of currently active workers.
+    */
     activeWorkers: number;
     /**
-       * Queue of jobs waiting to be completed.
-       */
+    * Queue of jobs waiting to be completed.
+    */
     private jobQueue;
     /**
-       * "Resolve" callbacks of the waitForCompletion() promises.
-       */
+    * "Resolve" callbacks of the waitForCompletion() promises.
+    */
     private waiting;
     /**
-       * Constructs a new instance of concurrency manager.
-       * @param {number} maxConcurrency Maximum number of workers running in parallel.
-       */
+    * Constructs a new instance of concurrency manager.
+    * @param {number} maxConcurrency Maximum number of workers running in parallel.
+    */
     constructor(maxConcurrency: number);
     /**
-       * Takes a waiting job out of the queue and runs it.
-       */
+    * Takes a waiting job out of the queue and runs it.
+    */
     private runNextJob;
     /**
-       * Pass a job (a time-demanding async function) to the concurrency manager. \
-       * The time of the job's execution depends on the concurrency manager itself
-       * (given a generous enough `maxConcurrency` value, it might be immediate,
-       * but this is not guaranteed).
-       * @param worker Async function to be executed (job to be processed).
-       */
+    * Pass a job (a time-demanding async function) to the concurrency manager. \
+    * The time of the job's execution depends on the concurrency manager itself
+    * (given a generous enough `maxConcurrency` value, it might be immediate,
+    * but this is not guaranteed).
+    * @param worker Async function to be executed (job to be processed).
+    */
     addJob(job: () => Promise<any>): void;
     /**
-       * Waits until there is no running nor waiting job. \
-       * If the concurrency manager is idle at the time of calling this function,
-       * it waits until at least one job is completed (can be "presubscribed").
-       * @returns Promise, resolved after there is no running/waiting worker.
-       */
+    * Waits until there is no running nor waiting job. \
+    * If the concurrency manager is idle at the time of calling this function,
+    * it waits until at least one job is completed (can be "presubscribed").
+    * @returns Promise, resolved after there is no running/waiting worker.
+    */
     waitForCompletion(): Promise<void>;
 }

package/build/utils/concurrency.js

@@ -5,31 +5,31 @@ Object.defineProperty(exports, "__esModule", { value: true });
  */
 class Concurrency {
     /**
-       * Constructs a new instance of concurrency manager.
-       * @param {number} maxConcurrency Maximum number of workers running in parallel.
-       */
+    * Constructs a new instance of concurrency manager.
+    * @param {number} maxConcurrency Maximum number of workers running in parallel.
+    */
     constructor(maxConcurrency) {
         /**
-           * Maximum number of workers running in parallel. If set to `null`, there is no limit.
-           */
+        * Maximum number of workers running in parallel. If set to `null`, there is no limit.
+        */
         this.maxConcurrency = 1;
         /**
-           * Number of currently active workers.
-           */
+        * Number of currently active workers.
+        */
         this.activeWorkers = 0;
         /**
-           * Queue of jobs waiting to be completed.
-           */
+        * Queue of jobs waiting to be completed.
+        */
         this.jobQueue = [];
         /**
-           * "Resolve" callbacks of the waitForCompletion() promises.
-           */
+        * "Resolve" callbacks of the waitForCompletion() promises.
+        */
         this.waiting = [];
         this.maxConcurrency = maxConcurrency;
     }
     /**
-       * Takes a waiting job out of the queue and runs it.
-       */
+    * Takes a waiting job out of the queue and runs it.
+    */
     runNextJob() {
         const job = this.jobQueue.pop();
         if (job) {
@@ -49,12 +49,12 @@ class Concurrency {
         }
     }
     /**
-       * Pass a job (a time-demanding async function) to the concurrency manager. \
-       * The time of the job's execution depends on the concurrency manager itself
-       * (given a generous enough `maxConcurrency` value, it might be immediate,
-       * but this is not guaranteed).
-       * @param worker Async function to be executed (job to be processed).
-       */
+    * Pass a job (a time-demanding async function) to the concurrency manager. \
+    * The time of the job's execution depends on the concurrency manager itself
+    * (given a generous enough `maxConcurrency` value, it might be immediate,
+    * but this is not guaranteed).
+    * @param worker Async function to be executed (job to be processed).
+    */
     addJob(job) {
         // console.debug("Adding a worker!");
         this.jobQueue.push(job);
@@ -67,11 +67,11 @@ class Concurrency {
         }
     }
     /**
-       * Waits until there is no running nor waiting job. \
-       * If the concurrency manager is idle at the time of calling this function,
-       * it waits until at least one job is completed (can be "presubscribed").
-       * @returns Promise, resolved after there is no running/waiting worker.
-       */
+    * Waits until there is no running nor waiting job. \
+    * If the concurrency manager is idle at the time of calling this function,
+    * it waits until at least one job is completed (can be "presubscribed").
+    * @returns Promise, resolved after there is no running/waiting worker.
+    */
     waitForCompletion() {
         return new Promise((res) => {
             this.waiting.push(res);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",