mx-cloud 0.0.17 → 0.0.18

package/build/interpret.d.ts

@@ -85,6 +85,10 @@ export default class Interpreter extends EventEmitter {
      * Sets the abort flag to immediately stop all operations
      */
     abort(): void;
+    /**
+     * Returns the current abort status
+     */
+    getIsAborted(): boolean;
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\

package/build/interpret.js

@@ -316,6 +316,12 @@ class Interpreter extends events_1.EventEmitter {
     abort() {
         this.isAborted = true;
     }
+    /**
+     * Returns the current abort status
+     */
+    getIsAborted() {
+        return this.isAborted;
+    }
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\
@@ -582,7 +588,15 @@ class Interpreter extends events_1.EventEmitter {
                         }
                         if (methodName === 'waitForLoadState') {
                             try {
-                                yield executeAction(invokee, methodName, step.args);
+                                // Add timeout if not already specified
+                                let args = step.args;
+                                if (Array.isArray(args) && args.length === 1) {
+                                    args = [args[0], { timeout: 30000 }];
+                                }
+                                else if (!Array.isArray(args)) {
+                                    args = [args, { timeout: 30000 }];
+                                }
+                                yield executeAction(invokee, methodName, args);
                             }
                             catch (error) {
                                 yield executeAction(invokee, methodName, 'domcontentloaded');
@@ -639,7 +653,17 @@ class Interpreter extends events_1.EventEmitter {
                     debugLog("Workflow aborted, stopping scrapeCurrentPage");
                     return;
                 }
-                const results = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                // Add timeout to prevent hanging on page evaluation
+                const evaluationPromise = page.evaluate((cfg) => window.scrapeList(cfg), config);
+                const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Page evaluation timeout')), 10000));
+                let results;
+                try {
+                    results = yield Promise.race([evaluationPromise, timeoutPromise]);
+                }
+                catch (error) {
+                    debugLog(`Page evaluation failed: ${error.message}`);
+                    return;
+                }
                 const newResults = results.filter(item => {
                     const uniqueKey = JSON.stringify(item);
                     if (scrapedItems.has(uniqueKey))
@@ -696,14 +720,22 @@ class Interpreter extends events_1.EventEmitter {
             });
             // Enhanced button finder with retry mechanism for both CSS and XPath selectors
             const findWorkingButton = (selectors) => __awaiter(this, void 0, void 0, function* () {
+                const startTime = Date.now();
+                const MAX_BUTTON_SEARCH_TIME = 15000;
                 let updatedSelectors = [...selectors];
                 for (let i = 0; i < selectors.length; i++) {
+                    // Check overall timeout
+                    if (Date.now() - startTime > MAX_BUTTON_SEARCH_TIME) {
+                        debugLog(`Button search timeout reached (${MAX_BUTTON_SEARCH_TIME}ms), aborting`);
+                        break;
+                    }
                     const selector = selectors[i];
                     let retryCount = 0;
                     let selectorSuccess = false;
                     while (retryCount < MAX_RETRIES && !selectorSuccess) {
                         try {
-                            const button = yield waitForSelectorUniversal(selector, { timeout: 10000 });
+                            // Reduce timeout to prevent hanging on slow selectors
+                            const button = yield waitForSelectorUniversal(selector, { timeout: 2000 });
                             if (button) {
                                 debugLog('Found working selector:', selector);
                                 return {
@@ -712,16 +744,30 @@ class Interpreter extends events_1.EventEmitter {
                                     updatedSelectors
                                 };
                             }
+                            else {
+                                // Treat null result as failed attempt
+                                retryCount++;
+                                debugLog(`Selector "${selector}" not found: attempt ${retryCount}/${MAX_RETRIES}`);
+                                if (retryCount < MAX_RETRIES) {
+                                    yield page.waitForTimeout(RETRY_DELAY);
+                                }
+                                else {
+                                    debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
+                                    updatedSelectors = updatedSelectors.filter(s => s !== selector);
+                                    selectorSuccess = true; // Exit retry loop for this selector
+                                }
+                            }
                         }
                         catch (error) {
                             retryCount++;
-                            debugLog(`Selector "${selector}" failed: attempt ${retryCount}/${MAX_RETRIES}`);
+                            debugLog(`Selector "${selector}" error: attempt ${retryCount}/${MAX_RETRIES} - ${error.message}`);
                             if (retryCount < MAX_RETRIES) {
                                 yield page.waitForTimeout(RETRY_DELAY);
                             }
                             else {
                                 debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
                                 updatedSelectors = updatedSelectors.filter(s => s !== selector);
+                                selectorSuccess = true; // Exit retry loop for this selector
                             }
                         }
                     }
@@ -748,6 +794,10 @@ class Interpreter extends events_1.EventEmitter {
             });
             let availableSelectors = config.pagination.selector.split(',');
             let unchangedResultCounter = 0;
+            let paginationIterations = 0;
+            const MAX_PAGINATION_ITERATIONS = 100; // Prevent infinite pagination
+            const paginationStartTime = Date.now();
+            const MAX_PAGINATION_TIME = 30 * 60 * 1000; // 30 minutes max for pagination
             try {
                 while (true) {
                     // Check abort flag at start of each pagination iteration
@@ -755,6 +805,19 @@ class Interpreter extends events_1.EventEmitter {
                         this.log('Workflow aborted during pagination loop', logger_1.Level.WARN);
                         return allResults;
                     }
+                    // Pagination circuit breakers
+                    if (++paginationIterations > MAX_PAGINATION_ITERATIONS) {
+                        debugLog(`Maximum pagination iterations reached (${MAX_PAGINATION_ITERATIONS}), stopping`);
+                        return allResults;
+                    }
+                    if (Date.now() - paginationStartTime > MAX_PAGINATION_TIME) {
+                        debugLog('Maximum pagination time reached (10 minutes), stopping');
+                        return allResults;
+                    }
+                    // Add async yield every 5 iterations to prevent event loop blocking
+                    if (paginationIterations % 5 === 0) {
+                        yield new Promise(resolve => setImmediate(resolve));
+                    }
                     switch (config.pagination.type) {
                         case 'scrollDown': {
                             let previousResultCount = allResults.length;
@@ -971,10 +1034,23 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit())
                                 return allResults;
                             let loadMoreCounter = 0;
-                            // let previousResultCount = allResults.length;
-                            // let noNewItemsCounter = 0;
-                            // const MAX_NO_NEW_ITEMS = 2;
+                            const MAX_LOAD_MORE_ITERATIONS = 100; // Prevent infinite load more
+                            const loadMoreStartTime = Date.now();
+                            const MAX_LOAD_MORE_TIME = 30 * 60 * 1000; // 5 minutes max for load more
                             while (true) {
+                                // Load more circuit breakers
+                                if (loadMoreCounter >= MAX_LOAD_MORE_ITERATIONS) {
+                                    debugLog(`Maximum load more iterations reached (${MAX_LOAD_MORE_ITERATIONS}), stopping`);
+                                    return allResults;
+                                }
+                                if (Date.now() - loadMoreStartTime > MAX_LOAD_MORE_TIME) {
+                                    debugLog('Maximum load more time reached (5 minutes), stopping');
+                                    return allResults;
+                                }
+                                // Add async yield every 3 iterations
+                                if (loadMoreCounter % 3 === 0 && loadMoreCounter > 0) {
+                                    yield new Promise(resolve => setImmediate(resolve));
+                                }
                                 // Find working button with retry mechanism
                                 const { button: loadMoreButton, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
                                 availableSelectors = updatedSelectors;
@@ -1730,12 +1806,35 @@ class Interpreter extends events_1.EventEmitter {
                     selector.includes('@id=');
                 let count = 0;
                 if (isXPath) {
+                    // Add timeout to prevent XPath hanging
                     const locator = page.locator(`xpath=${selector}`);
-                    count = yield locator.count();
+                    const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('XPath timeout')), 5000));
+                    try {
+                        count = yield Promise.race([
+                            locator.count(),
+                            timeoutPromise
+                        ]);
+                    }
+                    catch (error) {
+                        // XPath timed out or failed
+                        return false;
+                    }
                 }
                 else {
-                    const elements = yield page.$$(selector);
-                    count = elements ? elements.length : 0;
+                    // Add timeout to CSS selector operations
+                    try {
+                        const elementsPromise = page.$$(selector);
+                        const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('CSS selector timeout')), 5000));
+                        const elements = yield Promise.race([
+                            elementsPromise,
+                            timeoutPromise
+                        ]);
+                        count = elements ? elements.length : 0;
+                    }
+                    catch (error) {
+                        // CSS selector timed out or failed
+                        return false;
+                    }
                 }
                 // For list selectors, we need multiple elements
                 if (isListSelector) {
@@ -1998,12 +2097,26 @@ class Interpreter extends events_1.EventEmitter {
             /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
             let loopIterations = 0;
             const MAX_LOOP_ITERATIONS = 1000; // Circuit breaker
+            let consecutiveFailures = 0;
+            const MAX_CONSECUTIVE_FAILURES = 10;
+            const startTime = Date.now();
+            const MAX_EXECUTION_TIME = 30 * 60 * 1000; // 30 minutes max
             while (true) {
-                // Circuit breaker to prevent infinite loops
+                // Multiple circuit breakers to prevent infinite loops
                 if (++loopIterations > MAX_LOOP_ITERATIONS) {
                     this.log('Maximum loop iterations reached, terminating to prevent infinite loop', logger_1.Level.ERROR);
                     return;
                 }
+                // Time-based circuit breaker
+                if (Date.now() - startTime > MAX_EXECUTION_TIME) {
+                    this.log('Maximum execution time reached (30 minutes), terminating workflow', logger_1.Level.ERROR);
+                    return;
+                }
+                // Failure-based circuit breaker
+                if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+                    this.log('Too many consecutive failures, terminating to prevent hang', logger_1.Level.ERROR);
+                    return;
+                }
                 // Check abort flag immediately
                 if (this.isAborted) {
                     this.log('Workflow aborted in runLoop', logger_1.Level.WARN);
@@ -2094,11 +2207,19 @@ class Interpreter extends events_1.EventEmitter {
                         usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
                         workflowCopy.splice(actionId, 1);
                         console.log(`Action with ID ${action.id} removed from the workflow copy.`);
-                        // Reset loop iteration counter on successful action
-                        loopIterations = 0;
+                        // Reset counters on successful action (but keep some history to prevent infinite resets)
+                        loopIterations = Math.max(0, loopIterations - 10);
+                        consecutiveFailures = 0;
+                        // Add async yield to prevent event loop blocking
+                        if (loopIterations % 10 === 0) {
+                            yield new Promise(resolve => setImmediate(resolve));
+                        }
                     }
                     catch (e) {
                         this.log(e, logger_1.Level.ERROR);
+                        consecutiveFailures++;
+                        // Add delay on failures to prevent tight error loops
+                        yield new Promise(resolve => setTimeout(resolve, Math.min(1000, consecutiveFailures * 200)));
                         // Don't crash on individual action failures - continue with next iteration
                         continue;
                     }
@@ -2112,9 +2233,32 @@ class Interpreter extends events_1.EventEmitter {
     }
     ensureScriptsLoaded(page) {
         return __awaiter(this, void 0, void 0, function* () {
-            const isScriptLoaded = yield page.evaluate(() => typeof window.scrape === 'function' && typeof window.scrapeSchema === 'function' && typeof window.scrapeList === 'function' && typeof window.scrapeListAuto === 'function' && typeof window.scrollDown === 'function' && typeof window.scrollUp === 'function');
-            if (!isScriptLoaded) {
-                yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+            try {
+                // Add timeout to prevent hanging on script evaluation
+                const evaluationPromise = page.evaluate(() => typeof window.scrape === 'function' &&
+                    typeof window.scrapeSchema === 'function' &&
+                    typeof window.scrapeList === 'function' &&
+                    typeof window.scrapeListAuto === 'function' &&
+                    typeof window.scrollDown === 'function' &&
+                    typeof window.scrollUp === 'function');
+                const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Script check timeout')), 3000));
+                const isScriptLoaded = yield Promise.race([
+                    evaluationPromise,
+                    timeoutPromise
+                ]);
+                if (!isScriptLoaded) {
+                    yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+                }
+            }
+            catch (error) {
+                // If script check fails, try to add the script anyway
+                this.log(`Script check failed, adding script anyway: ${error.message}`, logger_1.Level.WARN);
+                try {
+                    yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+                }
+                catch (scriptError) {
+                    this.log(`Failed to add script: ${scriptError.message}`, logger_1.Level.ERROR);
+                }
             }
         });
     }

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.17",
+  "version": "0.0.18",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",
   "scripts": {
     "test": "jest",
-    "build": "tsc",
+    "build": "npm run clean && tsc",
     "lint": "eslint .",
     "clean": "rimraf ./build"
   },
@@ -23,8 +23,5 @@
     "playwright": "^1.50.0",
     "playwright-extra": "^4.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2"
-  },
-  "devDependencies": {
-    "@types/node": "^24.3.1"
   }
 }