mx-cloud 0.0.16 → 0.0.18

package/build/interpret.d.ts

@@ -54,6 +54,7 @@ export default class Interpreter extends EventEmitter {
     private options;
     private concurrency;
     private stopper;
+    private isAborted;
     private log;
     private blocker;
     private cumulativeResults;
@@ -80,6 +81,14 @@ export default class Interpreter extends EventEmitter {
      * @returns True if `where` is applicable in the given context, false otherwise
      */
     private applicable;
+    /**
+     * Sets the abort flag to immediately stop all operations
+     */
+    abort(): void;
+    /**
+     * Returns the current abort status
+     */
+    getIsAborted(): boolean;
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\

package/build/interpret.js

@@ -63,6 +63,7 @@ class Interpreter extends events_1.EventEmitter {
         var _a;
         super();
         this.stopper = null;
+        this.isAborted = false;
         this.blocker = null;
         this.cumulativeResults = [];
         this.autohealFailures = [];
@@ -90,7 +91,9 @@ class Interpreter extends events_1.EventEmitter {
         adblocker_playwright_1.PlaywrightBlocker.fromLists(cross_fetch_1.default, ['https://easylist.to/easylist/easylist.txt']).then(blocker => {
             this.blocker = blocker;
         }).catch(err => {
-            this.log(`Failed to initialize ad-blocker:`, logger_1.Level.ERROR);
+            this.log(`Failed to initialize ad-blocker: ${err.message}`, logger_1.Level.ERROR);
+            // Continue without ad-blocker rather than crashing
+            this.blocker = null;
         });
     }
     trackAutohealFailure(error) {
@@ -307,6 +310,18 @@ class Interpreter extends events_1.EventEmitter {
             }
         });
     }
+    /**
+     * Sets the abort flag to immediately stop all operations
+     */
+    abort() {
+        this.isAborted = true;
+    }
+    /**
+     * Returns the current abort status
+     */
+    getIsAborted() {
+        return this.isAborted;
+    }
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\
@@ -319,6 +334,11 @@ class Interpreter extends events_1.EventEmitter {
     carryOutSteps(page, steps) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b;
+            // Check abort flag at start of execution
+            if (this.isAborted) {
+                this.log('Workflow aborted, stopping execution', logger_1.Level.WARN);
+                return;
+            }
             /**
              * Defines overloaded (or added) methods/actions usable in the workflow.
              * If a method overloads any existing method of the Page class, it accepts the same set
@@ -376,6 +396,11 @@ class Interpreter extends events_1.EventEmitter {
                 }),
                 scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
                     var _a;
+                    // Check abort flag at start of scraping
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping scrapeSchema', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                         this.options.debugChannel.setActionType('scrapeSchema');
                     }
@@ -429,6 +454,11 @@ class Interpreter extends events_1.EventEmitter {
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
                     var _a, _b;
+                    // Check abort flag at start of scraping
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping scrapeList', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                         this.options.debugChannel.setActionType('scrapeList');
                     }
@@ -497,9 +527,15 @@ class Interpreter extends events_1.EventEmitter {
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                         this.options.debugChannel.setActionType('script');
                     }
-                    const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
-                    const x = new AsyncFunction('page', 'log', code);
-                    yield x(page, this.log);
+                    try {
+                        const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
+                        const x = new AsyncFunction('page', 'log', code);
+                        yield x(page, this.log);
+                    }
+                    catch (error) {
+                        this.log(`Script execution failed: ${error.message}`, logger_1.Level.ERROR);
+                        throw new Error(`Script execution error: ${error.message}`);
+                    }
                 }),
                 flag: () => __awaiter(this, void 0, void 0, function* () {
                     return new Promise((res) => {
@@ -527,6 +563,11 @@ class Interpreter extends events_1.EventEmitter {
                 }
             });
             for (const step of steps) {
+                // Check abort flag before each step
+                if (this.isAborted) {
+                    this.log('Workflow aborted during step execution', logger_1.Level.WARN);
+                    return;
+                }
                 this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
                 try {
                     if (step.action in wawActions) {
@@ -547,7 +588,15 @@ class Interpreter extends events_1.EventEmitter {
                         }
                         if (methodName === 'waitForLoadState') {
                             try {
-                                yield executeAction(invokee, methodName, step.args);
+                                // Add timeout if not already specified
+                                let args = step.args;
+                                if (Array.isArray(args) && args.length === 1) {
+                                    args = [args[0], { timeout: 30000 }];
+                                }
+                                else if (!Array.isArray(args)) {
+                                    args = [args, { timeout: 30000 }];
+                                }
+                                yield executeAction(invokee, methodName, args);
                             }
                             catch (error) {
                                 yield executeAction(invokee, methodName, 'domcontentloaded');
@@ -583,6 +632,11 @@ class Interpreter extends events_1.EventEmitter {
     }
     handlePagination(page, config) {
         return __awaiter(this, void 0, void 0, function* () {
+            // Check abort flag at start of pagination
+            if (this.isAborted) {
+                this.log('Workflow aborted, stopping pagination', logger_1.Level.WARN);
+                return [];
+            }
             let allResults = [];
             let previousHeight = 0;
             let scrapedItems = new Set();
@@ -594,7 +648,22 @@ class Interpreter extends events_1.EventEmitter {
                 console.log(`[Page ${visitedUrls.size}] [URL: ${page.url()}] ${message}`, ...args);
             };
             const scrapeCurrentPage = () => __awaiter(this, void 0, void 0, function* () {
-                const results = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                // Check abort flag before scraping current page
+                if (this.isAborted) {
+                    debugLog("Workflow aborted, stopping scrapeCurrentPage");
+                    return;
+                }
+                // Add timeout to prevent hanging on page evaluation
+                const evaluationPromise = page.evaluate((cfg) => window.scrapeList(cfg), config);
+                const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Page evaluation timeout')), 10000));
+                let results;
+                try {
+                    results = yield Promise.race([evaluationPromise, timeoutPromise]);
+                }
+                catch (error) {
+                    debugLog(`Page evaluation failed: ${error.message}`);
+                    return;
+                }
                 const newResults = results.filter(item => {
                     const uniqueKey = JSON.stringify(item);
                     if (scrapedItems.has(uniqueKey))
@@ -651,14 +720,22 @@ class Interpreter extends events_1.EventEmitter {
             });
             // Enhanced button finder with retry mechanism for both CSS and XPath selectors
             const findWorkingButton = (selectors) => __awaiter(this, void 0, void 0, function* () {
+                const startTime = Date.now();
+                const MAX_BUTTON_SEARCH_TIME = 15000;
                 let updatedSelectors = [...selectors];
                 for (let i = 0; i < selectors.length; i++) {
+                    // Check overall timeout
+                    if (Date.now() - startTime > MAX_BUTTON_SEARCH_TIME) {
+                        debugLog(`Button search timeout reached (${MAX_BUTTON_SEARCH_TIME}ms), aborting`);
+                        break;
+                    }
                     const selector = selectors[i];
                     let retryCount = 0;
                     let selectorSuccess = false;
                     while (retryCount < MAX_RETRIES && !selectorSuccess) {
                         try {
-                            const button = yield waitForSelectorUniversal(selector, { timeout: 10000 });
+                            // Reduce timeout to prevent hanging on slow selectors
+                            const button = yield waitForSelectorUniversal(selector, { timeout: 2000 });
                             if (button) {
                                 debugLog('Found working selector:', selector);
                                 return {
@@ -667,16 +744,30 @@ class Interpreter extends events_1.EventEmitter {
                                     updatedSelectors
                                 };
                             }
+                            else {
+                                // Treat null result as failed attempt
+                                retryCount++;
+                                debugLog(`Selector "${selector}" not found: attempt ${retryCount}/${MAX_RETRIES}`);
+                                if (retryCount < MAX_RETRIES) {
+                                    yield page.waitForTimeout(RETRY_DELAY);
+                                }
+                                else {
+                                    debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
+                                    updatedSelectors = updatedSelectors.filter(s => s !== selector);
+                                    selectorSuccess = true; // Exit retry loop for this selector
+                                }
+                            }
                         }
                         catch (error) {
                             retryCount++;
-                            debugLog(`Selector "${selector}" failed: attempt ${retryCount}/${MAX_RETRIES}`);
+                            debugLog(`Selector "${selector}" error: attempt ${retryCount}/${MAX_RETRIES} - ${error.message}`);
                             if (retryCount < MAX_RETRIES) {
                                 yield page.waitForTimeout(RETRY_DELAY);
                             }
                             else {
                                 debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
                                 updatedSelectors = updatedSelectors.filter(s => s !== selector);
+                                selectorSuccess = true; // Exit retry loop for this selector
                             }
                         }
                     }
@@ -703,8 +794,30 @@ class Interpreter extends events_1.EventEmitter {
             });
             let availableSelectors = config.pagination.selector.split(',');
             let unchangedResultCounter = 0;
+            let paginationIterations = 0;
+            const MAX_PAGINATION_ITERATIONS = 100; // Prevent infinite pagination
+            const paginationStartTime = Date.now();
+            const MAX_PAGINATION_TIME = 30 * 60 * 1000; // 30 minutes max for pagination
             try {
                 while (true) {
+                    // Check abort flag at start of each pagination iteration
+                    if (this.isAborted) {
+                        this.log('Workflow aborted during pagination loop', logger_1.Level.WARN);
+                        return allResults;
+                    }
+                    // Pagination circuit breakers
+                    if (++paginationIterations > MAX_PAGINATION_ITERATIONS) {
+                        debugLog(`Maximum pagination iterations reached (${MAX_PAGINATION_ITERATIONS}), stopping`);
+                        return allResults;
+                    }
+                    if (Date.now() - paginationStartTime > MAX_PAGINATION_TIME) {
+                        debugLog('Maximum pagination time reached (10 minutes), stopping');
+                        return allResults;
+                    }
+                    // Add async yield every 5 iterations to prevent event loop blocking
+                    if (paginationIterations % 5 === 0) {
+                        yield new Promise(resolve => setImmediate(resolve));
+                    }
                     switch (config.pagination.type) {
                         case 'scrollDown': {
                             let previousResultCount = allResults.length;
@@ -712,9 +825,14 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit()) {
                                 return allResults;
                             }
-                            yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                            yield page.evaluate(() => {
+                                const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                window.scrollTo(0, scrollHeight);
+                            });
                             yield page.waitForTimeout(2000);
-                            const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                            const currentHeight = yield page.evaluate(() => {
+                                return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                            });
                             const currentResultCount = allResults.length;
                             if (currentResultCount === previousResultCount) {
                                 unchangedResultCounter++;
@@ -916,10 +1034,23 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit())
                                 return allResults;
                             let loadMoreCounter = 0;
-                            // let previousResultCount = allResults.length;
-                            // let noNewItemsCounter = 0;
-                            // const MAX_NO_NEW_ITEMS = 2;
+                            const MAX_LOAD_MORE_ITERATIONS = 100; // Prevent infinite load more
+                            const loadMoreStartTime = Date.now();
+                            const MAX_LOAD_MORE_TIME = 30 * 60 * 1000; // 5 minutes max for load more
                             while (true) {
+                                // Load more circuit breakers
+                                if (loadMoreCounter >= MAX_LOAD_MORE_ITERATIONS) {
+                                    debugLog(`Maximum load more iterations reached (${MAX_LOAD_MORE_ITERATIONS}), stopping`);
+                                    return allResults;
+                                }
+                                if (Date.now() - loadMoreStartTime > MAX_LOAD_MORE_TIME) {
+                                    debugLog('Maximum load more time reached (5 minutes), stopping');
+                                    return allResults;
+                                }
+                                // Add async yield every 3 iterations
+                                if (loadMoreCounter % 3 === 0 && loadMoreCounter > 0) {
+                                    yield new Promise(resolve => setImmediate(resolve));
+                                }
                                 // Find working button with retry mechanism
                                 const { button: loadMoreButton, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
                                 availableSelectors = updatedSelectors;
@@ -969,9 +1100,14 @@ class Interpreter extends events_1.EventEmitter {
                                 }
                                 // Wait for content to load and check scroll height
                                 yield page.waitForTimeout(2000);
-                                yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                                yield page.evaluate(() => {
+                                    const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                    window.scrollTo(0, scrollHeight);
+                                });
                                 yield page.waitForTimeout(2000);
-                                const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                                const currentHeight = yield page.evaluate(() => {
+                                    return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                });
                                 const heightChanged = currentHeight !== previousHeight;
                                 previousHeight = currentHeight;
                                 yield scrapeCurrentPage();
@@ -1670,12 +1806,35 @@ class Interpreter extends events_1.EventEmitter {
                     selector.includes('@id=');
                 let count = 0;
                 if (isXPath) {
+                    // Add timeout to prevent XPath hanging
                     const locator = page.locator(`xpath=${selector}`);
-                    count = yield locator.count();
+                    const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('XPath timeout')), 5000));
+                    try {
+                        count = yield Promise.race([
+                            locator.count(),
+                            timeoutPromise
+                        ]);
+                    }
+                    catch (error) {
+                        // XPath timed out or failed
+                        return false;
+                    }
                 }
                 else {
-                    const elements = yield page.$$(selector);
-                    count = elements ? elements.length : 0;
+                    // Add timeout to CSS selector operations
+                    try {
+                        const elementsPromise = page.$$(selector);
+                        const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('CSS selector timeout')), 5000));
+                        const elements = yield Promise.race([
+                            elementsPromise,
+                            timeoutPromise
+                        ]);
+                        count = elements ? elements.length : 0;
+                    }
+                    catch (error) {
+                        // CSS selector timed out or failed
+                        return false;
+                    }
                 }
                 // For list selectors, we need multiple elements
                 if (isListSelector) {
@@ -1936,7 +2095,33 @@ class Interpreter extends events_1.EventEmitter {
                 this.concurrency.addJob(() => this.runLoop(popup, workflowCopy));
             });
             /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
+            let loopIterations = 0;
+            const MAX_LOOP_ITERATIONS = 1000; // Circuit breaker
+            let consecutiveFailures = 0;
+            const MAX_CONSECUTIVE_FAILURES = 10;
+            const startTime = Date.now();
+            const MAX_EXECUTION_TIME = 30 * 60 * 1000; // 30 minutes max
             while (true) {
+                // Multiple circuit breakers to prevent infinite loops
+                if (++loopIterations > MAX_LOOP_ITERATIONS) {
+                    this.log('Maximum loop iterations reached, terminating to prevent infinite loop', logger_1.Level.ERROR);
+                    return;
+                }
+                // Time-based circuit breaker
+                if (Date.now() - startTime > MAX_EXECUTION_TIME) {
+                    this.log('Maximum execution time reached (30 minutes), terminating workflow', logger_1.Level.ERROR);
+                    return;
+                }
+                // Failure-based circuit breaker
+                if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+                    this.log('Too many consecutive failures, terminating to prevent hang', logger_1.Level.ERROR);
+                    return;
+                }
+                // Check abort flag immediately
+                if (this.isAborted) {
+                    this.log('Workflow aborted in runLoop', logger_1.Level.WARN);
+                    return;
+                }
                 // Checks whether the page was closed from outside,
                 //  or the workflow execution has been stopped via `interpreter.stop()`
                 if (p.isClosed() || !this.stopper) {
@@ -2001,6 +2186,11 @@ class Interpreter extends events_1.EventEmitter {
                 console.log("MATCHED ACTION ID:", actionId);
                 this.log(`Matched ${JSON.stringify(action === null || action === void 0 ? void 0 : action.where)}`, logger_1.Level.LOG);
                 if (action) { // action is matched
+                    // Check abort flag before executing action
+                    if (this.isAborted) {
+                        this.log('Workflow aborted before action execution', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.activeId) {
                         this.options.debugChannel.activeId(actionId);
                     }
@@ -2017,14 +2207,21 @@ class Interpreter extends events_1.EventEmitter {
                         usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
                         workflowCopy.splice(actionId, 1);
                         console.log(`Action with ID ${action.id} removed from the workflow copy.`);
-                        // const newSelectors = this.getPreviousSelectors(workflow, actionId);
+                        // Reset counters on successful action (but keep some history to prevent infinite resets)
+                        loopIterations = Math.max(0, loopIterations - 10);
+                        consecutiveFailures = 0;
+                        // Add async yield to prevent event loop blocking
+                        if (loopIterations % 10 === 0) {
+                            yield new Promise(resolve => setImmediate(resolve));
+                        }
                     }
                     catch (e) {
                         this.log(e, logger_1.Level.ERROR);
-                        console.log(`Action with ID ${action.id} failed: ${e.message}`);
-                        // Still remove the failed action to prevent infinite loops
-                        workflowCopy.splice(actionId, 1);
-                        console.log(`Failed action removed. Remaining actions: ${workflowCopy.length}`);
+                        consecutiveFailures++;
+                        // Add delay on failures to prevent tight error loops
+                        yield new Promise(resolve => setTimeout(resolve, Math.min(1000, consecutiveFailures * 200)));
+                        // Don't crash on individual action failures - continue with next iteration
+                        continue;
                     }
                 }
                 else {
@@ -2036,9 +2233,32 @@ class Interpreter extends events_1.EventEmitter {
     }
     ensureScriptsLoaded(page) {
         return __awaiter(this, void 0, void 0, function* () {
-            const isScriptLoaded = yield page.evaluate(() => typeof window.scrape === 'function' && typeof window.scrapeSchema === 'function' && typeof window.scrapeList === 'function' && typeof window.scrapeListAuto === 'function' && typeof window.scrollDown === 'function' && typeof window.scrollUp === 'function');
-            if (!isScriptLoaded) {
-                yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+            try {
+                // Add timeout to prevent hanging on script evaluation
+                const evaluationPromise = page.evaluate(() => typeof window.scrape === 'function' &&
+                    typeof window.scrapeSchema === 'function' &&
+                    typeof window.scrapeList === 'function' &&
+                    typeof window.scrapeListAuto === 'function' &&
+                    typeof window.scrollDown === 'function' &&
+                    typeof window.scrollUp === 'function');
+                const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Script check timeout')), 3000));
+                const isScriptLoaded = yield Promise.race([
+                    evaluationPromise,
+                    timeoutPromise
+                ]);
+                if (!isScriptLoaded) {
+                    yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+                }
+            }
+            catch (error) {
+                // If script check fails, try to add the script anyway
+                this.log(`Script check failed, adding script anyway: ${error.message}`, logger_1.Level.WARN);
+                try {
+                    yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+                }
+                catch (scriptError) {
+                    this.log(`Failed to add script: ${scriptError.message}`, logger_1.Level.ERROR);
+                }
             }
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.16",
+  "version": "0.0.18",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",