mx-cloud 0.0.16 → 0.0.17

package/build/interpret.d.ts

@@ -54,6 +54,7 @@ export default class Interpreter extends EventEmitter {
     private options;
     private concurrency;
     private stopper;
+    private isAborted;
     private log;
     private blocker;
     private cumulativeResults;
@@ -80,6 +81,10 @@ export default class Interpreter extends EventEmitter {
      * @returns True if `where` is applicable in the given context, false otherwise
      */
     private applicable;
+    /**
+     * Sets the abort flag to immediately stop all operations
+     */
+    abort(): void;
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\

package/build/interpret.js

@@ -63,6 +63,7 @@ class Interpreter extends events_1.EventEmitter {
         var _a;
         super();
         this.stopper = null;
+        this.isAborted = false;
         this.blocker = null;
         this.cumulativeResults = [];
         this.autohealFailures = [];
@@ -90,7 +91,9 @@ class Interpreter extends events_1.EventEmitter {
         adblocker_playwright_1.PlaywrightBlocker.fromLists(cross_fetch_1.default, ['https://easylist.to/easylist/easylist.txt']).then(blocker => {
             this.blocker = blocker;
         }).catch(err => {
-            this.log(`Failed to initialize ad-blocker:`, logger_1.Level.ERROR);
+            this.log(`Failed to initialize ad-blocker: ${err.message}`, logger_1.Level.ERROR);
+            // Continue without ad-blocker rather than crashing
+            this.blocker = null;
         });
     }
     trackAutohealFailure(error) {
@@ -307,6 +310,12 @@ class Interpreter extends events_1.EventEmitter {
             }
         });
     }
+    /**
+     * Sets the abort flag to immediately stop all operations
+     */
+    abort() {
+        this.isAborted = true;
+    }
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\
@@ -319,6 +328,11 @@ class Interpreter extends events_1.EventEmitter {
     carryOutSteps(page, steps) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b;
+            // Check abort flag at start of execution
+            if (this.isAborted) {
+                this.log('Workflow aborted, stopping execution', logger_1.Level.WARN);
+                return;
+            }
             /**
              * Defines overloaded (or added) methods/actions usable in the workflow.
              * If a method overloads any existing method of the Page class, it accepts the same set
@@ -376,6 +390,11 @@ class Interpreter extends events_1.EventEmitter {
                 }),
                 scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
                     var _a;
+                    // Check abort flag at start of scraping
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping scrapeSchema', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                         this.options.debugChannel.setActionType('scrapeSchema');
                     }
@@ -429,6 +448,11 @@ class Interpreter extends events_1.EventEmitter {
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
                     var _a, _b;
+                    // Check abort flag at start of scraping
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping scrapeList', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                         this.options.debugChannel.setActionType('scrapeList');
                     }
@@ -497,9 +521,15 @@ class Interpreter extends events_1.EventEmitter {
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                         this.options.debugChannel.setActionType('script');
                     }
-                    const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
-                    const x = new AsyncFunction('page', 'log', code);
-                    yield x(page, this.log);
+                    try {
+                        const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
+                        const x = new AsyncFunction('page', 'log', code);
+                        yield x(page, this.log);
+                    }
+                    catch (error) {
+                        this.log(`Script execution failed: ${error.message}`, logger_1.Level.ERROR);
+                        throw new Error(`Script execution error: ${error.message}`);
+                    }
                 }),
                 flag: () => __awaiter(this, void 0, void 0, function* () {
                     return new Promise((res) => {
@@ -527,6 +557,11 @@ class Interpreter extends events_1.EventEmitter {
                 }
             });
             for (const step of steps) {
+                // Check abort flag before each step
+                if (this.isAborted) {
+                    this.log('Workflow aborted during step execution', logger_1.Level.WARN);
+                    return;
+                }
                 this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
                 try {
                     if (step.action in wawActions) {
@@ -583,6 +618,11 @@ class Interpreter extends events_1.EventEmitter {
     }
     handlePagination(page, config) {
         return __awaiter(this, void 0, void 0, function* () {
+            // Check abort flag at start of pagination
+            if (this.isAborted) {
+                this.log('Workflow aborted, stopping pagination', logger_1.Level.WARN);
+                return [];
+            }
             let allResults = [];
             let previousHeight = 0;
             let scrapedItems = new Set();
@@ -594,6 +634,11 @@ class Interpreter extends events_1.EventEmitter {
                 console.log(`[Page ${visitedUrls.size}] [URL: ${page.url()}] ${message}`, ...args);
             };
             const scrapeCurrentPage = () => __awaiter(this, void 0, void 0, function* () {
+                // Check abort flag before scraping current page
+                if (this.isAborted) {
+                    debugLog("Workflow aborted, stopping scrapeCurrentPage");
+                    return;
+                }
                 const results = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
                 const newResults = results.filter(item => {
                     const uniqueKey = JSON.stringify(item);
@@ -705,6 +750,11 @@ class Interpreter extends events_1.EventEmitter {
             let unchangedResultCounter = 0;
             try {
                 while (true) {
+                    // Check abort flag at start of each pagination iteration
+                    if (this.isAborted) {
+                        this.log('Workflow aborted during pagination loop', logger_1.Level.WARN);
+                        return allResults;
+                    }
                     switch (config.pagination.type) {
                         case 'scrollDown': {
                             let previousResultCount = allResults.length;
@@ -712,9 +762,14 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit()) {
                                 return allResults;
                             }
-                            yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                            yield page.evaluate(() => {
+                                const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                window.scrollTo(0, scrollHeight);
+                            });
                             yield page.waitForTimeout(2000);
-                            const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                            const currentHeight = yield page.evaluate(() => {
+                                return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                            });
                             const currentResultCount = allResults.length;
                             if (currentResultCount === previousResultCount) {
                                 unchangedResultCounter++;
@@ -969,9 +1024,14 @@ class Interpreter extends events_1.EventEmitter {
                                 }
                                 // Wait for content to load and check scroll height
                                 yield page.waitForTimeout(2000);
-                                yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                                yield page.evaluate(() => {
+                                    const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                    window.scrollTo(0, scrollHeight);
+                                });
                                 yield page.waitForTimeout(2000);
-                                const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                                const currentHeight = yield page.evaluate(() => {
+                                    return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                });
                                 const heightChanged = currentHeight !== previousHeight;
                                 previousHeight = currentHeight;
                                 yield scrapeCurrentPage();
@@ -1936,7 +1996,19 @@ class Interpreter extends events_1.EventEmitter {
                 this.concurrency.addJob(() => this.runLoop(popup, workflowCopy));
             });
             /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
+            let loopIterations = 0;
+            const MAX_LOOP_ITERATIONS = 1000; // Circuit breaker
             while (true) {
+                // Circuit breaker to prevent infinite loops
+                if (++loopIterations > MAX_LOOP_ITERATIONS) {
+                    this.log('Maximum loop iterations reached, terminating to prevent infinite loop', logger_1.Level.ERROR);
+                    return;
+                }
+                // Check abort flag immediately
+                if (this.isAborted) {
+                    this.log('Workflow aborted in runLoop', logger_1.Level.WARN);
+                    return;
+                }
                 // Checks whether the page was closed from outside,
                 //  or the workflow execution has been stopped via `interpreter.stop()`
                 if (p.isClosed() || !this.stopper) {
@@ -2001,6 +2073,11 @@ class Interpreter extends events_1.EventEmitter {
                 console.log("MATCHED ACTION ID:", actionId);
                 this.log(`Matched ${JSON.stringify(action === null || action === void 0 ? void 0 : action.where)}`, logger_1.Level.LOG);
                 if (action) { // action is matched
+                    // Check abort flag before executing action
+                    if (this.isAborted) {
+                        this.log('Workflow aborted before action execution', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.activeId) {
                         this.options.debugChannel.activeId(actionId);
                     }
@@ -2017,14 +2094,13 @@ class Interpreter extends events_1.EventEmitter {
                         usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
                         workflowCopy.splice(actionId, 1);
                         console.log(`Action with ID ${action.id} removed from the workflow copy.`);
-                        // const newSelectors = this.getPreviousSelectors(workflow, actionId);
+                        // Reset loop iteration counter on successful action
+                        loopIterations = 0;
                     }
                     catch (e) {
                         this.log(e, logger_1.Level.ERROR);
-                        console.log(`Action with ID ${action.id} failed: ${e.message}`);
-                        // Still remove the failed action to prevent infinite loops
-                        workflowCopy.splice(actionId, 1);
-                        console.log(`Failed action removed. Remaining actions: ${workflowCopy.length}`);
+                        // Don't crash on individual action failures - continue with next iteration
+                        continue;
                     }
                 }
                 else {

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.16",
+  "version": "0.0.17",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",
   "scripts": {
     "test": "jest",
-    "build": "npm run clean && tsc",
+    "build": "tsc",
     "lint": "eslint .",
     "clean": "rimraf ./build"
   },
@@ -23,5 +23,8 @@
     "playwright": "^1.50.0",
     "playwright-extra": "^4.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2"
+  },
+  "devDependencies": {
+    "@types/node": "^24.3.1"
   }
 }