mx-cloud 0.0.15 → 0.0.17

package/build/interpret.d.ts

@@ -54,6 +54,7 @@ export default class Interpreter extends EventEmitter {
     private options;
     private concurrency;
     private stopper;
+    private isAborted;
     private log;
     private blocker;
     private cumulativeResults;
@@ -80,6 +81,10 @@ export default class Interpreter extends EventEmitter {
      * @returns True if `where` is applicable in the given context, false otherwise
      */
     private applicable;
+    /**
+     * Sets the abort flag to immediately stop all operations
+     */
+    abort(): void;
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\

package/build/interpret.js

@@ -63,6 +63,7 @@ class Interpreter extends events_1.EventEmitter {
         var _a;
         super();
         this.stopper = null;
+        this.isAborted = false;
         this.blocker = null;
         this.cumulativeResults = [];
         this.autohealFailures = [];
@@ -90,7 +91,9 @@ class Interpreter extends events_1.EventEmitter {
         adblocker_playwright_1.PlaywrightBlocker.fromLists(cross_fetch_1.default, ['https://easylist.to/easylist/easylist.txt']).then(blocker => {
             this.blocker = blocker;
         }).catch(err => {
-            this.log(`Failed to initialize ad-blocker:`, logger_1.Level.ERROR);
+            this.log(`Failed to initialize ad-blocker: ${err.message}`, logger_1.Level.ERROR);
+            // Continue without ad-blocker rather than crashing
+            this.blocker = null;
         });
     }
     trackAutohealFailure(error) {
@@ -307,6 +310,12 @@ class Interpreter extends events_1.EventEmitter {
             }
         });
     }
+    /**
+     * Sets the abort flag to immediately stop all operations
+     */
+    abort() {
+        this.isAborted = true;
+    }
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\
@@ -318,7 +327,12 @@ class Interpreter extends events_1.EventEmitter {
    */
     carryOutSteps(page, steps) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a;
+            var _a, _b;
+            // Check abort flag at start of execution
+            if (this.isAborted) {
+                this.log('Workflow aborted, stopping execution', logger_1.Level.WARN);
+                return;
+            }
             /**
              * Defines overloaded (or added) methods/actions usable in the workflow.
              * If a method overloads any existing method of the Page class, it accepts the same set
@@ -376,6 +390,11 @@ class Interpreter extends events_1.EventEmitter {
                 }),
                 scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
                     var _a;
+                    // Check abort flag at start of scraping
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping scrapeSchema', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                         this.options.debugChannel.setActionType('scrapeSchema');
                     }
@@ -388,21 +407,52 @@ class Interpreter extends events_1.EventEmitter {
                     if (!this.cumulativeResults || !Array.isArray(this.cumulativeResults)) {
                         this.cumulativeResults = [];
                     }
+                    const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
                     if (this.cumulativeResults.length === 0) {
-                        this.cumulativeResults.push({});
+                        // First execution - create initial row
+                        const newRow = {};
+                        Object.entries(resultToProcess).forEach(([key, value]) => {
+                            if (value !== undefined) {
+                                newRow[key] = value;
+                            }
+                        });
+                        this.cumulativeResults.push(newRow);
                     }
-                    const mergedResult = this.cumulativeResults[0];
-                    const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
-                    Object.entries(resultToProcess).forEach(([key, value]) => {
-                        if (value !== undefined) {
-                            mergedResult[key] = value;
+                    else {
+                        // Check if any keys from new result already exist in the last row
+                        const lastRow = this.cumulativeResults[this.cumulativeResults.length - 1];
+                        const newResultKeys = Object.keys(resultToProcess).filter(key => resultToProcess[key] !== undefined);
+                        const hasRepeatedKeys = newResultKeys.some(key => lastRow.hasOwnProperty(key));
+                        if (hasRepeatedKeys) {
+                            // Keys are repeated - create a new row
+                            const newRow = {};
+                            Object.entries(resultToProcess).forEach(([key, value]) => {
+                                if (value !== undefined) {
+                                    newRow[key] = value;
+                                }
+                            });
+                            this.cumulativeResults.push(newRow);
                         }
-                    });
-                    console.log("Updated merged result:", mergedResult);
-                    yield this.options.serializableCallback([mergedResult]);
+                        else {
+                            // No repeated keys - merge with the last row
+                            Object.entries(resultToProcess).forEach(([key, value]) => {
+                                if (value !== undefined) {
+                                    lastRow[key] = value;
+                                }
+                            });
+                        }
+                    }
+                    console.log("Total accumulated rows:", this.cumulativeResults.length);
+                    console.log("Current results:", this.cumulativeResults);
+                    yield this.options.serializableCallback(this.cumulativeResults);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
                     var _a, _b;
+                    // Check abort flag at start of scraping
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping scrapeList', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                         this.options.debugChannel.setActionType('scrapeList');
                     }
@@ -410,17 +460,37 @@ class Interpreter extends events_1.EventEmitter {
                         yield this.options.serializableCallback({});
                         return;
                     }
-                    yield this.ensureScriptsLoaded(page);
-                    if ((_b = this.options.debugChannel) === null || _b === void 0 ? void 0 : _b.incrementScrapeListIndex) {
-                        this.options.debugChannel.incrementScrapeListIndex();
-                    }
-                    if (!config.pagination) {
-                        const scrapeResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                    try {
+                        yield this.ensureScriptsLoaded(page);
+                        if ((_b = this.options.debugChannel) === null || _b === void 0 ? void 0 : _b.incrementScrapeListIndex) {
+                            this.options.debugChannel.incrementScrapeListIndex();
+                        }
+                        let scrapeResults = [];
+                        if (!config.pagination) {
+                            scrapeResults = yield page.evaluate((cfg) => {
+                                try {
+                                    return window.scrapeList(cfg);
+                                }
+                                catch (error) {
+                                    console.warn('ScrapeList evaluation failed:', error.message);
+                                    return []; // Return empty array instead of failing
+                                }
+                            }, config);
+                        }
+                        else {
+                            scrapeResults = yield this.handlePagination(page, config);
+                        }
+                        // Ensure we always have an array
+                        if (!Array.isArray(scrapeResults)) {
+                            scrapeResults = [];
+                        }
+                        console.log(`ScrapeList completed with ${scrapeResults.length} results`);
                         yield this.options.serializableCallback(scrapeResults);
                     }
-                    else {
-                        const scrapeResults = yield this.handlePagination(page, config);
-                        yield this.options.serializableCallback(scrapeResults);
+                    catch (error) {
+                        console.error('ScrapeList action failed completely:', error.message);
+                        // Don't throw error, just return empty array
+                        yield this.options.serializableCallback([]);
                     }
                 }),
                 scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
@@ -451,9 +521,15 @@ class Interpreter extends events_1.EventEmitter {
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
                         this.options.debugChannel.setActionType('script');
                     }
-                    const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
-                    const x = new AsyncFunction('page', 'log', code);
-                    yield x(page, this.log);
+                    try {
+                        const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
+                        const x = new AsyncFunction('page', 'log', code);
+                        yield x(page, this.log);
+                    }
+                    catch (error) {
+                        this.log(`Script execution failed: ${error.message}`, logger_1.Level.ERROR);
+                        throw new Error(`Script execution error: ${error.message}`);
+                    }
                 }),
                 flag: () => __awaiter(this, void 0, void 0, function* () {
                     return new Promise((res) => {
@@ -481,54 +557,72 @@ class Interpreter extends events_1.EventEmitter {
                 }
             });
             for (const step of steps) {
-                this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
-                if (step.action in wawActions) {
-                    // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
-                    const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
-                    yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
+                // Check abort flag before each step
+                if (this.isAborted) {
+                    this.log('Workflow aborted during step execution', logger_1.Level.WARN);
+                    return;
                 }
-                else {
-                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
-                        this.options.debugChannel.setActionType(String(step.action));
-                    }
-                    // Implements the dot notation for the "method name" in the workflow
-                    const levels = String(step.action).split('.');
-                    const methodName = levels[levels.length - 1];
-                    let invokee = page;
-                    for (const level of levels.splice(0, levels.length - 1)) {
-                        invokee = invokee[level];
+                this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
+                try {
+                    if (step.action in wawActions) {
+                        // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
+                        const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
+                        yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
                     }
-                    if (methodName === 'waitForLoadState') {
-                        try {
-                            yield executeAction(invokee, methodName, step.args);
+                    else {
+                        if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                            this.options.debugChannel.setActionType(String(step.action));
                         }
-                        catch (error) {
-                            yield executeAction(invokee, methodName, 'domcontentloaded');
+                        // Implements the dot notation for the "method name" in the workflow
+                        const levels = String(step.action).split('.');
+                        const methodName = levels[levels.length - 1];
+                        let invokee = page;
+                        for (const level of levels.splice(0, levels.length - 1)) {
+                            invokee = invokee[level];
                         }
-                    }
-                    else if (methodName === 'click') {
-                        try {
-                            yield executeAction(invokee, methodName, step.args);
+                        if (methodName === 'waitForLoadState') {
+                            try {
+                                yield executeAction(invokee, methodName, step.args);
+                            }
+                            catch (error) {
+                                yield executeAction(invokee, methodName, 'domcontentloaded');
+                            }
                         }
-                        catch (error) {
+                        else if (methodName === 'click') {
                             try {
-                                yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                                yield executeAction(invokee, methodName, step.args);
                             }
                             catch (error) {
-                                continue;
+                                try {
+                                    yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                                }
+                                catch (error) {
+                                    this.log(`Click action failed for selector ${(_b = step.args) === null || _b === void 0 ? void 0 : _b[0]}: ${error.message}`, logger_1.Level.WARN);
+                                    continue; // Skip to next action
+                                }
                             }
                         }
-                    }
-                    else {
-                        yield executeAction(invokee, methodName, step.args);
+                        else {
+                            yield executeAction(invokee, methodName, step.args);
+                        }
                     }
                 }
+                catch (error) {
+                    this.log(`Action ${String(step.action)} failed: ${error.message}`, logger_1.Level.WARN);
+                    // Continue to next action instead of breaking
+                    continue;
+                }
                 yield new Promise((res) => { setTimeout(res, 500); });
             }
         });
     }
     handlePagination(page, config) {
         return __awaiter(this, void 0, void 0, function* () {
+            // Check abort flag at start of pagination
+            if (this.isAborted) {
+                this.log('Workflow aborted, stopping pagination', logger_1.Level.WARN);
+                return [];
+            }
             let allResults = [];
             let previousHeight = 0;
             let scrapedItems = new Set();
@@ -540,6 +634,11 @@ class Interpreter extends events_1.EventEmitter {
                 console.log(`[Page ${visitedUrls.size}] [URL: ${page.url()}] ${message}`, ...args);
             };
             const scrapeCurrentPage = () => __awaiter(this, void 0, void 0, function* () {
+                // Check abort flag before scraping current page
+                if (this.isAborted) {
+                    debugLog("Workflow aborted, stopping scrapeCurrentPage");
+                    return;
+                }
                 const results = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
                 const newResults = results.filter(item => {
                     const uniqueKey = JSON.stringify(item);
@@ -651,6 +750,11 @@ class Interpreter extends events_1.EventEmitter {
             let unchangedResultCounter = 0;
             try {
                 while (true) {
+                    // Check abort flag at start of each pagination iteration
+                    if (this.isAborted) {
+                        this.log('Workflow aborted during pagination loop', logger_1.Level.WARN);
+                        return allResults;
+                    }
                     switch (config.pagination.type) {
                         case 'scrollDown': {
                             let previousResultCount = allResults.length;
@@ -658,9 +762,14 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit()) {
                                 return allResults;
                             }
-                            yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                            yield page.evaluate(() => {
+                                const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                window.scrollTo(0, scrollHeight);
+                            });
                             yield page.waitForTimeout(2000);
-                            const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                            const currentHeight = yield page.evaluate(() => {
+                                return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                            });
                             const currentResultCount = allResults.length;
                             if (currentResultCount === previousResultCount) {
                                 unchangedResultCounter++;
@@ -915,9 +1024,14 @@ class Interpreter extends events_1.EventEmitter {
                                 }
                                 // Wait for content to load and check scroll height
                                 yield page.waitForTimeout(2000);
-                                yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                                yield page.evaluate(() => {
+                                    const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                    window.scrollTo(0, scrollHeight);
+                                });
                                 yield page.waitForTimeout(2000);
-                                const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                                const currentHeight = yield page.evaluate(() => {
+                                    return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                });
                                 const heightChanged = currentHeight !== previousHeight;
                                 previousHeight = currentHeight;
                                 yield scrapeCurrentPage();
@@ -1882,7 +1996,19 @@ class Interpreter extends events_1.EventEmitter {
                 this.concurrency.addJob(() => this.runLoop(popup, workflowCopy));
             });
             /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
+            let loopIterations = 0;
+            const MAX_LOOP_ITERATIONS = 1000; // Circuit breaker
             while (true) {
+                // Circuit breaker to prevent infinite loops
+                if (++loopIterations > MAX_LOOP_ITERATIONS) {
+                    this.log('Maximum loop iterations reached, terminating to prevent infinite loop', logger_1.Level.ERROR);
+                    return;
+                }
+                // Check abort flag immediately
+                if (this.isAborted) {
+                    this.log('Workflow aborted in runLoop', logger_1.Level.WARN);
+                    return;
+                }
                 // Checks whether the page was closed from outside,
                 //  or the workflow execution has been stopped via `interpreter.stop()`
                 if (p.isClosed() || !this.stopper) {
@@ -1895,26 +2021,29 @@ class Interpreter extends events_1.EventEmitter {
                     yield p.close();
                     return;
                 }
-                const newSelectors = this.getSelectors(workflowCopy);
-                newSelectors.forEach(selector => {
-                    if (!selectors.includes(selector)) {
-                        selectors.push(selector);
-                    }
-                });
-                let pageState = {};
-                let getStateTest = "Hello";
-                try {
-                    pageState = yield this.getState(p, workflowCopy, selectors);
-                    selectors = [];
-                    console.log("Empty selectors:", selectors);
-                }
-                catch (e) {
-                    this.log('The browser has been closed.');
+                if (workflowCopy.length === 0) {
+                    this.log('All actions completed. Workflow finished.', logger_1.Level.LOG);
                     return;
                 }
-                if (this.options.debug) {
-                    this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, logger_1.Level.WARN);
-                }
+                // const newSelectors = this.getSelectors(workflowCopy);
+                //   newSelectors.forEach(selector => {
+                //     if (!selectors.includes(selector)) {
+                //       selectors.push(selector);
+                //     }
+                // });
+                // let pageState = {};
+                // let getStateTest = "Hello";
+                // try {
+                //   pageState = await this.getState(p, workflowCopy, selectors);
+                //   selectors = [];
+                //   console.log("Empty selectors:", selectors)
+                // } catch (e: any) {
+                //   this.log('The browser has been closed.');
+                //   return;
+                // }
+                // if (this.options.debug) {
+                //   this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, Level.WARN);
+                // }
                 // const actionId = workflow.findIndex((step) => {
                 //   const isApplicable = this.applicable(step.where, pageState, usedActions);
                 //   console.log("-------------------------------------------------------------");
@@ -1934,15 +2063,21 @@ class Interpreter extends events_1.EventEmitter {
                 //     console.log("SCHEMA CHANGES:", changes);
                 //   }
                 // }
-                actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
-                if (actionId !== -1 && workflowCopy[actionId]) {
-                    workflowCopy[actionId] = yield this.validateWorkflowAction(p, workflowCopy[actionId]);
-                }
+                // actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
+                // if (actionId !== -1 && workflowCopy[actionId]) {
+                //   workflowCopy[actionId] = await this.validateWorkflowAction(p, workflowCopy[actionId]);
+                // }
+                const actionId = workflowCopy.length - 1;
                 const action = workflowCopy[actionId];
                 console.log("MATCHED ACTION:", action);
                 console.log("MATCHED ACTION ID:", actionId);
                 this.log(`Matched ${JSON.stringify(action === null || action === void 0 ? void 0 : action.where)}`, logger_1.Level.LOG);
                 if (action) { // action is matched
+                    // Check abort flag before executing action
+                    if (this.isAborted) {
+                        this.log('Workflow aborted before action execution', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.activeId) {
                         this.options.debugChannel.activeId(actionId);
                     }
@@ -1959,10 +2094,13 @@ class Interpreter extends events_1.EventEmitter {
                         usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
                         workflowCopy.splice(actionId, 1);
                         console.log(`Action with ID ${action.id} removed from the workflow copy.`);
-                        // const newSelectors = this.getPreviousSelectors(workflow, actionId);
+                        // Reset loop iteration counter on successful action
+                        loopIterations = 0;
                     }
                     catch (e) {
                         this.log(e, logger_1.Level.ERROR);
+                        // Don't crash on individual action failures - continue with next iteration
+                        continue;
                     }
                 }
                 else {

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.15",
+  "version": "0.0.17",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",
   "scripts": {
     "test": "jest",
-    "build": "npm run clean && tsc",
+    "build": "tsc",
     "lint": "eslint .",
     "clean": "rimraf ./build"
   },
@@ -23,5 +23,8 @@
     "playwright": "^1.50.0",
     "playwright-extra": "^4.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2"
+  },
+  "devDependencies": {
+    "@types/node": "^24.3.1"
   }
 }