mx-cloud 0.0.17 → 0.0.19

package/build/interpret.d.ts

@@ -59,6 +59,9 @@ export default class Interpreter extends EventEmitter {
     private blocker;
     private cumulativeResults;
     private autohealFailures;
+    private namedResults;
+    private screenshotCounter;
+    private serializableDataByType;
     constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
     trackAutohealFailure(error: string): void;
     private applyAdBlocker;
@@ -85,6 +88,10 @@ export default class Interpreter extends EventEmitter {
      * Sets the abort flag to immediately stop all operations
      */
     abort(): void;
+    /**
+     * Returns the current abort status
+     */
+    getIsAborted(): boolean;
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\

package/build/interpret.js

@@ -67,6 +67,12 @@ class Interpreter extends events_1.EventEmitter {
         this.blocker = null;
         this.cumulativeResults = [];
         this.autohealFailures = [];
+        this.namedResults = {};
+        this.screenshotCounter = 0;
+        this.serializableDataByType = {
+            scrapeList: {},
+            scrapeSchema: {}
+        };
         this.workflow = workflow.workflow;
         this.initializedWorkflow = null;
         this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => {
@@ -316,6 +322,12 @@ class Interpreter extends events_1.EventEmitter {
     abort() {
         this.isAborted = true;
     }
+    /**
+     * Returns the current abort status
+     */
+    getIsAborted() {
+        return this.isAborted;
+    }
     /**
    * Given a Playwright's page object and a "declarative" list of actions, this function
    * calls all mentioned functions on the Page object.\
@@ -342,13 +354,22 @@ class Interpreter extends events_1.EventEmitter {
              * Beware of false linter errors - here, we know better!
              */
             const wawActions = {
-                screenshot: (params) => __awaiter(this, void 0, void 0, function* () {
+                screenshot: (params, nameOverride) => __awaiter(this, void 0, void 0, function* () {
                     var _a;
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
-                        this.options.debugChannel.setActionType('screenshot');
+                        this.options.debugChannel.setActionType("screenshot");
                     }
                     const screenshotBuffer = yield page.screenshot(Object.assign(Object.assign({}, params), { path: undefined }));
-                    yield this.options.binaryCallback(screenshotBuffer, 'image/png');
+                    const baseName = nameOverride || "Screenshot";
+                    // ✅ Use the typed class property
+                    this.screenshotCounter += 1;
+                    const screenshotName = `${baseName} ${this.screenshotCounter}`;
+                    // ✅ Pass structured metadata to binaryCallback
+                    yield this.options.binaryCallback({
+                        name: screenshotName,
+                        data: screenshotBuffer,
+                        mimeType: "image/png",
+                    }, "image/png");
                 }),
                 enqueueLinks: (selector) => __awaiter(this, void 0, void 0, function* () {
                     var _a;
@@ -444,7 +465,31 @@ class Interpreter extends events_1.EventEmitter {
                     }
                     console.log("Total accumulated rows:", this.cumulativeResults.length);
                     console.log("Current results:", this.cumulativeResults);
-                    yield this.options.serializableCallback(this.cumulativeResults);
+                    // ✅ Append schema results under "scrapeSchema" → name
+                    const actionType = "scrapeSchema";
+                    const actionName = schema.__name || "Texts";
+                    if (!this.namedResults[actionType])
+                        this.namedResults[actionType] = {};
+                    this.namedResults[actionType][actionName] = this.cumulativeResults;
+                    if (!this.serializableDataByType[actionType])
+                        this.serializableDataByType[actionType] = {};
+                    if (!this.serializableDataByType[actionType])
+                        this.serializableDataByType[actionType] = {};
+                    if (!this.serializableDataByType[actionType][actionName]) {
+                        this.serializableDataByType[actionType][actionName] = {};
+                    }
+                    for (const row of this.cumulativeResults) {
+                        for (const [key, value] of Object.entries(row)) {
+                            if (value !== undefined) {
+                                this.serializableDataByType[actionType][actionName][key] = value;
+                            }
+                        }
+                    }
+                    // now emit full structured object
+                    yield this.options.serializableCallback({
+                        scrapeList: this.serializableDataByType.scrapeList,
+                        scrapeSchema: this.serializableDataByType.scrapeSchema
+                    });
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
                     var _a, _b;
@@ -485,12 +530,29 @@ class Interpreter extends events_1.EventEmitter {
                             scrapeResults = [];
                         }
                         console.log(`ScrapeList completed with ${scrapeResults.length} results`);
-                        yield this.options.serializableCallback(scrapeResults);
+                        // ✅ Append list results under "scrapeList" → name
+                        const actionType = "scrapeList";
+                        const actionName = config.__name || "List";
+                        if (!this.serializableDataByType[actionType])
+                            this.serializableDataByType[actionType] = {};
+                        if (!this.serializableDataByType[actionType][actionName]) {
+                            this.serializableDataByType[actionType][actionName] = [];
+                        }
+                        this.serializableDataByType[actionType][actionName].push(...scrapeResults);
+                        yield this.options.serializableCallback({
+                            scrapeList: this.serializableDataByType.scrapeList,
+                            scrapeSchema: this.serializableDataByType.scrapeSchema
+                        });
                     }
                     catch (error) {
                         console.error('ScrapeList action failed completely:', error.message);
                         // Don't throw error, just return empty array
-                        yield this.options.serializableCallback([]);
+                        const actionType = "scrapeList";
+                        const actionName = config.__name || "List";
+                        if (!this.namedResults[actionType])
+                            this.namedResults[actionType] = {};
+                        this.namedResults[actionType][actionName] = [];
+                        yield this.options.serializableCallback(this.namedResults);
                     }
                 }),
                 scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
@@ -563,6 +625,41 @@ class Interpreter extends events_1.EventEmitter {
                     return;
                 }
                 this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
+                try {
+                    const debug = this.options.debugChannel;
+                    if (debug === null || debug === void 0 ? void 0 : debug.setActionType) {
+                        debug.setActionType(String(step.action));
+                    }
+                    // Safely extract name for this step
+                    let stepName = null;
+                    // If the workflow step itself has a name
+                    if (step === null || step === void 0 ? void 0 : step.name) {
+                        stepName = step.name;
+                    }
+                    // If the first arg is an object with a __name property
+                    else if (Array.isArray(step === null || step === void 0 ? void 0 : step.args) &&
+                        step.args.length > 0 &&
+                        typeof step.args[0] === "object" &&
+                        "__name" in step.args[0]) {
+                        stepName = step.args[0].__name;
+                    }
+                    // If args itself is an object with a __name property
+                    else if (typeof (step === null || step === void 0 ? void 0 : step.args) === "object" &&
+                        (step === null || step === void 0 ? void 0 : step.args) !== null &&
+                        "__name" in step.args) {
+                        stepName = step.args.__name;
+                    }
+                    // Default fallback
+                    if (!stepName) {
+                        stepName = String(step.action);
+                    }
+                    if (debug && typeof debug.setActionName === "function") {
+                        debug.setActionName(stepName);
+                    }
+                }
+                catch (err) {
+                    this.log(`Failed to set action name/type: ${err.message}`, logger_1.Level.WARN);
+                }
                 try {
                     if (step.action in wawActions) {
                         // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
@@ -582,7 +679,15 @@ class Interpreter extends events_1.EventEmitter {
                         }
                         if (methodName === 'waitForLoadState') {
                             try {
-                                yield executeAction(invokee, methodName, step.args);
+                                // Add timeout if not already specified
+                                let args = step.args;
+                                if (Array.isArray(args) && args.length === 1) {
+                                    args = [args[0], { timeout: 30000 }];
+                                }
+                                else if (!Array.isArray(args)) {
+                                    args = [args, { timeout: 30000 }];
+                                }
+                                yield executeAction(invokee, methodName, args);
                             }
                             catch (error) {
                                 yield executeAction(invokee, methodName, 'domcontentloaded');
@@ -639,7 +744,17 @@ class Interpreter extends events_1.EventEmitter {
                     debugLog("Workflow aborted, stopping scrapeCurrentPage");
                     return;
                 }
-                const results = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                // Add timeout to prevent hanging on page evaluation
+                const evaluationPromise = page.evaluate((cfg) => window.scrapeList(cfg), config);
+                const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Page evaluation timeout')), 10000));
+                let results;
+                try {
+                    results = yield Promise.race([evaluationPromise, timeoutPromise]);
+                }
+                catch (error) {
+                    debugLog(`Page evaluation failed: ${error.message}`);
+                    return;
+                }
                 const newResults = results.filter(item => {
                     const uniqueKey = JSON.stringify(item);
                     if (scrapedItems.has(uniqueKey))
@@ -696,14 +811,22 @@ class Interpreter extends events_1.EventEmitter {
             });
             // Enhanced button finder with retry mechanism for both CSS and XPath selectors
             const findWorkingButton = (selectors) => __awaiter(this, void 0, void 0, function* () {
+                const startTime = Date.now();
+                const MAX_BUTTON_SEARCH_TIME = 15000;
                 let updatedSelectors = [...selectors];
                 for (let i = 0; i < selectors.length; i++) {
+                    // Check overall timeout
+                    if (Date.now() - startTime > MAX_BUTTON_SEARCH_TIME) {
+                        debugLog(`Button search timeout reached (${MAX_BUTTON_SEARCH_TIME}ms), aborting`);
+                        break;
+                    }
                     const selector = selectors[i];
                     let retryCount = 0;
                     let selectorSuccess = false;
                     while (retryCount < MAX_RETRIES && !selectorSuccess) {
                         try {
-                            const button = yield waitForSelectorUniversal(selector, { timeout: 10000 });
+                            // Reduce timeout to prevent hanging on slow selectors
+                            const button = yield waitForSelectorUniversal(selector, { timeout: 2000 });
                             if (button) {
                                 debugLog('Found working selector:', selector);
                                 return {
@@ -712,16 +835,30 @@ class Interpreter extends events_1.EventEmitter {
                                     updatedSelectors
                                 };
                             }
+                            else {
+                                // Treat null result as failed attempt
+                                retryCount++;
+                                debugLog(`Selector "${selector}" not found: attempt ${retryCount}/${MAX_RETRIES}`);
+                                if (retryCount < MAX_RETRIES) {
+                                    yield page.waitForTimeout(RETRY_DELAY);
+                                }
+                                else {
+                                    debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
+                                    updatedSelectors = updatedSelectors.filter(s => s !== selector);
+                                    selectorSuccess = true; // Exit retry loop for this selector
+                                }
+                            }
                         }
                         catch (error) {
                             retryCount++;
-                            debugLog(`Selector "${selector}" failed: attempt ${retryCount}/${MAX_RETRIES}`);
+                            debugLog(`Selector "${selector}" error: attempt ${retryCount}/${MAX_RETRIES} - ${error.message}`);
                             if (retryCount < MAX_RETRIES) {
                                 yield page.waitForTimeout(RETRY_DELAY);
                             }
                             else {
                                 debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
                                 updatedSelectors = updatedSelectors.filter(s => s !== selector);
+                                selectorSuccess = true; // Exit retry loop for this selector
                             }
                         }
                     }
@@ -748,6 +885,10 @@ class Interpreter extends events_1.EventEmitter {
             });
             let availableSelectors = config.pagination.selector.split(',');
             let unchangedResultCounter = 0;
+            let paginationIterations = 0;
+            const MAX_PAGINATION_ITERATIONS = 100; // Prevent infinite pagination
+            const paginationStartTime = Date.now();
+            const MAX_PAGINATION_TIME = 30 * 60 * 1000; // 30 minutes max for pagination
             try {
                 while (true) {
                     // Check abort flag at start of each pagination iteration
@@ -755,6 +896,19 @@ class Interpreter extends events_1.EventEmitter {
                         this.log('Workflow aborted during pagination loop', logger_1.Level.WARN);
                         return allResults;
                     }
+                    // Pagination circuit breakers
+                    if (++paginationIterations > MAX_PAGINATION_ITERATIONS) {
+                        debugLog(`Maximum pagination iterations reached (${MAX_PAGINATION_ITERATIONS}), stopping`);
+                        return allResults;
+                    }
+                    if (Date.now() - paginationStartTime > MAX_PAGINATION_TIME) {
+                        debugLog('Maximum pagination time reached (10 minutes), stopping');
+                        return allResults;
+                    }
+                    // Add async yield every 5 iterations to prevent event loop blocking
+                    if (paginationIterations % 5 === 0) {
+                        yield new Promise(resolve => setImmediate(resolve));
+                    }
                     switch (config.pagination.type) {
                         case 'scrollDown': {
                             let previousResultCount = allResults.length;
@@ -971,10 +1125,23 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit())
                                 return allResults;
                             let loadMoreCounter = 0;
-                            // let previousResultCount = allResults.length;
-                            // let noNewItemsCounter = 0;
-                            // const MAX_NO_NEW_ITEMS = 2;
+                            const MAX_LOAD_MORE_ITERATIONS = 100; // Prevent infinite load more
+                            const loadMoreStartTime = Date.now();
+                            const MAX_LOAD_MORE_TIME = 30 * 60 * 1000; // 5 minutes max for load more
                             while (true) {
+                                // Load more circuit breakers
+                                if (loadMoreCounter >= MAX_LOAD_MORE_ITERATIONS) {
+                                    debugLog(`Maximum load more iterations reached (${MAX_LOAD_MORE_ITERATIONS}), stopping`);
+                                    return allResults;
+                                }
+                                if (Date.now() - loadMoreStartTime > MAX_LOAD_MORE_TIME) {
+                                    debugLog('Maximum load more time reached (5 minutes), stopping');
+                                    return allResults;
+                                }
+                                // Add async yield every 3 iterations
+                                if (loadMoreCounter % 3 === 0 && loadMoreCounter > 0) {
+                                    yield new Promise(resolve => setImmediate(resolve));
+                                }
                                 // Find working button with retry mechanism
                                 const { button: loadMoreButton, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
                                 availableSelectors = updatedSelectors;
@@ -1730,12 +1897,35 @@ class Interpreter extends events_1.EventEmitter {
                     selector.includes('@id=');
                 let count = 0;
                 if (isXPath) {
+                    // Add timeout to prevent XPath hanging
                     const locator = page.locator(`xpath=${selector}`);
-                    count = yield locator.count();
+                    const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('XPath timeout')), 5000));
+                    try {
+                        count = yield Promise.race([
+                            locator.count(),
+                            timeoutPromise
+                        ]);
+                    }
+                    catch (error) {
+                        // XPath timed out or failed
+                        return false;
+                    }
                 }
                 else {
-                    const elements = yield page.$$(selector);
-                    count = elements ? elements.length : 0;
+                    // Add timeout to CSS selector operations
+                    try {
+                        const elementsPromise = page.$$(selector);
+                        const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('CSS selector timeout')), 5000));
+                        const elements = yield Promise.race([
+                            elementsPromise,
+                            timeoutPromise
+                        ]);
+                        count = elements ? elements.length : 0;
+                    }
+                    catch (error) {
+                        // CSS selector timed out or failed
+                        return false;
+                    }
                 }
                 // For list selectors, we need multiple elements
                 if (isListSelector) {
@@ -1998,12 +2188,26 @@ class Interpreter extends events_1.EventEmitter {
             /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
             let loopIterations = 0;
             const MAX_LOOP_ITERATIONS = 1000; // Circuit breaker
+            let consecutiveFailures = 0;
+            const MAX_CONSECUTIVE_FAILURES = 10;
+            const startTime = Date.now();
+            const MAX_EXECUTION_TIME = 30 * 60 * 1000; // 30 minutes max
             while (true) {
-                // Circuit breaker to prevent infinite loops
+                // Multiple circuit breakers to prevent infinite loops
                 if (++loopIterations > MAX_LOOP_ITERATIONS) {
                     this.log('Maximum loop iterations reached, terminating to prevent infinite loop', logger_1.Level.ERROR);
                     return;
                 }
+                // Time-based circuit breaker
+                if (Date.now() - startTime > MAX_EXECUTION_TIME) {
+                    this.log('Maximum execution time reached (30 minutes), terminating workflow', logger_1.Level.ERROR);
+                    return;
+                }
+                // Failure-based circuit breaker
+                if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+                    this.log('Too many consecutive failures, terminating to prevent hang', logger_1.Level.ERROR);
+                    return;
+                }
                 // Check abort flag immediately
                 if (this.isAborted) {
                     this.log('Workflow aborted in runLoop', logger_1.Level.WARN);
@@ -2094,11 +2298,19 @@ class Interpreter extends events_1.EventEmitter {
                         usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
                         workflowCopy.splice(actionId, 1);
                         console.log(`Action with ID ${action.id} removed from the workflow copy.`);
-                        // Reset loop iteration counter on successful action
-                        loopIterations = 0;
+                        // Reset counters on successful action (but keep some history to prevent infinite resets)
+                        loopIterations = Math.max(0, loopIterations - 10);
+                        consecutiveFailures = 0;
+                        // Add async yield to prevent event loop blocking
+                        if (loopIterations % 10 === 0) {
+                            yield new Promise(resolve => setImmediate(resolve));
+                        }
                     }
                     catch (e) {
                         this.log(e, logger_1.Level.ERROR);
+                        consecutiveFailures++;
+                        // Add delay on failures to prevent tight error loops
+                        yield new Promise(resolve => setTimeout(resolve, Math.min(1000, consecutiveFailures * 200)));
                         // Don't crash on individual action failures - continue with next iteration
                         continue;
                     }
@@ -2112,9 +2324,32 @@ class Interpreter extends events_1.EventEmitter {
     }
     ensureScriptsLoaded(page) {
         return __awaiter(this, void 0, void 0, function* () {
-            const isScriptLoaded = yield page.evaluate(() => typeof window.scrape === 'function' && typeof window.scrapeSchema === 'function' && typeof window.scrapeList === 'function' && typeof window.scrapeListAuto === 'function' && typeof window.scrollDown === 'function' && typeof window.scrollUp === 'function');
-            if (!isScriptLoaded) {
-                yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+            try {
+                // Add timeout to prevent hanging on script evaluation
+                const evaluationPromise = page.evaluate(() => typeof window.scrape === 'function' &&
+                    typeof window.scrapeSchema === 'function' &&
+                    typeof window.scrapeList === 'function' &&
+                    typeof window.scrapeListAuto === 'function' &&
+                    typeof window.scrollDown === 'function' &&
+                    typeof window.scrollUp === 'function');
+                const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Script check timeout')), 3000));
+                const isScriptLoaded = yield Promise.race([
+                    evaluationPromise,
+                    timeoutPromise
+                ]);
+                if (!isScriptLoaded) {
+                    yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+                }
+            }
+            catch (error) {
+                // If script check fails, try to add the script anyway
+                this.log(`Script check failed, adding script anyway: ${error.message}`, logger_1.Level.WARN);
+                try {
+                    yield page.addInitScript({ path: path_1.default.join(__dirname, 'browserSide', 'scraper.js') });
+                }
+                catch (scriptError) {
+                    this.log(`Failed to add script: ${scriptError.message}`, logger_1.Level.ERROR);
+                }
             }
         });
     }

package/build/preprocessor.js

@@ -34,6 +34,8 @@ class Preprocessor {
                 what: joi_1.default.array().items({
                     action: joi_1.default.string().required(),
                     args: joi_1.default.array().items(joi_1.default.any()),
+                    name: joi_1.default.string(),
+                    actionId: joi_1.default.string()
                 }).required(),
             })).required(),
         });

package/build/types/workflow.d.ts

@@ -27,6 +27,8 @@ export type CustomFunctions = 'scrape' | 'scrapeSchema' | 'scroll' | 'screenshot
 export type What = {
     action: MethodNames<Page> | CustomFunctions;
     args?: any[];
+    name?: string;
+    actionId?: string;
 };
 export type PageState = Partial<BaseConditions>;
 export type ParamType = Record<string, any>;

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.17",
+  "version": "0.0.19",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",
   "scripts": {
     "test": "jest",
-    "build": "tsc",
+    "build": "npm run clean && tsc",
     "lint": "eslint .",
     "clean": "rimraf ./build"
   },
@@ -23,8 +23,5 @@
     "playwright": "^1.50.0",
     "playwright-extra": "^4.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2"
-  },
-  "devDependencies": {
-    "@types/node": "^24.3.1"
   }
 }