mx-cloud 0.0.22 → 0.0.24

package/build/interpret.d.ts

@@ -160,5 +160,10 @@ export default class Interpreter extends EventEmitter {
      */
     run(page: Page, params?: ParamType): Promise<void>;
     stop(): Promise<void>;
+    /**
+     * Cleanup method to release resources and prevent memory leaks
+     * Call this when the interpreter is no longer needed
+     */
+    cleanup(): Promise<void>;
 }
 export {};

package/build/interpret.js

@@ -394,8 +394,9 @@ class Interpreter extends events_1.EventEmitter {
                     for (const link of links) {
                         // eslint-disable-next-line
                         this.concurrency.addJob(() => __awaiter(this, void 0, void 0, function* () {
+                            let newPage = null;
                             try {
-                                const newPage = yield context.newPage();
+                                newPage = yield context.newPage();
                                 yield newPage.goto(link);
                                 yield newPage.waitForLoadState('networkidle');
                                 yield this.runLoop(newPage, this.initializedWorkflow);
@@ -406,6 +407,16 @@ class Interpreter extends events_1.EventEmitter {
                                 // the interpreter by throwing).
                                 this.log(e, logger_1.Level.ERROR);
                             }
+                            finally {
+                                if (newPage && !newPage.isClosed()) {
+                                    try {
+                                        yield newPage.close();
+                                    }
+                                    catch (closeError) {
+                                        this.log('Failed to close enqueued page', logger_1.Level.WARN);
+                                    }
+                                }
+                            }
                         }));
                     }
                     yield page.close();
@@ -419,9 +430,8 @@ class Interpreter extends events_1.EventEmitter {
                     const scrapeResults = yield page.evaluate((s) => window.scrape(s !== null && s !== void 0 ? s : null), selector);
                     yield this.options.serializableCallback(scrapeResults);
                 }),
-                scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
+                scrapeSchema: (schema_1, ...args_1) => __awaiter(this, [schema_1, ...args_1], void 0, function* (schema, actionName = "") {
                     var _a;
-                    // Check abort flag at start of scraping
                     if (this.isAborted) {
                         this.log('Workflow aborted, stopping scrapeSchema', logger_1.Level.WARN);
                         return;
@@ -440,7 +450,6 @@ class Interpreter extends events_1.EventEmitter {
                     }
                     const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
                     if (this.cumulativeResults.length === 0) {
-                        // First execution - create initial row
                         const newRow = {};
                         Object.entries(resultToProcess).forEach(([key, value]) => {
                             if (value !== undefined) {
@@ -450,12 +459,10 @@ class Interpreter extends events_1.EventEmitter {
                         this.cumulativeResults.push(newRow);
                     }
                     else {
-                        // Check if any keys from new result already exist in the last row
                         const lastRow = this.cumulativeResults[this.cumulativeResults.length - 1];
                         const newResultKeys = Object.keys(resultToProcess).filter(key => resultToProcess[key] !== undefined);
                         const hasRepeatedKeys = newResultKeys.some(key => lastRow.hasOwnProperty(key));
                         if (hasRepeatedKeys) {
-                            // Keys are repeated - create a new row
                             const newRow = {};
                             Object.entries(resultToProcess).forEach(([key, value]) => {
                                 if (value !== undefined) {
@@ -465,7 +472,6 @@ class Interpreter extends events_1.EventEmitter {
                             this.cumulativeResults.push(newRow);
                         }
                         else {
-                            // No repeated keys - merge with the last row
                             Object.entries(resultToProcess).forEach(([key, value]) => {
                                 if (value !== undefined) {
                                     lastRow[key] = value;
@@ -473,30 +479,24 @@ class Interpreter extends events_1.EventEmitter {
                             });
                         }
                     }
-                    console.log("Total accumulated rows:", this.cumulativeResults.length);
-                    console.log("Current results:", this.cumulativeResults);
-                    // ✅ Append schema results under "scrapeSchema" → name
                     const actionType = "scrapeSchema";
-                    const actionName = schema.__name || "Texts";
+                    const name = actionName || "Texts";
                     if (!this.namedResults[actionType])
                         this.namedResults[actionType] = {};
-                    this.namedResults[actionType][actionName] = this.cumulativeResults;
+                    this.namedResults[actionType][name] = this.cumulativeResults;
                     if (!this.serializableDataByType[actionType])
                         this.serializableDataByType[actionType] = {};
-                    if (!this.serializableDataByType[actionType][actionName]) {
-                        this.serializableDataByType[actionType][actionName] = [];
+                    if (!this.serializableDataByType[actionType][name]) {
+                        this.serializableDataByType[actionType][name] = [];
                     }
-                    // Store as array (matching cumulativeResults structure)
-                    this.serializableDataByType[actionType][actionName] = [...this.cumulativeResults];
-                    // now emit full structured object
+                    this.serializableDataByType[actionType][name] = [...this.cumulativeResults];
                     yield this.options.serializableCallback({
                         scrapeList: this.serializableDataByType.scrapeList,
                         scrapeSchema: this.serializableDataByType.scrapeSchema
                     });
                 }),
-                scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
+                scrapeList: (config_1, ...args_1) => __awaiter(this, [config_1, ...args_1], void 0, function* (config, actionName = "") {
                     var _a, _b;
-                    // Check abort flag at start of scraping
                     if (this.isAborted) {
                         this.log('Workflow aborted, stopping scrapeList', logger_1.Level.WARN);
                         return;
@@ -522,36 +522,31 @@ class Interpreter extends events_1.EventEmitter {
                                 }
                                 catch (error) {
                                     console.warn('ScrapeList evaluation failed:', error.message);
-                                    return []; // Return empty array instead of failing
+                                    return [];
                                 }
                             }, config);
                         }
                         else {
                             paginationUsed = true;
-                            scrapeResults = yield this.handlePagination(page, config);
+                            scrapeResults = yield this.handlePagination(page, config, actionName);
                         }
-                        // Ensure we always have an array
                         if (!Array.isArray(scrapeResults)) {
                             scrapeResults = [];
                         }
                         console.log(`ScrapeList completed with ${scrapeResults.length} results`);
-                        // Only process and callback if pagination wasn't used
-                        // (handlePagination already handles storage and callbacks internally)
                         if (!paginationUsed) {
-                            // ✅ Append list results under "scrapeList" → name
                             const actionType = "scrapeList";
-                            let actionName = config.__name || "";
-                            // If no name provided, generate a unique one
-                            if (!actionName || actionName.trim() === "") {
+                            let name = actionName || "";
+                            if (!name || name.trim() === "") {
                                 this.scrapeListCounter++;
-                                actionName = `List ${this.scrapeListCounter}`;
+                                name = `List ${this.scrapeListCounter}`;
                             }
                             if (!this.serializableDataByType[actionType])
                                 this.serializableDataByType[actionType] = {};
-                            if (!this.serializableDataByType[actionType][actionName]) {
-                                this.serializableDataByType[actionType][actionName] = [];
+                            if (!this.serializableDataByType[actionType][name]) {
+                                this.serializableDataByType[actionType][name] = [];
                             }
-                            this.serializableDataByType[actionType][actionName].push(...scrapeResults);
+                            this.serializableDataByType[actionType][name].push(...scrapeResults);
                             yield this.options.serializableCallback({
                                 scrapeList: this.serializableDataByType.scrapeList,
                                 scrapeSchema: this.serializableDataByType.scrapeSchema
@@ -560,15 +555,18 @@ class Interpreter extends events_1.EventEmitter {
                     }
                     catch (error) {
                         console.error('ScrapeList action failed completely:', error.message);
-                        // Don't throw error, just return empty array
                         const actionType = "scrapeList";
-                        const actionName = config.__name || "List";
+                        let name = actionName || "";
+                        if (!name || name.trim() === "") {
+                            this.scrapeListCounter++;
+                            name = `List ${this.scrapeListCounter}`;
+                        }
                         if (!this.namedResults[actionType])
                             this.namedResults[actionType] = {};
-                        this.namedResults[actionType][actionName] = [];
+                        this.namedResults[actionType][name] = [];
                         if (!this.serializableDataByType[actionType])
                             this.serializableDataByType[actionType] = {};
-                        this.serializableDataByType[actionType][actionName] = [];
+                        this.serializableDataByType[actionType][name] = [];
                         yield this.options.serializableCallback({
                             scrapeList: this.serializableDataByType.scrapeList,
                             scrapeSchema: this.serializableDataByType.scrapeSchema
@@ -651,25 +649,7 @@ class Interpreter extends events_1.EventEmitter {
                     if (debug === null || debug === void 0 ? void 0 : debug.setActionType) {
                         debug.setActionType(String(step.action));
                     }
-                    // Safely extract name for this step
-                    if (step === null || step === void 0 ? void 0 : step.name) {
-                        stepName = step.name;
-                    }
-                    else if (Array.isArray(step === null || step === void 0 ? void 0 : step.args) &&
-                        step.args.length > 0 &&
-                        typeof step.args[0] === "object" &&
-                        "__name" in step.args[0]) {
-                        stepName = step.args[0].__name;
-                    }
-                    else if (typeof (step === null || step === void 0 ? void 0 : step.args) === "object" &&
-                        (step === null || step === void 0 ? void 0 : step.args) !== null &&
-                        "__name" in step.args) {
-                        stepName = step.args.__name;
-                    }
-                    // Default fallback
-                    if (!stepName) {
-                        stepName = String(step.action);
-                    }
+                    stepName = (step === null || step === void 0 ? void 0 : step.name) || String(step.action);
                     if (debug && typeof debug.setActionName === "function") {
                         debug.setActionName(stepName);
                     }
@@ -682,9 +662,12 @@ class Interpreter extends events_1.EventEmitter {
                         // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
                         const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
                         if (step.action === 'screenshot') {
-                            // call the screenshot handler directly to allow the extra name parameter
                             yield wawActions.screenshot(...(params !== null && params !== void 0 ? params : []), stepName !== null && stepName !== void 0 ? stepName : undefined);
                         }
+                        else if (step.action === 'scrapeList' || step.action === 'scrapeSchema') {
+                            const actionName = step.name || "";
+                            yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []), actionName);
+                        }
                         else {
                             yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
                         }
@@ -744,16 +727,14 @@ class Interpreter extends events_1.EventEmitter {
             }
         });
     }
-    handlePagination(page, config) {
-        return __awaiter(this, void 0, void 0, function* () {
-            // Check abort flag at start of pagination
+    handlePagination(page_1, config_1) {
+        return __awaiter(this, arguments, void 0, function* (page, config, providedActionName = "") {
             if (this.isAborted) {
                 this.log('Workflow aborted, stopping pagination', logger_1.Level.WARN);
                 return [];
             }
-            // Generate action name for this scrapeList
             const actionType = "scrapeList";
-            let actionName = config.__name || "";
+            let actionName = providedActionName || "";
             if (!actionName || actionName.trim() === "") {
                 this.scrapeListCounter++;
                 actionName = `List ${this.scrapeListCounter}`;
@@ -2224,9 +2205,10 @@ class Interpreter extends events_1.EventEmitter {
             * User-requested concurrency should be entirely managed by the concurrency manager,
             * e.g. via `enqueueLinks`.
             */
-            p.on('popup', (popup) => {
+            const popupHandler = (popup) => {
                 this.concurrency.addJob(() => this.runLoop(popup, workflowCopy));
-            });
+            };
+            p.on('popup', popupHandler);
             /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
             let loopIterations = 0;
             const MAX_LOOP_ITERATIONS = 1000; // Circuit breaker
@@ -2234,41 +2216,58 @@ class Interpreter extends events_1.EventEmitter {
             const MAX_CONSECUTIVE_FAILURES = 10;
             const startTime = Date.now();
             const MAX_EXECUTION_TIME = 30 * 60 * 1000; // 30 minutes max
+            // Cleanup function to remove popup listener
+            const cleanup = () => {
+                try {
+                    if (!p.isClosed()) {
+                        p.removeListener('popup', popupHandler);
+                    }
+                }
+                catch (cleanupError) {
+                }
+            };
             while (true) {
                 // Multiple circuit breakers to prevent infinite loops
                 if (++loopIterations > MAX_LOOP_ITERATIONS) {
                     this.log('Maximum loop iterations reached, terminating to prevent infinite loop', logger_1.Level.ERROR);
+                    cleanup();
                     return;
                 }
                 // Time-based circuit breaker
                 if (Date.now() - startTime > MAX_EXECUTION_TIME) {
                     this.log('Maximum execution time reached (30 minutes), terminating workflow', logger_1.Level.ERROR);
+                    cleanup();
                     return;
                 }
                 // Failure-based circuit breaker
                 if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
                     this.log('Too many consecutive failures, terminating to prevent hang', logger_1.Level.ERROR);
+                    cleanup();
                     return;
                 }
                 // Check abort flag immediately
                 if (this.isAborted) {
                     this.log('Workflow aborted in runLoop', logger_1.Level.WARN);
+                    cleanup();
                     return;
                 }
                 // Checks whether the page was closed from outside,
                 //  or the workflow execution has been stopped via `interpreter.stop()`
                 if (p.isClosed() || !this.stopper) {
+                    cleanup();
                     return;
                 }
                 try {
                     yield p.waitForLoadState();
                 }
                 catch (e) {
+                    cleanup();
                     yield p.close();
                     return;
                 }
                 if (workflowCopy.length === 0) {
                     this.log('All actions completed. Workflow finished.', logger_1.Level.LOG);
+                    cleanup();
                     return;
                 }
                 // const newSelectors = this.getSelectors(workflowCopy);
@@ -2359,6 +2358,7 @@ class Interpreter extends events_1.EventEmitter {
                 }
                 else {
                     //await this.disableAdBlocker(p);
+                    cleanup();
                     return;
                 }
             }
@@ -2444,5 +2444,47 @@ class Interpreter extends events_1.EventEmitter {
             }
         });
     }
+    /**
+     * Cleanup method to release resources and prevent memory leaks
+     * Call this when the interpreter is no longer needed
+     */
+    cleanup() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                // Stop any running workflows first
+                if (this.stopper) {
+                    try {
+                        yield this.stop();
+                    }
+                    catch (error) {
+                        this.log(`Error stopping workflow during cleanup: ${error.message}`, logger_1.Level.WARN);
+                    }
+                }
+                // Clear ad-blocker resources
+                if (this.blocker) {
+                    try {
+                        this.blocker = null;
+                        this.log('Ad-blocker resources cleared', logger_1.Level.DEBUG);
+                    }
+                    catch (error) {
+                        this.log(`Error cleaning up ad-blocker: ${error.message}`, logger_1.Level.WARN);
+                    }
+                }
+                // Clear accumulated data to free memory
+                this.cumulativeResults = [];
+                this.autohealFailures = [];
+                this.namedResults = {};
+                this.serializableDataByType = { scrapeList: {}, scrapeSchema: {} };
+                // Reset state
+                this.isAborted = false;
+                this.initializedWorkflow = null;
+                this.log('Interpreter cleanup completed', logger_1.Level.DEBUG);
+            }
+            catch (error) {
+                this.log(`Error during interpreter cleanup: ${error.message}`, logger_1.Level.ERROR);
+                throw error;
+            }
+        });
+    }
 }
 exports.default = Interpreter;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.22",
+  "version": "0.0.24",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",