maxun-core 0.0.21 → 0.0.23

package/build/interpret.d.ts

@@ -53,10 +53,15 @@ export default class Interpreter extends EventEmitter {
     private options;
     private concurrency;
     private stopper;
+    private isAborted;
     private log;
     private blocker;
     private cumulativeResults;
     constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
+    /**
+     * Sets the abort flag to immediately stop all operations
+     */
+    abort(): void;
     private applyAdBlocker;
     private disableAdBlocker;
     private getSelectors;

package/build/interpret.js

@@ -52,6 +52,7 @@ class Interpreter extends events_1.EventEmitter {
         var _a;
         super();
         this.stopper = null;
+        this.isAborted = false;
         this.blocker = null;
         this.cumulativeResults = [];
         this.workflow = workflow.workflow;
@@ -83,6 +84,12 @@ class Interpreter extends events_1.EventEmitter {
             this.blocker = null;
         });
     }
+    /**
+     * Sets the abort flag to immediately stop all operations
+     */
+    abort() {
+        this.isAborted = true;
+    }
     applyAdBlocker(page) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.blocker) {
@@ -304,6 +311,10 @@ class Interpreter extends events_1.EventEmitter {
     carryOutSteps(page, steps) {
         var _a;
         return __awaiter(this, void 0, void 0, function* () {
+            if (this.isAborted) {
+                this.log('Workflow aborted, stopping execution', logger_1.Level.WARN);
+                return;
+            }
             /**
              * Defines overloaded (or added) methods/actions usable in the workflow.
              * If a method overloads any existing method of the Page class, it accepts the same set
@@ -361,6 +372,10 @@ class Interpreter extends events_1.EventEmitter {
                 }),
                 scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
                     var _e;
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping scrapeSchema', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_e = this.options.debugChannel) === null || _e === void 0 ? void 0 : _e.setActionType) {
                         this.options.debugChannel.setActionType('scrapeSchema');
                     }
@@ -388,6 +403,10 @@ class Interpreter extends events_1.EventEmitter {
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
                     var _f, _g;
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping scrapeList', logger_1.Level.WARN);
+                        return;
+                    }
                     if ((_f = this.options.debugChannel) === null || _f === void 0 ? void 0 : _f.setActionType) {
                         this.options.debugChannel.setActionType('scrapeList');
                     }
@@ -528,6 +547,10 @@ class Interpreter extends events_1.EventEmitter {
     }
     handlePagination(page, config) {
         return __awaiter(this, void 0, void 0, function* () {
+            if (this.isAborted) {
+                this.log('Workflow aborted, stopping pagination', logger_1.Level.WARN);
+                return [];
+            }
             let allResults = [];
             let previousHeight = 0;
             let scrapedItems = new Set();
@@ -539,6 +562,11 @@ class Interpreter extends events_1.EventEmitter {
                 console.log(`[Page ${visitedUrls.size}] [URL: ${page.url()}] ${message}`, ...args);
             };
             const scrapeCurrentPage = () => __awaiter(this, void 0, void 0, function* () {
+                // Check abort flag before scraping current page
+                if (this.isAborted) {
+                    debugLog("Workflow aborted, stopping scrapeCurrentPage");
+                    return;
+                }
                 const results = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
                 const newResults = results.filter(item => {
                     const uniqueKey = JSON.stringify(item);
@@ -617,6 +645,10 @@ class Interpreter extends events_1.EventEmitter {
             let unchangedResultCounter = 0;
             try {
                 while (true) {
+                    if (this.isAborted) {
+                        this.log('Workflow aborted during pagination loop', logger_1.Level.WARN);
+                        return allResults;
+                    }
                     switch (config.pagination.type) {
                         case 'scrollDown': {
                             let previousResultCount = allResults.length;
@@ -624,9 +656,14 @@ class Interpreter extends events_1.EventEmitter {
                             if (checkLimit()) {
                                 return allResults;
                             }
-                            yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                            yield page.evaluate(() => {
+                                const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                window.scrollTo(0, scrollHeight);
+                            });
                             yield page.waitForTimeout(2000);
-                            const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                            const currentHeight = yield page.evaluate(() => {
+                                return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                            });
                             const currentResultCount = allResults.length;
                             if (currentResultCount === previousResultCount) {
                                 unchangedResultCounter++;
@@ -832,6 +869,10 @@ class Interpreter extends events_1.EventEmitter {
                             // let noNewItemsCounter = 0;
                             // const MAX_NO_NEW_ITEMS = 2;
                             while (true) {
+                                if (this.isAborted) {
+                                    this.log('Workflow aborted during pagination loop', logger_1.Level.WARN);
+                                    return allResults;
+                                }
                                 // Find working button with retry mechanism
                                 const { button: loadMoreButton, workingSelector, updatedSelectors } = yield findWorkingButton(availableSelectors);
                                 availableSelectors = updatedSelectors;
@@ -881,9 +922,14 @@ class Interpreter extends events_1.EventEmitter {
                                 }
                                 // Wait for content to load and check scroll height
                                 yield page.waitForTimeout(2000);
-                                yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+                                yield page.evaluate(() => {
+                                    const scrollHeight = Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                    window.scrollTo(0, scrollHeight);
+                                });
                                 yield page.waitForTimeout(2000);
-                                const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                                const currentHeight = yield page.evaluate(() => {
+                                    return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight);
+                                });
                                 const heightChanged = currentHeight !== previousHeight;
                                 previousHeight = currentHeight;
                                 yield scrapeCurrentPage();
@@ -962,6 +1008,10 @@ class Interpreter extends events_1.EventEmitter {
     runLoop(p, workflow) {
         var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {
+            if (this.isAborted) {
+                this.log('Workflow aborted in runLoop', logger_1.Level.WARN);
+                return;
+            }
             let workflowCopy = JSON.parse(JSON.stringify(workflow));
             workflowCopy = this.removeSpecialSelectors(workflowCopy);
             // apply ad-blocker to the current page
@@ -988,6 +1038,10 @@ class Interpreter extends events_1.EventEmitter {
             let loopIterations = 0;
             const MAX_LOOP_ITERATIONS = 1000; // Circuit breaker
             while (true) {
+                if (this.isAborted) {
+                    this.log('Workflow aborted during step execution', logger_1.Level.WARN);
+                    return;
+                }
                 // Circuit breaker to prevent infinite loops
                 if (++loopIterations > MAX_LOOP_ITERATIONS) {
                     this.log('Maximum loop iterations reached, terminating to prevent infinite loop', logger_1.Level.ERROR);
@@ -1005,30 +1059,33 @@ class Interpreter extends events_1.EventEmitter {
                     yield p.close();
                     return;
                 }
-                let pageState = {};
-                try {
-                    // Check if page is still valid before accessing state
-                    if (p.isClosed()) {
-                        this.log('Page was closed during execution', logger_1.Level.WARN);
-                        return;
-                    }
-                    pageState = yield this.getState(p, workflowCopy, selectors);
-                    selectors = [];
-                    console.log("Empty selectors:", selectors);
-                }
-                catch (e) {
-                    this.log(`Failed to get page state: ${e.message}`, logger_1.Level.ERROR);
-                    // If state access fails, attempt graceful recovery
-                    if (p.isClosed()) {
-                        this.log('Browser has been closed, terminating workflow', logger_1.Level.WARN);
-                        return;
-                    }
-                    // For other errors, continue with empty state to avoid complete failure
-                    pageState = { url: p.url(), selectors: [], cookies: {} };
-                }
-                if (this.options.debug) {
-                    this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, logger_1.Level.WARN);
+                if (workflowCopy.length === 0) {
+                    this.log('All actions completed. Workflow finished.', logger_1.Level.LOG);
+                    return;
                 }
+                // let pageState = {};
+                // try {
+                //   // Check if page is still valid before accessing state
+                //   if (p.isClosed()) {
+                //     this.log('Page was closed during execution', Level.WARN);
+                //     return;
+                //   }
+                //   pageState = await this.getState(p, workflowCopy, selectors);
+                //   selectors = [];
+                //   console.log("Empty selectors:", selectors)
+                // } catch (e: any) {
+                //   this.log(`Failed to get page state: ${e.message}`, Level.ERROR);
+                //   // If state access fails, attempt graceful recovery
+                //   if (p.isClosed()) {
+                //     this.log('Browser has been closed, terminating workflow', Level.WARN);
+                //     return;
+                //   }
+                //   // For other errors, continue with empty state to avoid complete failure
+                //   pageState = { url: p.url(), selectors: [], cookies: {} };
+                // }
+                // if (this.options.debug) {
+                //   this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, Level.WARN);
+                // }
                 // const actionId = workflow.findIndex((step) => {
                 //   const isApplicable = this.applicable(step.where, pageState, usedActions);
                 //   console.log("-------------------------------------------------------------");
@@ -1038,7 +1095,8 @@ class Interpreter extends events_1.EventEmitter {
                 //   console.log("-------------------------------------------------------------");
                 //   return isApplicable;
                 // });
-                actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
+                // actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
+                const actionId = workflowCopy.length - 1;
                 const action = workflowCopy[actionId];
                 console.log("MATCHED ACTION:", action);
                 console.log("MATCHED ACTION ID:", actionId);
@@ -1053,6 +1111,10 @@ class Interpreter extends events_1.EventEmitter {
                         return;
                     }
                     lastAction = action;
+                    if (this.isAborted) {
+                        this.log('Workflow aborted before action execution', logger_1.Level.WARN);
+                        return;
+                    }
                     try {
                         console.log("Carrying out:", action.what);
                         yield this.carryOutSteps(p, action.what);
@@ -1060,12 +1122,12 @@ class Interpreter extends events_1.EventEmitter {
                         workflowCopy.splice(actionId, 1);
                         console.log(`Action with ID ${action.id} removed from the workflow copy.`);
                         // const newSelectors = this.getPreviousSelectors(workflow, actionId);
-                        const newSelectors = this.getSelectors(workflowCopy);
-                        newSelectors.forEach(selector => {
-                            if (!selectors.includes(selector)) {
-                                selectors.push(selector);
-                            }
-                        });
+                        // const newSelectors = this.getSelectors(workflowCopy);
+                        // newSelectors.forEach(selector => {
+                        //     if (!selectors.includes(selector)) {
+                        //         selectors.push(selector);
+                        //     }
+                        // });
                         // Reset loop iteration counter on successful action
                         loopIterations = 0;
                     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.21",
+  "version": "0.0.23",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",