maxun-core 0.0.5 → 0.0.6

package/build/browserSide/scraper.js

@@ -240,12 +240,12 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                             else if (attribute === 'src') {
                                 // Handle relative 'src' URLs
                                 const src = fieldElement.getAttribute('src');
-                                record[label] = src ? new URL(src, baseUrl).href : null;
+                                record[label] = src ? new URL(src, window.location.origin).href : null;
                             }
                             else if (attribute === 'href') {
                                 // Handle relative 'href' URLs
                                 const href = fieldElement.getAttribute('href');
-                                record[label] = href ? new URL(href, baseUrl).href : null;
+                                record[label] = href ? new URL(href, window.location.origin).href : null;
                             }
                             else {
                                 record[label] = fieldElement.getAttribute(attribute);

package/build/interpret.d.ts

@@ -56,6 +56,7 @@ export default class Interpreter extends EventEmitter {
     constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
     private applyAdBlocker;
     private disableAdBlocker;
+    private getSelectors;
     /**
       * Returns the context object from given Page and the current workflow.\
       * \
@@ -84,6 +85,7 @@ export default class Interpreter extends EventEmitter {
    */
     private carryOutSteps;
     private handlePagination;
+    private getMatchingActionId;
     private runLoop;
     private ensureScriptsLoaded;
     /**

package/build/interpret.js

@@ -75,7 +75,7 @@ class Interpreter extends events_1.EventEmitter {
                 oldLog(...args);
             };
         }
-        adblocker_playwright_1.PlaywrightBlocker.fromPrebuiltAdsAndTracking(cross_fetch_1.default).then(blocker => {
+        adblocker_playwright_1.PlaywrightBlocker.fromLists(cross_fetch_1.default, ['https://easylist.to/easylist/easylist.txt']).then(blocker => {
             this.blocker = blocker;
         }).catch(err => {
             this.log(`Failed to initialize ad-blocker:`, logger_1.Level.ERROR);
@@ -95,6 +95,43 @@ class Interpreter extends events_1.EventEmitter {
             }
         });
     }
+    // private getSelectors(workflow: Workflow, actionId: number): string[] {
+    //   const selectors: string[] = [];
+    //   // Validate actionId
+    //   if (actionId <= 0) {
+    //       console.log("No previous selectors to collect.");
+    //       return selectors; // Empty array as there are no previous steps
+    //   }
+    //   // Iterate from the start up to (but not including) actionId
+    //   for (let index = 0; index < actionId; index++) {
+    //       const currentSelectors = workflow[index]?.where?.selectors;
+    //       console.log(`Selectors at step ${index}:`, currentSelectors);
+    //       if (currentSelectors && currentSelectors.length > 0) {
+    //           currentSelectors.forEach((selector) => {
+    //               if (!selectors.includes(selector)) {
+    //                   selectors.push(selector); // Avoid duplicates
+    //               }
+    //           });
+    //       }
+    //   }
+    //   console.log("Collected Selectors:", selectors);
+    //   return selectors;
+    // }
+    getSelectors(workflow) {
+        var _a, _b;
+        const selectorsSet = new Set();
+        if (workflow.length === 0) {
+            return [];
+        }
+        for (let index = workflow.length - 1; index >= 0; index--) {
+            const currentSelectors = (_b = (_a = workflow[index]) === null || _a === void 0 ? void 0 : _a.where) === null || _b === void 0 ? void 0 : _b.selectors;
+            if (currentSelectors && currentSelectors.length > 0) {
+                currentSelectors.forEach((selector) => selectorsSet.add(selector));
+                return Array.from(selectorsSet);
+            }
+        }
+        return [];
+    }
     /**
       * Returns the context object from given Page and the current workflow.\
       * \
@@ -104,44 +141,52 @@ class Interpreter extends events_1.EventEmitter {
       * @param workflow Current **initialized** workflow (array of where-what pairs).
       * @returns {PageState} State of the current page.
       */
-    getState(page, workflow) {
+    getState(page, workflowCopy, selectors) {
         return __awaiter(this, void 0, void 0, function* () {
             /**
              * All the selectors present in the current Workflow
              */
-            const selectors = preprocessor_1.default.extractSelectors(workflow);
+            // const selectors = Preprocessor.extractSelectors(workflow);
+            // console.log("Current selectors:", selectors);
             /**
               * Determines whether the element targetted by the selector is [actionable](https://playwright.dev/docs/actionability).
               * @param selector Selector to be queried
               * @returns True if the targetted element is actionable, false otherwise.
               */
-            const actionable = (selector) => __awaiter(this, void 0, void 0, function* () {
-                try {
-                    const proms = [
-                        page.isEnabled(selector, { timeout: 500 }),
-                        page.isVisible(selector, { timeout: 500 }),
-                    ];
-                    return yield Promise.all(proms).then((bools) => bools.every((x) => x));
-                }
-                catch (e) {
-                    // log(<Error>e, Level.ERROR);
-                    return false;
-                }
-            });
+            // const actionable = async (selector: string): Promise<boolean> => {
+            //   try {
+            //     const proms = [
+            //       page.isEnabled(selector, { timeout: 5000 }),
+            //       page.isVisible(selector, { timeout: 5000 }),
+            //     ];
+            //     return await Promise.all(proms).then((bools) => bools.every((x) => x));
+            //   } catch (e) {
+            //     // log(<Error>e, Level.ERROR);
+            //     return false;
+            //   }
+            // };
             /**
               * Object of selectors present in the current page.
               */
-            const presentSelectors = yield Promise.all(selectors.map((selector) => __awaiter(this, void 0, void 0, function* () {
-                if (yield actionable(selector)) {
-                    return [selector];
-                }
-                return [];
-            }))).then((x) => x.flat());
+            // const presentSelectors: SelectorArray = await Promise.all(
+            //   selectors.map(async (selector) => {
+            //     if (await actionable(selector)) {
+            //       return [selector];
+            //     }
+            //     return [];
+            //   }),
+            // ).then((x) => x.flat());
+            const action = workflowCopy[workflowCopy.length - 1];
+            // console.log("Next action:", action)
+            let url = page.url();
+            if (action && action.where.url !== url && action.where.url !== "about:blank") {
+                url = action.where.url;
+            }
             return {
-                url: page.url(),
+                url,
                 cookies: (yield page.context().cookies([page.url()]))
                     .reduce((p, cookie) => (Object.assign(Object.assign({}, p), { [cookie.name]: cookie.value })), {}),
-                selectors: presentSelectors,
+                selectors,
             };
         });
     }
@@ -228,7 +273,6 @@ class Interpreter extends events_1.EventEmitter {
    * @param steps Array of actions.
    */
     carryOutSteps(page, steps) {
-        var _a;
         return __awaiter(this, void 0, void 0, function* () {
             /**
              * Defines overloaded (or added) methods/actions usable in the workflow.
@@ -300,6 +344,7 @@ class Interpreter extends events_1.EventEmitter {
                     console.log("CUMULATIVE results:", this.cumulativeResults);
                     console.log("MERGED results:", mergedResult);
                     yield this.options.serializableCallback(mergedResult);
+                    // await this.options.serializableCallback(scrapeResult);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
                     yield this.ensureScriptsLoaded(page);
@@ -338,6 +383,15 @@ class Interpreter extends events_1.EventEmitter {
                     });
                 }),
             };
+            const executeAction = (invokee, methodName, args) => __awaiter(this, void 0, void 0, function* () {
+                console.log("Executing action:", methodName, args);
+                if (!args || Array.isArray(args)) {
+                    yield invokee[methodName](...(args !== null && args !== void 0 ? args : []));
+                }
+                else {
+                    yield invokee[methodName](args);
+                }
+            });
             for (const step of steps) {
                 this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
                 if (step.action in wawActions) {
@@ -353,11 +407,24 @@ class Interpreter extends events_1.EventEmitter {
                     for (const level of levels.splice(0, levels.length - 1)) {
                         invokee = invokee[level];
                     }
-                    if (!step.args || Array.isArray(step.args)) {
-                        yield invokee[methodName](...((_a = step.args) !== null && _a !== void 0 ? _a : []));
+                    if (methodName === 'waitForLoadState') {
+                        try {
+                            yield executeAction(invokee, methodName, step.args);
+                        }
+                        catch (error) {
+                            yield executeAction(invokee, methodName, 'domcontentloaded');
+                        }
+                    }
+                    else if (methodName === 'click') {
+                        try {
+                            yield executeAction(invokee, methodName, step.args);
+                        }
+                        catch (error) {
+                            yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                        }
                     }
                     else {
-                        yield invokee[methodName](step.args);
+                        yield executeAction(invokee, methodName, step.args);
                     }
                 }
                 yield new Promise((res) => { setTimeout(res, 500); });
@@ -396,6 +463,7 @@ class Interpreter extends events_1.EventEmitter {
                         break;
                     case 'clickNext':
                         const pageResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                        // console.log("Page results:", pageResults);
                         // Filter out already scraped items
                         const newResults = pageResults.filter(item => {
                             const uniqueKey = JSON.stringify(item);
@@ -413,7 +481,7 @@ class Interpreter extends events_1.EventEmitter {
                             return allResults; // No more pages to scrape
                         }
                         yield Promise.all([
-                            nextButton.click(),
+                            nextButton.dispatchEvent('click'),
                             page.waitForNavigation({ waitUntil: 'networkidle' })
                         ]);
                         yield page.waitForTimeout(1000);
@@ -428,7 +496,7 @@ class Interpreter extends events_1.EventEmitter {
                                 return allResults;
                             }
                             // Click the 'Load More' button to load additional items
-                            yield loadMoreButton.click();
+                            yield loadMoreButton.dispatchEvent('click');
                             yield page.waitForTimeout(2000); // Wait for new items to load
                             // After clicking 'Load More', scroll down to load more items
                             yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
@@ -462,13 +530,30 @@ class Interpreter extends events_1.EventEmitter {
             return allResults;
         });
     }
+    getMatchingActionId(workflow, pageState, usedActions) {
+        for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
+            const step = workflow[actionId];
+            const isApplicable = this.applicable(step.where, pageState, usedActions);
+            console.log("-------------------------------------------------------------");
+            console.log(`Where:`, step.where);
+            console.log(`Page state:`, pageState);
+            console.log(`Match result: ${isApplicable}`);
+            console.log("-------------------------------------------------------------");
+            if (isApplicable) {
+                return actionId;
+            }
+        }
+    }
     runLoop(p, workflow) {
         var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {
+            const workflowCopy = JSON.parse(JSON.stringify(workflow));
             // apply ad-blocker to the current page
             yield this.applyAdBlocker(p);
             const usedActions = [];
+            let selectors = [];
             let lastAction = null;
+            let actionId = -1;
             let repeatCount = 0;
             /**
             *  Enables the interpreter functionality for popup windows.
@@ -476,7 +561,7 @@ class Interpreter extends events_1.EventEmitter {
             * e.g. via `enqueueLinks`.
             */
             p.on('popup', (popup) => {
-                this.concurrency.addJob(() => this.runLoop(popup, workflow));
+                this.concurrency.addJob(() => this.runLoop(popup, workflowCopy));
             });
             /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
             while (true) {
@@ -493,8 +578,11 @@ class Interpreter extends events_1.EventEmitter {
                     return;
                 }
                 let pageState = {};
+                let getStateTest = "Hello";
                 try {
-                    pageState = yield this.getState(p, workflow);
+                    pageState = yield this.getState(p, workflowCopy, selectors);
+                    selectors = [];
+                    console.log("Empty selectors:", selectors);
                 }
                 catch (e) {
                     this.log('The browser has been closed.');
@@ -503,27 +591,43 @@ class Interpreter extends events_1.EventEmitter {
                 if (this.options.debug) {
                     this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, logger_1.Level.WARN);
                 }
-                const actionId = workflow.findIndex((step) => {
-                    const isApplicable = this.applicable(step.where, pageState, usedActions);
-                    console.log(`Where:`, step.where);
-                    console.log(`Page state:`, pageState);
-                    console.log(`Match result: ${isApplicable}`);
-                    return isApplicable;
-                });
-                const action = workflow[actionId];
+                // const actionId = workflow.findIndex((step) => {
+                //   const isApplicable = this.applicable(step.where, pageState, usedActions);
+                //   console.log("-------------------------------------------------------------");
+                //   console.log(`Where:`, step.where);
+                //   console.log(`Page state:`, pageState);
+                //   console.log(`Match result: ${isApplicable}`);
+                //   console.log("-------------------------------------------------------------");
+                //   return isApplicable;
+                // });
+                actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
+                const action = workflowCopy[actionId];
+                console.log("MATCHED ACTION:", action);
+                console.log("MATCHED ACTION ID:", actionId);
                 this.log(`Matched ${JSON.stringify(action === null || action === void 0 ? void 0 : action.where)}`, logger_1.Level.LOG);
                 if (action) { // action is matched
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.activeId) {
                         this.options.debugChannel.activeId(actionId);
                     }
                     repeatCount = action === lastAction ? repeatCount + 1 : 0;
-                    if (this.options.maxRepeats && repeatCount >= this.options.maxRepeats) {
+                    console.log("REPEAT COUNT", repeatCount);
+                    if (this.options.maxRepeats && repeatCount > this.options.maxRepeats) {
                         return;
                     }
                     lastAction = action;
                     try {
+                        console.log("Carrying out:", action.what);
                         yield this.carryOutSteps(p, action.what);
                         usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
+                        workflowCopy.splice(actionId, 1);
+                        console.log(`Action with ID ${action.id} removed from the workflow copy.`);
+                        // const newSelectors = this.getPreviousSelectors(workflow, actionId);
+                        const newSelectors = this.getSelectors(workflowCopy);
+                        newSelectors.forEach(selector => {
+                            if (!selectors.includes(selector)) {
+                                selectors.push(selector);
+                            }
+                        });
                     }
                     catch (e) {
                         this.log(e, logger_1.Level.ERROR);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",