maxun-core 0.0.4 → 0.0.6

package/build/browserSide/scraper.js

@@ -192,10 +192,11 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                 return undefined;
             switch (attribute) {
                 case 'href':
-                    const relativeHref = elem.getAttribute('href'); // Get the href attribute
-                    return relativeHref ? new URL(relativeHref, window.location.origin).href : null; // Convert to full URL
+                    const relativeHref = elem.getAttribute('href');
+                    return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
                 case 'src':
-                    return elem.getAttribute('src');
+                    const relativeSrc = elem.getAttribute('src');
+                    return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
                 case 'innerText':
                     return elem.innerText;
                 case 'textContent':
@@ -204,7 +205,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     return elem.innerText;
             }
         }, (key) => key // Use the original key in the output
-        ));
+        )) || [];
     };
     /**
    * Scrapes multiple lists of similar items based on a template item.
@@ -237,10 +238,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                                 record[label] = fieldElement.innerHTML.trim();
                             }
                             else if (attribute === 'src') {
-                                record[label] = fieldElement.src;
+                                // Handle relative 'src' URLs
+                                const src = fieldElement.getAttribute('src');
+                                record[label] = src ? new URL(src, window.location.origin).href : null;
                             }
                             else if (attribute === 'href') {
-                                record[label] = fieldElement.href;
+                                // Handle relative 'href' URLs
+                                const href = fieldElement.getAttribute('href');
+                                record[label] = href ? new URL(href, window.location.origin).href : null;
                             }
                             else {
                                 record[label] = fieldElement.getAttribute(attribute);

package/build/interpret.d.ts

@@ -2,6 +2,31 @@
 import { Page } from 'playwright';
 import { EventEmitter } from 'events';
 import { WorkflowFile, ParamType } from './types/workflow';
+/**
+ * Extending the Window interface for custom scraping functions.
+ */
+declare global {
+    interface Window {
+        scrape: (selector: string | null) => Record<string, string>[];
+        scrapeSchema: (schema: Record<string, {
+            selector: string;
+            tag: string;
+            attribute: string;
+        }>) => Record<string, any>;
+        scrapeList: (config: {
+            listSelector: string;
+            fields: any;
+            limit?: number;
+            pagination: any;
+        }) => Record<string, any>[];
+        scrapeListAuto: (listSelector: string) => {
+            selector: string;
+            innerText: string;
+        }[];
+        scrollDown: (pages?: number) => void;
+        scrollUp: (pages?: number) => void;
+    }
+}
 /**
  * Defines optional intepreter options (passed in constructor)
  */
@@ -27,9 +52,11 @@ export default class Interpreter extends EventEmitter {
     private stopper;
     private log;
     private blocker;
+    private cumulativeResults;
     constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>);
     private applyAdBlocker;
     private disableAdBlocker;
+    private getSelectors;
     /**
       * Returns the context object from given Page and the current workflow.\
       * \
@@ -58,6 +85,7 @@ export default class Interpreter extends EventEmitter {
    */
     private carryOutSteps;
     private handlePagination;
+    private getMatchingActionId;
     private runLoop;
     private ensureScriptsLoaded;
     /**

package/build/interpret.js

@@ -53,9 +53,12 @@ class Interpreter extends events_1.EventEmitter {
         super();
         this.stopper = null;
         this.blocker = null;
+        this.cumulativeResults = [];
         this.workflow = workflow.workflow;
         this.initializedWorkflow = null;
-        this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => { (0, logger_1.default)(JSON.stringify(data), logger_1.Level.WARN); }, binaryCallback: () => { (0, logger_1.default)('Received binary data, thrashing them.', logger_1.Level.WARN); }, debug: false, debugChannel: {} }, options);
+        this.options = Object.assign({ maxRepeats: 5, maxConcurrency: 5, serializableCallback: (data) => {
+                (0, logger_1.default)(JSON.stringify(data), logger_1.Level.WARN);
+            }, binaryCallback: () => { (0, logger_1.default)('Received binary data, thrashing them.', logger_1.Level.WARN); }, debug: false, debugChannel: {} }, options);
         this.concurrency = new concurrency_1.default(this.options.maxConcurrency);
         this.log = (...args) => (0, logger_1.default)(...args);
         const error = preprocessor_1.default.validateWorkflow(workflow);
@@ -72,7 +75,7 @@ class Interpreter extends events_1.EventEmitter {
                 oldLog(...args);
             };
         }
-        adblocker_playwright_1.PlaywrightBlocker.fromPrebuiltAdsAndTracking(cross_fetch_1.default).then(blocker => {
+        adblocker_playwright_1.PlaywrightBlocker.fromLists(cross_fetch_1.default, ['https://easylist.to/easylist/easylist.txt']).then(blocker => {
             this.blocker = blocker;
         }).catch(err => {
             this.log(`Failed to initialize ad-blocker:`, logger_1.Level.ERROR);
@@ -92,6 +95,43 @@ class Interpreter extends events_1.EventEmitter {
             }
         });
     }
+    // private getSelectors(workflow: Workflow, actionId: number): string[] {
+    //   const selectors: string[] = [];
+    //   // Validate actionId
+    //   if (actionId <= 0) {
+    //       console.log("No previous selectors to collect.");
+    //       return selectors; // Empty array as there are no previous steps
+    //   }
+    //   // Iterate from the start up to (but not including) actionId
+    //   for (let index = 0; index < actionId; index++) {
+    //       const currentSelectors = workflow[index]?.where?.selectors;
+    //       console.log(`Selectors at step ${index}:`, currentSelectors);
+    //       if (currentSelectors && currentSelectors.length > 0) {
+    //           currentSelectors.forEach((selector) => {
+    //               if (!selectors.includes(selector)) {
+    //                   selectors.push(selector); // Avoid duplicates
+    //               }
+    //           });
+    //       }
+    //   }
+    //   console.log("Collected Selectors:", selectors);
+    //   return selectors;
+    // }
+    getSelectors(workflow) {
+        var _a, _b;
+        const selectorsSet = new Set();
+        if (workflow.length === 0) {
+            return [];
+        }
+        for (let index = workflow.length - 1; index >= 0; index--) {
+            const currentSelectors = (_b = (_a = workflow[index]) === null || _a === void 0 ? void 0 : _a.where) === null || _b === void 0 ? void 0 : _b.selectors;
+            if (currentSelectors && currentSelectors.length > 0) {
+                currentSelectors.forEach((selector) => selectorsSet.add(selector));
+                return Array.from(selectorsSet);
+            }
+        }
+        return [];
+    }
     /**
       * Returns the context object from given Page and the current workflow.\
       * \
@@ -101,44 +141,52 @@ class Interpreter extends events_1.EventEmitter {
       * @param workflow Current **initialized** workflow (array of where-what pairs).
       * @returns {PageState} State of the current page.
       */
-    getState(page, workflow) {
+    getState(page, workflowCopy, selectors) {
         return __awaiter(this, void 0, void 0, function* () {
             /**
              * All the selectors present in the current Workflow
              */
-            const selectors = preprocessor_1.default.extractSelectors(workflow);
+            // const selectors = Preprocessor.extractSelectors(workflow);
+            // console.log("Current selectors:", selectors);
             /**
               * Determines whether the element targetted by the selector is [actionable](https://playwright.dev/docs/actionability).
               * @param selector Selector to be queried
               * @returns True if the targetted element is actionable, false otherwise.
               */
-            const actionable = (selector) => __awaiter(this, void 0, void 0, function* () {
-                try {
-                    const proms = [
-                        page.isEnabled(selector, { timeout: 500 }),
-                        page.isVisible(selector, { timeout: 500 }),
-                    ];
-                    return yield Promise.all(proms).then((bools) => bools.every((x) => x));
-                }
-                catch (e) {
-                    // log(<Error>e, Level.ERROR);
-                    return false;
-                }
-            });
+            // const actionable = async (selector: string): Promise<boolean> => {
+            //   try {
+            //     const proms = [
+            //       page.isEnabled(selector, { timeout: 5000 }),
+            //       page.isVisible(selector, { timeout: 5000 }),
+            //     ];
+            //     return await Promise.all(proms).then((bools) => bools.every((x) => x));
+            //   } catch (e) {
+            //     // log(<Error>e, Level.ERROR);
+            //     return false;
+            //   }
+            // };
             /**
               * Object of selectors present in the current page.
               */
-            const presentSelectors = yield Promise.all(selectors.map((selector) => __awaiter(this, void 0, void 0, function* () {
-                if (yield actionable(selector)) {
-                    return [selector];
-                }
-                return [];
-            }))).then((x) => x.flat());
+            // const presentSelectors: SelectorArray = await Promise.all(
+            //   selectors.map(async (selector) => {
+            //     if (await actionable(selector)) {
+            //       return [selector];
+            //     }
+            //     return [];
+            //   }),
+            // ).then((x) => x.flat());
+            const action = workflowCopy[workflowCopy.length - 1];
+            // console.log("Next action:", action)
+            let url = page.url();
+            if (action && action.where.url !== url && action.where.url !== "about:blank") {
+                url = action.where.url;
+            }
             return {
-                url: page.url(),
+                url,
                 cookies: (yield page.context().cookies([page.url()]))
                     .reduce((p, cookie) => (Object.assign(Object.assign({}, p), { [cookie.name]: cookie.value })), {}),
-                selectors: presentSelectors,
+                selectors,
             };
         });
     }
@@ -225,7 +273,6 @@ class Interpreter extends events_1.EventEmitter {
    * @param steps Array of actions.
    */
     carryOutSteps(page, steps) {
-        var _a;
         return __awaiter(this, void 0, void 0, function* () {
             /**
              * Defines overloaded (or added) methods/actions usable in the workflow.
@@ -273,7 +320,31 @@ class Interpreter extends events_1.EventEmitter {
                 scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResult = yield page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
-                    yield this.options.serializableCallback(scrapeResult);
+                    const newResults = Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult];
+                    newResults.forEach((result) => {
+                        Object.entries(result).forEach(([key, value]) => {
+                            const keyExists = this.cumulativeResults.some((item) => key in item && item[key] !== undefined);
+                            if (!keyExists) {
+                                this.cumulativeResults.push({ [key]: value });
+                            }
+                        });
+                    });
+                    const mergedResult = [
+                        Object.fromEntries(Object.entries(this.cumulativeResults.reduce((acc, curr) => {
+                            Object.entries(curr).forEach(([key, value]) => {
+                                // If the key doesn't exist or the current value is not undefined, add/update it
+                                if (value !== undefined) {
+                                    acc[key] = value;
+                                }
+                            });
+                            return acc;
+                        }, {})))
+                    ];
+                    // Log cumulative results after each action
+                    console.log("CUMULATIVE results:", this.cumulativeResults);
+                    console.log("MERGED results:", mergedResult);
+                    yield this.options.serializableCallback(mergedResult);
+                    // await this.options.serializableCallback(scrapeResult);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
                     yield this.ensureScriptsLoaded(page);
@@ -312,8 +383,17 @@ class Interpreter extends events_1.EventEmitter {
                     });
                 }),
             };
+            const executeAction = (invokee, methodName, args) => __awaiter(this, void 0, void 0, function* () {
+                console.log("Executing action:", methodName, args);
+                if (!args || Array.isArray(args)) {
+                    yield invokee[methodName](...(args !== null && args !== void 0 ? args : []));
+                }
+                else {
+                    yield invokee[methodName](args);
+                }
+            });
             for (const step of steps) {
-                this.log(`Launching ${step.action}`, logger_1.Level.LOG);
+                this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
                 if (step.action in wawActions) {
                     // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
                     const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
@@ -321,17 +401,30 @@ class Interpreter extends events_1.EventEmitter {
                 }
                 else {
                     // Implements the dot notation for the "method name" in the workflow
-                    const levels = step.action.split('.');
+                    const levels = String(step.action).split('.');
                     const methodName = levels[levels.length - 1];
                     let invokee = page;
                     for (const level of levels.splice(0, levels.length - 1)) {
                         invokee = invokee[level];
                     }
-                    if (!step.args || Array.isArray(step.args)) {
-                        yield invokee[methodName](...((_a = step.args) !== null && _a !== void 0 ? _a : []));
+                    if (methodName === 'waitForLoadState') {
+                        try {
+                            yield executeAction(invokee, methodName, step.args);
+                        }
+                        catch (error) {
+                            yield executeAction(invokee, methodName, 'domcontentloaded');
+                        }
+                    }
+                    else if (methodName === 'click') {
+                        try {
+                            yield executeAction(invokee, methodName, step.args);
+                        }
+                        catch (error) {
+                            yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                        }
                     }
                     else {
-                        yield invokee[methodName](step.args);
+                        yield executeAction(invokee, methodName, step.args);
                     }
                 }
                 yield new Promise((res) => { setTimeout(res, 500); });
@@ -370,6 +463,7 @@ class Interpreter extends events_1.EventEmitter {
                         break;
                     case 'clickNext':
                         const pageResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                        // console.log("Page results:", pageResults);
                         // Filter out already scraped items
                         const newResults = pageResults.filter(item => {
                             const uniqueKey = JSON.stringify(item);
@@ -387,7 +481,7 @@ class Interpreter extends events_1.EventEmitter {
                             return allResults; // No more pages to scrape
                         }
                         yield Promise.all([
-                            nextButton.click(),
+                            nextButton.dispatchEvent('click'),
                             page.waitForNavigation({ waitUntil: 'networkidle' })
                         ]);
                         yield page.waitForTimeout(1000);
@@ -402,7 +496,7 @@ class Interpreter extends events_1.EventEmitter {
                                 return allResults;
                             }
                             // Click the 'Load More' button to load additional items
-                            yield loadMoreButton.click();
+                            yield loadMoreButton.dispatchEvent('click');
                             yield page.waitForTimeout(2000); // Wait for new items to load
                             // After clicking 'Load More', scroll down to load more items
                             yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
@@ -436,13 +530,30 @@ class Interpreter extends events_1.EventEmitter {
             return allResults;
         });
     }
+    getMatchingActionId(workflow, pageState, usedActions) {
+        for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
+            const step = workflow[actionId];
+            const isApplicable = this.applicable(step.where, pageState, usedActions);
+            console.log("-------------------------------------------------------------");
+            console.log(`Where:`, step.where);
+            console.log(`Page state:`, pageState);
+            console.log(`Match result: ${isApplicable}`);
+            console.log("-------------------------------------------------------------");
+            if (isApplicable) {
+                return actionId;
+            }
+        }
+    }
     runLoop(p, workflow) {
         var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {
+            const workflowCopy = JSON.parse(JSON.stringify(workflow));
             // apply ad-blocker to the current page
             yield this.applyAdBlocker(p);
             const usedActions = [];
+            let selectors = [];
             let lastAction = null;
+            let actionId = -1;
             let repeatCount = 0;
             /**
             *  Enables the interpreter functionality for popup windows.
@@ -450,7 +561,7 @@ class Interpreter extends events_1.EventEmitter {
             * e.g. via `enqueueLinks`.
             */
             p.on('popup', (popup) => {
-                this.concurrency.addJob(() => this.runLoop(popup, workflow));
+                this.concurrency.addJob(() => this.runLoop(popup, workflowCopy));
             });
             /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */
             while (true) {
@@ -467,8 +578,11 @@ class Interpreter extends events_1.EventEmitter {
                     return;
                 }
                 let pageState = {};
+                let getStateTest = "Hello";
                 try {
-                    pageState = yield this.getState(p, workflow);
+                    pageState = yield this.getState(p, workflowCopy, selectors);
+                    selectors = [];
+                    console.log("Empty selectors:", selectors);
                 }
                 catch (e) {
                     this.log('The browser has been closed.');
@@ -477,21 +591,43 @@ class Interpreter extends events_1.EventEmitter {
                 if (this.options.debug) {
                     this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, logger_1.Level.WARN);
                 }
-                const actionId = workflow.findIndex((step) => this.applicable(step.where, pageState, usedActions));
-                const action = workflow[actionId];
+                // const actionId = workflow.findIndex((step) => {
+                //   const isApplicable = this.applicable(step.where, pageState, usedActions);
+                //   console.log("-------------------------------------------------------------");
+                //   console.log(`Where:`, step.where);
+                //   console.log(`Page state:`, pageState);
+                //   console.log(`Match result: ${isApplicable}`);
+                //   console.log("-------------------------------------------------------------");
+                //   return isApplicable;
+                // });
+                actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
+                const action = workflowCopy[actionId];
+                console.log("MATCHED ACTION:", action);
+                console.log("MATCHED ACTION ID:", actionId);
                 this.log(`Matched ${JSON.stringify(action === null || action === void 0 ? void 0 : action.where)}`, logger_1.Level.LOG);
                 if (action) { // action is matched
                     if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.activeId) {
                         this.options.debugChannel.activeId(actionId);
                     }
                     repeatCount = action === lastAction ? repeatCount + 1 : 0;
-                    if (this.options.maxRepeats && repeatCount >= this.options.maxRepeats) {
+                    console.log("REPEAT COUNT", repeatCount);
+                    if (this.options.maxRepeats && repeatCount > this.options.maxRepeats) {
                         return;
                     }
                     lastAction = action;
                     try {
+                        console.log("Carrying out:", action.what);
                         yield this.carryOutSteps(p, action.what);
                         usedActions.push((_b = action.id) !== null && _b !== void 0 ? _b : 'undefined');
+                        workflowCopy.splice(actionId, 1);
+                        console.log(`Action with ID ${action.id} removed from the workflow copy.`);
+                        // const newSelectors = this.getPreviousSelectors(workflow, actionId);
+                        const newSelectors = this.getSelectors(workflowCopy);
+                        newSelectors.forEach(selector => {
+                            if (!selectors.includes(selector)) {
+                                selectors.push(selector);
+                            }
+                        });
                     }
                     catch (e) {
                         this.log(e, logger_1.Level.ERROR);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",