maxun-core 0.0.15 → 0.0.16

package/build/browserSide/scraper.js

@@ -466,6 +466,20 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     return element.innerHTML.trim();
                 }
                 else if (attribute === 'src' || attribute === 'href') {
+                    if (attribute === 'href' && element.tagName !== 'A') {
+                        const parentElement = element.parentElement;
+                        if (parentElement && parentElement.tagName === 'A') {
+                            const parentHref = parentElement.getAttribute('href');
+                            if (parentHref) {
+                                try {
+                                    return new URL(parentHref, baseURL).href;
+                                }
+                                catch (e) {
+                                    return parentHref;
+                                }
+                            }
+                        }
+                    }
                     const attrValue = element.getAttribute(attribute);
                     const dataAttr = attrValue || element.getAttribute('data-' + attribute);
                     if (!dataAttr || dataAttr.trim() === '') {

package/build/interpret.d.ts

@@ -31,14 +31,16 @@ declare global {
  * Defines optional intepreter options (passed in constructor)
  */
 interface InterpreterOptions {
+    mode?: string;
     maxRepeats: number;
     maxConcurrency: number;
     serializableCallback: (output: any) => (void | Promise<void>);
     binaryCallback: (output: any, mimeType: string) => (void | Promise<void>);
     debug: boolean;
     debugChannel: Partial<{
-        activeId: Function;
-        debugMessage: Function;
+        activeId: (id: number) => void;
+        debugMessage: (msg: string) => void;
+        setActionType: (type: string) => void;
     }>;
 }
 /**

package/build/interpret.js

@@ -300,6 +300,7 @@ class Interpreter extends events_1.EventEmitter {
    * @param steps Array of actions.
    */
     carryOutSteps(page, steps) {
+        var _a;
         return __awaiter(this, void 0, void 0, function* () {
             /**
              * Defines overloaded (or added) methods/actions usable in the workflow.
@@ -311,10 +312,18 @@ class Interpreter extends events_1.EventEmitter {
              */
             const wawActions = {
                 screenshot: (params) => __awaiter(this, void 0, void 0, function* () {
+                    var _b;
+                    if ((_b = this.options.debugChannel) === null || _b === void 0 ? void 0 : _b.setActionType) {
+                        this.options.debugChannel.setActionType('screenshot');
+                    }
                     const screenshotBuffer = yield page.screenshot(Object.assign(Object.assign({}, params), { path: undefined }));
                     yield this.options.binaryCallback(screenshotBuffer, 'image/png');
                 }),
                 enqueueLinks: (selector) => __awaiter(this, void 0, void 0, function* () {
+                    var _c;
+                    if ((_c = this.options.debugChannel) === null || _c === void 0 ? void 0 : _c.setActionType) {
+                        this.options.debugChannel.setActionType('enqueueLinks');
+                    }
                     const links = yield page.locator(selector)
                         .evaluateAll(
                     // @ts-ignore
@@ -340,40 +349,50 @@ class Interpreter extends events_1.EventEmitter {
                     yield page.close();
                 }),
                 scrape: (selector) => __awaiter(this, void 0, void 0, function* () {
+                    var _d;
+                    if ((_d = this.options.debugChannel) === null || _d === void 0 ? void 0 : _d.setActionType) {
+                        this.options.debugChannel.setActionType('scrape');
+                    }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResults = yield page.evaluate((s) => window.scrape(s !== null && s !== void 0 ? s : null), selector);
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
+                    var _e;
+                    if ((_e = this.options.debugChannel) === null || _e === void 0 ? void 0 : _e.setActionType) {
+                        this.options.debugChannel.setActionType('scrapeSchema');
+                    }
+                    if (this.options.mode && this.options.mode === 'editor') {
+                        yield this.options.serializableCallback({});
+                        return;
+                    }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResult = yield page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
-                    const newResults = Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult];
-                    newResults.forEach((result) => {
-                        Object.entries(result).forEach(([key, value]) => {
-                            const keyExists = this.cumulativeResults.some((item) => key in item && item[key] !== undefined);
-                            if (!keyExists) {
-                                this.cumulativeResults.push({ [key]: value });
-                            }
-                        });
+                    if (!this.cumulativeResults || !Array.isArray(this.cumulativeResults)) {
+                        this.cumulativeResults = [];
+                    }
+                    if (this.cumulativeResults.length === 0) {
+                        this.cumulativeResults.push({});
+                    }
+                    const mergedResult = this.cumulativeResults[0];
+                    const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
+                    Object.entries(resultToProcess).forEach(([key, value]) => {
+                        if (value !== undefined) {
+                            mergedResult[key] = value;
+                        }
                     });
-                    const mergedResult = [
-                        Object.fromEntries(Object.entries(this.cumulativeResults.reduce((acc, curr) => {
-                            Object.entries(curr).forEach(([key, value]) => {
-                                // If the key doesn't exist or the current value is not undefined, add/update it
-                                if (value !== undefined) {
-                                    acc[key] = value;
-                                }
-                            });
-                            return acc;
-                        }, {})))
-                    ];
-                    // Log cumulative results after each action
-                    console.log("CUMULATIVE results:", this.cumulativeResults);
-                    console.log("MERGED results:", mergedResult);
-                    yield this.options.serializableCallback(mergedResult);
-                    // await this.options.serializableCallback(scrapeResult);
+                    console.log("Updated merged result:", mergedResult);
+                    yield this.options.serializableCallback([mergedResult]);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
+                    var _f;
+                    if ((_f = this.options.debugChannel) === null || _f === void 0 ? void 0 : _f.setActionType) {
+                        this.options.debugChannel.setActionType('scrapeList');
+                    }
+                    if (this.options.mode && this.options.mode === 'editor') {
+                        yield this.options.serializableCallback({});
+                        return;
+                    }
                     yield this.ensureScriptsLoaded(page);
                     if (!config.pagination) {
                         const scrapeResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
@@ -385,6 +404,10 @@ class Interpreter extends events_1.EventEmitter {
                     }
                 }),
                 scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
+                    var _g;
+                    if ((_g = this.options.debugChannel) === null || _g === void 0 ? void 0 : _g.setActionType) {
+                        this.options.debugChannel.setActionType('scrapeListAuto');
+                    }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResults = yield page.evaluate((listSelector) => {
                         return window.scrapeListAuto(listSelector);
@@ -392,6 +415,10 @@ class Interpreter extends events_1.EventEmitter {
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scroll: (pages) => __awaiter(this, void 0, void 0, function* () {
+                    var _h;
+                    if ((_h = this.options.debugChannel) === null || _h === void 0 ? void 0 : _h.setActionType) {
+                        this.options.debugChannel.setActionType('scroll');
+                    }
                     yield page.evaluate((pagesInternal) => __awaiter(this, void 0, void 0, function* () {
                         for (let i = 1; i <= (pagesInternal !== null && pagesInternal !== void 0 ? pagesInternal : 1); i += 1) {
                             // @ts-ignore
@@ -400,12 +427,20 @@ class Interpreter extends events_1.EventEmitter {
                     }), pages !== null && pages !== void 0 ? pages : 1);
                 }),
                 script: (code) => __awaiter(this, void 0, void 0, function* () {
+                    var _j;
+                    if ((_j = this.options.debugChannel) === null || _j === void 0 ? void 0 : _j.setActionType) {
+                        this.options.debugChannel.setActionType('script');
+                    }
                     const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
                     const x = new AsyncFunction('page', 'log', code);
                     yield x(page, this.log);
                 }),
                 flag: () => __awaiter(this, void 0, void 0, function* () {
                     return new Promise((res) => {
+                        var _a;
+                        if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                            this.options.debugChannel.setActionType('flag');
+                        }
                         this.emit('flag', page, res);
                     });
                 }),
@@ -433,6 +468,9 @@ class Interpreter extends events_1.EventEmitter {
                     yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
                 }
                 else {
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType(String(step.action));
+                    }
                     // Implements the dot notation for the "method name" in the workflow
                     const levels = String(step.action).split('.');
                     const methodName = levels[levels.length - 1];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "maxun-core",
-  "version": "0.0.15",
+  "version": "0.0.16",
   "description": "Core package for Maxun, responsible for data extraction",
   "main": "build/index.js",
   "typings": "build/index.d.ts",