mx-cloud 0.0.2 → 0.0.4

package/build/browserSide/scraper.js

@@ -466,6 +466,20 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     return element.innerHTML.trim();
                 }
                 else if (attribute === 'src' || attribute === 'href') {
+                    if (attribute === 'href' && element.tagName !== 'A') {
+                        const parentElement = element.parentElement;
+                        if (parentElement && parentElement.tagName === 'A') {
+                            const parentHref = parentElement.getAttribute('href');
+                            if (parentHref) {
+                                try {
+                                    return new URL(parentHref, baseURL).href;
+                                }
+                                catch (e) {
+                                    return parentHref;
+                                }
+                            }
+                        }
+                    }
                     const attrValue = element.getAttribute(attribute);
                     const dataAttr = attrValue || element.getAttribute('data-' + attribute);
                     if (!dataAttr || dataAttr.trim() === '') {

package/build/interpret.d.ts

@@ -36,8 +36,9 @@ interface InterpreterOptions {
     binaryCallback: (output: any, mimeType: string) => (void | Promise<void>);
     debug: boolean;
     debugChannel: Partial<{
-        activeId: Function;
-        debugMessage: Function;
+        activeId: (id: number) => void;
+        debugMessage: (msg: string) => void;
+        setActionType: (type: string) => void;
     }>;
 }
 /**

package/build/interpret.js

@@ -328,10 +328,18 @@ class Interpreter extends events_1.EventEmitter {
              */
             const wawActions = {
                 screenshot: (params) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('screenshot');
+                    }
                     const screenshotBuffer = yield page.screenshot(Object.assign(Object.assign({}, params), { path: undefined }));
                     yield this.options.binaryCallback(screenshotBuffer, 'image/png');
                 }),
                 enqueueLinks: (selector) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('enqueueLinks');
+                    }
                     const links = yield page.locator(selector)
                         .evaluateAll(
                     // @ts-ignore
@@ -357,40 +365,42 @@ class Interpreter extends events_1.EventEmitter {
                     yield page.close();
                 }),
                 scrape: (selector) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scrape');
+                    }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResults = yield page.evaluate((s) => window.scrape(s !== null && s !== void 0 ? s : null), selector);
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scrapeSchema');
+                    }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResult = yield page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
-                    const newResults = Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult];
-                    newResults.forEach((result) => {
-                        Object.entries(result).forEach(([key, value]) => {
-                            const keyExists = this.cumulativeResults.some((item) => key in item && item[key] !== undefined);
-                            if (!keyExists) {
-                                this.cumulativeResults.push({ [key]: value });
-                            }
-                        });
+                    if (!this.cumulativeResults || !Array.isArray(this.cumulativeResults)) {
+                        this.cumulativeResults = [];
+                    }
+                    if (this.cumulativeResults.length === 0) {
+                        this.cumulativeResults.push({});
+                    }
+                    const mergedResult = this.cumulativeResults[0];
+                    const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
+                    Object.entries(resultToProcess).forEach(([key, value]) => {
+                        if (value !== undefined) {
+                            mergedResult[key] = value;
+                        }
                     });
-                    const mergedResult = [
-                        Object.fromEntries(Object.entries(this.cumulativeResults.reduce((acc, curr) => {
-                            Object.entries(curr).forEach(([key, value]) => {
-                                // If the key doesn't exist or the current value is not undefined, add/update it
-                                if (value !== undefined) {
-                                    acc[key] = value;
-                                }
-                            });
-                            return acc;
-                        }, {})))
-                    ];
-                    // Log cumulative results after each action
-                    console.log("CUMULATIVE results:", this.cumulativeResults);
-                    console.log("MERGED results:", mergedResult);
-                    yield this.options.serializableCallback(mergedResult);
-                    // await this.options.serializableCallback(scrapeResult);
+                    console.log("Updated merged result:", mergedResult);
+                    yield this.options.serializableCallback([mergedResult]);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scrapeList');
+                    }
                     yield this.ensureScriptsLoaded(page);
                     let scrapeResults = [];
                     if (!config.pagination) {
@@ -402,6 +412,10 @@ class Interpreter extends events_1.EventEmitter {
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scrapeListAuto');
+                    }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResults = yield page.evaluate((listSelector) => {
                         return window.scrapeListAuto(listSelector);
@@ -409,6 +423,10 @@ class Interpreter extends events_1.EventEmitter {
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scroll: (pages) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scroll');
+                    }
                     yield page.evaluate((pagesInternal) => __awaiter(this, void 0, void 0, function* () {
                         for (let i = 1; i <= (pagesInternal !== null && pagesInternal !== void 0 ? pagesInternal : 1); i += 1) {
                             // @ts-ignore
@@ -417,12 +435,20 @@ class Interpreter extends events_1.EventEmitter {
                     }), pages !== null && pages !== void 0 ? pages : 1);
                 }),
                 script: (code) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('script');
+                    }
                     const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
                     const x = new AsyncFunction('page', 'log', code);
                     yield x(page, this.log);
                 }),
                 flag: () => __awaiter(this, void 0, void 0, function* () {
                     return new Promise((res) => {
+                        var _a;
+                        if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                            this.options.debugChannel.setActionType('flag');
+                        }
                         this.emit('flag', page, res);
                     });
                 }),
@@ -494,6 +520,7 @@ class Interpreter extends events_1.EventEmitter {
             let visitedUrls = new Set();
             const MAX_RETRIES = 3;
             const RETRY_DELAY = 1000; // 1 second delay between retries
+            const MAX_UNCHANGED_RESULTS = 5;
             const debugLog = (message, ...args) => {
                 console.log(`[Page ${visitedUrls.size}] [URL: ${page.url()}] ${message}`, ...args);
             };
@@ -572,28 +599,55 @@ class Interpreter extends events_1.EventEmitter {
                 }
             });
             let availableSelectors = config.pagination.selector.split(',');
+            let unchangedResultCounter = 0;
             try {
                 while (true) {
                     switch (config.pagination.type) {
                         case 'scrollDown': {
+                            let previousResultCount = allResults.length;
+                            yield scrapeCurrentPage();
+                            if (checkLimit()) {
+                                return allResults;
+                            }
                             yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
                             yield page.waitForTimeout(2000);
                             const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                            const currentResultCount = allResults.length;
+                            if (currentResultCount === previousResultCount) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
+                                    return allResults;
+                                }
+                            }
+                            else {
+                                unchangedResultCounter = 0;
+                            }
                             if (currentHeight === previousHeight) {
-                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
                             previousHeight = currentHeight;
                             break;
                         }
                         case 'scrollUp': {
+                            let previousResultCount = allResults.length;
+                            yield scrapeCurrentPage();
+                            if (checkLimit()) {
+                                return allResults;
+                            }
                             yield page.evaluate(() => window.scrollTo(0, 0));
                             yield page.waitForTimeout(2000);
                             const currentTopHeight = yield page.evaluate(() => document.documentElement.scrollTop);
+                            const currentResultCount = allResults.length;
+                            if (currentResultCount === previousResultCount) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
+                                    return allResults;
+                                }
+                            }
+                            else {
+                                unchangedResultCounter = 0;
+                            }
                             if (currentTopHeight === 0) {
-                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
                             previousHeight = currentTopHeight;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",