mx-cloud 0.0.3 → 0.0.5

package/build/interpret.d.ts

@@ -30,14 +30,16 @@ declare global {
  * Defines optional intepreter options (passed in constructor)
  */
 interface InterpreterOptions {
+    mode?: string;
     maxRepeats: number;
     maxConcurrency: number;
     serializableCallback: (output: any) => (void | Promise<void>);
     binaryCallback: (output: any, mimeType: string) => (void | Promise<void>);
     debug: boolean;
     debugChannel: Partial<{
-        activeId: Function;
-        debugMessage: Function;
+        activeId: (id: number) => void;
+        debugMessage: (msg: string) => void;
+        setActionType: (type: string) => void;
     }>;
 }
 /**

package/build/interpret.js

@@ -328,10 +328,18 @@ class Interpreter extends events_1.EventEmitter {
              */
             const wawActions = {
                 screenshot: (params) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('screenshot');
+                    }
                     const screenshotBuffer = yield page.screenshot(Object.assign(Object.assign({}, params), { path: undefined }));
                     yield this.options.binaryCallback(screenshotBuffer, 'image/png');
                 }),
                 enqueueLinks: (selector) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('enqueueLinks');
+                    }
                     const links = yield page.locator(selector)
                         .evaluateAll(
                     // @ts-ignore
@@ -357,40 +365,50 @@ class Interpreter extends events_1.EventEmitter {
                     yield page.close();
                 }),
                 scrape: (selector) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scrape');
+                    }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResults = yield page.evaluate((s) => window.scrape(s !== null && s !== void 0 ? s : null), selector);
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scrapeSchema: (schema) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scrapeSchema');
+                    }
+                    if (this.options.mode && this.options.mode === 'editor') {
+                        yield this.options.serializableCallback({});
+                        return;
+                    }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResult = yield page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
-                    const newResults = Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult];
-                    newResults.forEach((result) => {
-                        Object.entries(result).forEach(([key, value]) => {
-                            const keyExists = this.cumulativeResults.some((item) => key in item && item[key] !== undefined);
-                            if (!keyExists) {
-                                this.cumulativeResults.push({ [key]: value });
-                            }
-                        });
+                    if (!this.cumulativeResults || !Array.isArray(this.cumulativeResults)) {
+                        this.cumulativeResults = [];
+                    }
+                    if (this.cumulativeResults.length === 0) {
+                        this.cumulativeResults.push({});
+                    }
+                    const mergedResult = this.cumulativeResults[0];
+                    const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
+                    Object.entries(resultToProcess).forEach(([key, value]) => {
+                        if (value !== undefined) {
+                            mergedResult[key] = value;
+                        }
                     });
-                    const mergedResult = [
-                        Object.fromEntries(Object.entries(this.cumulativeResults.reduce((acc, curr) => {
-                            Object.entries(curr).forEach(([key, value]) => {
-                                // If the key doesn't exist or the current value is not undefined, add/update it
-                                if (value !== undefined) {
-                                    acc[key] = value;
-                                }
-                            });
-                            return acc;
-                        }, {})))
-                    ];
-                    // Log cumulative results after each action
-                    console.log("CUMULATIVE results:", this.cumulativeResults);
-                    console.log("MERGED results:", mergedResult);
-                    yield this.options.serializableCallback(mergedResult);
-                    // await this.options.serializableCallback(scrapeResult);
+                    console.log("Updated merged result:", mergedResult);
+                    yield this.options.serializableCallback([mergedResult]);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scrapeList');
+                    }
+                    if (this.options.mode && this.options.mode === 'editor') {
+                        yield this.options.serializableCallback({});
+                        return;
+                    }
                     yield this.ensureScriptsLoaded(page);
                     let scrapeResults = [];
                     if (!config.pagination) {
@@ -402,6 +420,10 @@ class Interpreter extends events_1.EventEmitter {
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scrapeListAuto');
+                    }
                     yield this.ensureScriptsLoaded(page);
                     const scrapeResults = yield page.evaluate((listSelector) => {
                         return window.scrapeListAuto(listSelector);
@@ -409,6 +431,10 @@ class Interpreter extends events_1.EventEmitter {
                     yield this.options.serializableCallback(scrapeResults);
                 }),
                 scroll: (pages) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scroll');
+                    }
                     yield page.evaluate((pagesInternal) => __awaiter(this, void 0, void 0, function* () {
                         for (let i = 1; i <= (pagesInternal !== null && pagesInternal !== void 0 ? pagesInternal : 1); i += 1) {
                             // @ts-ignore
@@ -417,12 +443,20 @@ class Interpreter extends events_1.EventEmitter {
                     }), pages !== null && pages !== void 0 ? pages : 1);
                 }),
                 script: (code) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('script');
+                    }
                     const AsyncFunction = Object.getPrototypeOf(() => __awaiter(this, void 0, void 0, function* () { })).constructor;
                     const x = new AsyncFunction('page', 'log', code);
                     yield x(page, this.log);
                 }),
                 flag: () => __awaiter(this, void 0, void 0, function* () {
                     return new Promise((res) => {
+                        var _a;
+                        if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                            this.options.debugChannel.setActionType('flag');
+                        }
                         this.emit('flag', page, res);
                     });
                 }),
@@ -494,6 +528,7 @@ class Interpreter extends events_1.EventEmitter {
             let visitedUrls = new Set();
             const MAX_RETRIES = 3;
             const RETRY_DELAY = 1000; // 1 second delay between retries
+            const MAX_UNCHANGED_RESULTS = 5;
             const debugLog = (message, ...args) => {
                 console.log(`[Page ${visitedUrls.size}] [URL: ${page.url()}] ${message}`, ...args);
             };
@@ -572,28 +607,55 @@ class Interpreter extends events_1.EventEmitter {
                 }
             });
             let availableSelectors = config.pagination.selector.split(',');
+            let unchangedResultCounter = 0;
             try {
                 while (true) {
                     switch (config.pagination.type) {
                         case 'scrollDown': {
+                            let previousResultCount = allResults.length;
+                            yield scrapeCurrentPage();
+                            if (checkLimit()) {
+                                return allResults;
+                            }
                             yield page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
                             yield page.waitForTimeout(2000);
                             const currentHeight = yield page.evaluate(() => document.body.scrollHeight);
+                            const currentResultCount = allResults.length;
+                            if (currentResultCount === previousResultCount) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
+                                    return allResults;
+                                }
+                            }
+                            else {
+                                unchangedResultCounter = 0;
+                            }
                             if (currentHeight === previousHeight) {
-                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
                             previousHeight = currentHeight;
                             break;
                         }
                         case 'scrollUp': {
+                            let previousResultCount = allResults.length;
+                            yield scrapeCurrentPage();
+                            if (checkLimit()) {
+                                return allResults;
+                            }
                             yield page.evaluate(() => window.scrollTo(0, 0));
                             yield page.waitForTimeout(2000);
                             const currentTopHeight = yield page.evaluate(() => document.documentElement.scrollTop);
+                            const currentResultCount = allResults.length;
+                            if (currentResultCount === previousResultCount) {
+                                unchangedResultCounter++;
+                                if (unchangedResultCounter >= MAX_UNCHANGED_RESULTS) {
+                                    return allResults;
+                                }
+                            }
+                            else {
+                                unchangedResultCounter = 0;
+                            }
                             if (currentTopHeight === 0) {
-                                const finalResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
-                                allResults = allResults.concat(finalResults);
                                 return allResults;
                             }
                             previousHeight = currentTopHeight;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",