mx-cloud 0.0.14 → 0.0.16

package/build/browserSide/scraper.js

@@ -287,8 +287,15 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
                     return (_e = element.textContent) === null || _e === void 0 ? void 0 : _e.trim();
                 case 'innerHTML':
                     return element.innerHTML;
-                case 'outerHTML':
-                    return element.outerHTML;
+                case 'outerHTML': {
+                    const clonedElement = element.cloneNode(true);
+                    const elementsWithMxId = clonedElement.querySelectorAll('[data-mx-id]');
+                    elementsWithMxId.forEach(el => el.removeAttribute('data-mx-id'));
+                    if (clonedElement.hasAttribute && clonedElement.hasAttribute('data-mx-id')) {
+                        clonedElement.removeAttribute('data-mx-id');
+                    }
+                    return clonedElement.outerHTML;
+                }
                 default:
                     return element.getAttribute(attribute) || ((_f = element.innerText) === null || _f === void 0 ? void 0 : _f.trim());
             }
@@ -359,6 +366,69 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
    */
     window.scrapeList = function (_a) {
         return __awaiter(this, arguments, void 0, function* ({ listSelector, fields, limit = 10 }) {
+            const isSitemapUrl = () => {
+                const url = window.location.href.toLowerCase();
+                return url.includes('sitemap') && url.includes('.xml');
+            };
+            const scrapeSitemapData = () => {
+                // Try to get the XML content from the page
+                let xmlContent = null;
+                // Method 1: Check if the page is already parsed as XML
+                if (document.documentElement.tagName.toLowerCase() === 'urlset') {
+                    xmlContent = document;
+                }
+                // Method 2: Try to get raw XML from pre tags (common browser display)
+                if (!xmlContent) {
+                    const preElement = document.querySelector('pre');
+                    if (preElement) {
+                        try {
+                            const parser = new DOMParser();
+                            xmlContent = parser.parseFromString(preElement.textContent, 'text/xml');
+                        }
+                        catch (e) {
+                            console.warn('Failed to parse XML from pre element:', e);
+                        }
+                    }
+                }
+                // Method 3: Try to parse the entire document as XML
+                if (!xmlContent) {
+                    try {
+                        const parser = new DOMParser();
+                        xmlContent = parser.parseFromString(document.documentElement.outerHTML, 'text/xml');
+                    }
+                    catch (e) {
+                        console.warn('Failed to parse document as XML:', e);
+                    }
+                }
+                if (!xmlContent) {
+                    console.error('Could not parse sitemap XML');
+                    return [];
+                }
+                // Extract URL entries from the sitemap
+                const urlElements = xmlContent.querySelectorAll('url');
+                const sitemapData = [];
+                urlElements.forEach((urlElement, index) => {
+                    if (limit && index >= limit)
+                        return;
+                    const locElement = urlElement.querySelector('loc');
+                    const lastmodElement = urlElement.querySelector('lastmod');
+                    const entry = {};
+                    if (locElement) {
+                        entry.loc = locElement.textContent.trim();
+                    }
+                    if (lastmodElement) {
+                        entry.lastmod = lastmodElement.textContent.trim();
+                    }
+                    // Only add entries that have at least a loc field
+                    if (entry.loc) {
+                        sitemapData.push(entry);
+                    }
+                });
+                return sitemapData;
+            };
+            if (isSitemapUrl()) {
+                return scrapeSitemapData();
+            }
             // XPath evaluation functions
             const queryInsideContext = (context, part) => {
                 try {

package/build/interpret.js

@@ -318,7 +318,7 @@ class Interpreter extends events_1.EventEmitter {
    */
     carryOutSteps(page, steps) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a;
+            var _a, _b;
             /**
              * Defines overloaded (or added) methods/actions usable in the workflow.
              * If a method overloads any existing method of the Page class, it accepts the same set
@@ -388,18 +388,44 @@ class Interpreter extends events_1.EventEmitter {
                     if (!this.cumulativeResults || !Array.isArray(this.cumulativeResults)) {
                         this.cumulativeResults = [];
                     }
+                    const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
                     if (this.cumulativeResults.length === 0) {
-                        this.cumulativeResults.push({});
+                        // First execution - create initial row
+                        const newRow = {};
+                        Object.entries(resultToProcess).forEach(([key, value]) => {
+                            if (value !== undefined) {
+                                newRow[key] = value;
+                            }
+                        });
+                        this.cumulativeResults.push(newRow);
                     }
-                    const mergedResult = this.cumulativeResults[0];
-                    const resultToProcess = Array.isArray(scrapeResult) ? scrapeResult[0] : scrapeResult;
-                    Object.entries(resultToProcess).forEach(([key, value]) => {
-                        if (value !== undefined) {
-                            mergedResult[key] = value;
+                    else {
+                        // Check if any keys from new result already exist in the last row
+                        const lastRow = this.cumulativeResults[this.cumulativeResults.length - 1];
+                        const newResultKeys = Object.keys(resultToProcess).filter(key => resultToProcess[key] !== undefined);
+                        const hasRepeatedKeys = newResultKeys.some(key => lastRow.hasOwnProperty(key));
+                        if (hasRepeatedKeys) {
+                            // Keys are repeated - create a new row
+                            const newRow = {};
+                            Object.entries(resultToProcess).forEach(([key, value]) => {
+                                if (value !== undefined) {
+                                    newRow[key] = value;
+                                }
+                            });
+                            this.cumulativeResults.push(newRow);
                         }
-                    });
-                    console.log("Updated merged result:", mergedResult);
-                    yield this.options.serializableCallback([mergedResult]);
+                        else {
+                            // No repeated keys - merge with the last row
+                            Object.entries(resultToProcess).forEach(([key, value]) => {
+                                if (value !== undefined) {
+                                    lastRow[key] = value;
+                                }
+                            });
+                        }
+                    }
+                    console.log("Total accumulated rows:", this.cumulativeResults.length);
+                    console.log("Current results:", this.cumulativeResults);
+                    yield this.options.serializableCallback(this.cumulativeResults);
                 }),
                 scrapeList: (config) => __awaiter(this, void 0, void 0, function* () {
                     var _a, _b;
@@ -410,17 +436,37 @@ class Interpreter extends events_1.EventEmitter {
                         yield this.options.serializableCallback({});
                         return;
                     }
-                    yield this.ensureScriptsLoaded(page);
-                    if ((_b = this.options.debugChannel) === null || _b === void 0 ? void 0 : _b.incrementScrapeListIndex) {
-                        this.options.debugChannel.incrementScrapeListIndex();
-                    }
-                    if (!config.pagination) {
-                        const scrapeResults = yield page.evaluate((cfg) => window.scrapeList(cfg), config);
+                    try {
+                        yield this.ensureScriptsLoaded(page);
+                        if ((_b = this.options.debugChannel) === null || _b === void 0 ? void 0 : _b.incrementScrapeListIndex) {
+                            this.options.debugChannel.incrementScrapeListIndex();
+                        }
+                        let scrapeResults = [];
+                        if (!config.pagination) {
+                            scrapeResults = yield page.evaluate((cfg) => {
+                                try {
+                                    return window.scrapeList(cfg);
+                                }
+                                catch (error) {
+                                    console.warn('ScrapeList evaluation failed:', error.message);
+                                    return []; // Return empty array instead of failing
+                                }
+                            }, config);
+                        }
+                        else {
+                            scrapeResults = yield this.handlePagination(page, config);
+                        }
+                        // Ensure we always have an array
+                        if (!Array.isArray(scrapeResults)) {
+                            scrapeResults = [];
+                        }
+                        console.log(`ScrapeList completed with ${scrapeResults.length} results`);
                         yield this.options.serializableCallback(scrapeResults);
                     }
-                    else {
-                        const scrapeResults = yield this.handlePagination(page, config);
-                        yield this.options.serializableCallback(scrapeResults);
+                    catch (error) {
+                        console.error('ScrapeList action failed completely:', error.message);
+                        // Don't throw error, just return empty array
+                        yield this.options.serializableCallback([]);
                     }
                 }),
                 scrapeListAuto: (config) => __awaiter(this, void 0, void 0, function* () {
@@ -482,46 +528,54 @@ class Interpreter extends events_1.EventEmitter {
             });
             for (const step of steps) {
                 this.log(`Launching ${String(step.action)}`, logger_1.Level.LOG);
-                if (step.action in wawActions) {
-                    // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
-                    const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
-                    yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
-                }
-                else {
-                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
-                        this.options.debugChannel.setActionType(String(step.action));
-                    }
-                    // Implements the dot notation for the "method name" in the workflow
-                    const levels = String(step.action).split('.');
-                    const methodName = levels[levels.length - 1];
-                    let invokee = page;
-                    for (const level of levels.splice(0, levels.length - 1)) {
-                        invokee = invokee[level];
+                try {
+                    if (step.action in wawActions) {
+                        // "Arrayifying" here should not be needed (TS + syntax checker - only arrays; but why not)
+                        const params = !step.args || Array.isArray(step.args) ? step.args : [step.args];
+                        yield wawActions[step.action](...(params !== null && params !== void 0 ? params : []));
                     }
-                    if (methodName === 'waitForLoadState') {
-                        try {
-                            yield executeAction(invokee, methodName, step.args);
+                    else {
+                        if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                            this.options.debugChannel.setActionType(String(step.action));
                         }
-                        catch (error) {
-                            yield executeAction(invokee, methodName, 'domcontentloaded');
+                        // Implements the dot notation for the "method name" in the workflow
+                        const levels = String(step.action).split('.');
+                        const methodName = levels[levels.length - 1];
+                        let invokee = page;
+                        for (const level of levels.splice(0, levels.length - 1)) {
+                            invokee = invokee[level];
                         }
-                    }
-                    else if (methodName === 'click') {
-                        try {
-                            yield executeAction(invokee, methodName, step.args);
+                        if (methodName === 'waitForLoadState') {
+                            try {
+                                yield executeAction(invokee, methodName, step.args);
+                            }
+                            catch (error) {
+                                yield executeAction(invokee, methodName, 'domcontentloaded');
+                            }
                         }
-                        catch (error) {
+                        else if (methodName === 'click') {
                             try {
-                                yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                                yield executeAction(invokee, methodName, step.args);
                             }
                             catch (error) {
-                                continue;
+                                try {
+                                    yield executeAction(invokee, methodName, [step.args[0], { force: true }]);
+                                }
+                                catch (error) {
+                                    this.log(`Click action failed for selector ${(_b = step.args) === null || _b === void 0 ? void 0 : _b[0]}: ${error.message}`, logger_1.Level.WARN);
+                                    continue; // Skip to next action
+                                }
                             }
                         }
+                        else {
+                            yield executeAction(invokee, methodName, step.args);
+                        }
                     }
-                    else {
-                        yield executeAction(invokee, methodName, step.args);
-                    }
+                }
+                catch (error) {
+                    this.log(`Action ${String(step.action)} failed: ${error.message}`, logger_1.Level.WARN);
+                    // Continue to next action instead of breaking
+                    continue;
                 }
                 yield new Promise((res) => { setTimeout(res, 500); });
             }
@@ -1895,26 +1949,29 @@ class Interpreter extends events_1.EventEmitter {
                     yield p.close();
                     return;
                 }
-                const newSelectors = this.getSelectors(workflowCopy);
-                newSelectors.forEach(selector => {
-                    if (!selectors.includes(selector)) {
-                        selectors.push(selector);
-                    }
-                });
-                let pageState = {};
-                let getStateTest = "Hello";
-                try {
-                    pageState = yield this.getState(p, workflowCopy, selectors);
-                    selectors = [];
-                    console.log("Empty selectors:", selectors);
-                }
-                catch (e) {
-                    this.log('The browser has been closed.');
+                if (workflowCopy.length === 0) {
+                    this.log('All actions completed. Workflow finished.', logger_1.Level.LOG);
                     return;
                 }
-                if (this.options.debug) {
-                    this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, logger_1.Level.WARN);
-                }
+                // const newSelectors = this.getSelectors(workflowCopy);
+                //   newSelectors.forEach(selector => {
+                //     if (!selectors.includes(selector)) {
+                //       selectors.push(selector);
+                //     }
+                // });
+                // let pageState = {};
+                // let getStateTest = "Hello";
+                // try {
+                //   pageState = await this.getState(p, workflowCopy, selectors);
+                //   selectors = [];
+                //   console.log("Empty selectors:", selectors)
+                // } catch (e: any) {
+                //   this.log('The browser has been closed.');
+                //   return;
+                // }
+                // if (this.options.debug) {
+                //   this.log(`Current state is: \n${JSON.stringify(pageState, null, 2)}`, Level.WARN);
+                // }
                 // const actionId = workflow.findIndex((step) => {
                 //   const isApplicable = this.applicable(step.where, pageState, usedActions);
                 //   console.log("-------------------------------------------------------------");
@@ -1934,10 +1991,11 @@ class Interpreter extends events_1.EventEmitter {
                 //     console.log("SCHEMA CHANGES:", changes);
                 //   }
                 // }
-                actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
-                if (actionId !== -1 && workflowCopy[actionId]) {
-                    workflowCopy[actionId] = yield this.validateWorkflowAction(p, workflowCopy[actionId]);
-                }
+                // actionId = this.getMatchingActionId(workflowCopy, pageState, usedActions);
+                // if (actionId !== -1 && workflowCopy[actionId]) {
+                //   workflowCopy[actionId] = await this.validateWorkflowAction(p, workflowCopy[actionId]);
+                // }
+                const actionId = workflowCopy.length - 1;
                 const action = workflowCopy[actionId];
                 console.log("MATCHED ACTION:", action);
                 console.log("MATCHED ACTION ID:", actionId);
@@ -1963,6 +2021,10 @@ class Interpreter extends events_1.EventEmitter {
                     }
                     catch (e) {
                         this.log(e, logger_1.Level.ERROR);
+                        console.log(`Action with ID ${action.id} failed: ${e.message}`);
+                        // Still remove the failed action to prevent infinite loops
+                        workflowCopy.splice(actionId, 1);
+                        console.log(`Failed action removed. Remaining actions: ${workflowCopy.length}`);
                     }
                 }
                 else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.14",
+  "version": "0.0.16",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",