mx-cloud 0.0.28 → 0.0.31

package/build/index.d.ts

@@ -1,5 +1,5 @@
 import Interpreter from './interpret';
 export default Interpreter;
 export { default as Preprocessor } from './preprocessor';
-export type { WorkflowFile, WhereWhatPair, Where, What, } from './types/workflow';
+export type { WorkflowFile, WhereWhatPair, Where, What, CustomFunctions, } from './types/workflow';
 export { unaryOperators, naryOperators, meta as metaOperators } from './types/logic';

package/build/interpret.js

@@ -54,6 +54,7 @@ const concurrency_1 = __importDefault(require("./utils/concurrency"));
 const preprocessor_1 = __importDefault(require("./preprocessor"));
 const logger_1 = __importStar(require("./utils/logger"));
 const selector_1 = require("./selector");
+const markdown_1 = require("./utils/markdown");
 /**
  * Class for running the Smart Workflows.
  */
@@ -251,15 +252,19 @@ class Interpreter extends events_1.EventEmitter {
                     }
                     yield page.close();
                 }),
-                scrape: (selector) => __awaiter(this, void 0, void 0, function* () {
-                    var _a;
-                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
-                        this.options.debugChannel.setActionType('scrape');
-                    }
-                    yield this.ensureScriptsLoaded(page);
-                    const scrapeResults = yield page.evaluate((s) => window.scrape(s !== null && s !== void 0 ? s : null), selector);
-                    yield this.callWithTimeout(() => this.options.serializableCallback(scrapeResults), 30000, 'serializableCallback (scrape)');
-                }),
+                // DEPRECATED: Old scrape action - commented out in favor of new workflow-based scrape action
+                // scrape: async (selector?: string) => {
+                //   if (this.options.debugChannel?.setActionType) {
+                //     this.options.debugChannel.setActionType('scrape');
+                //   }
+                //   await this.ensureScriptsLoaded(page);
+                //   const scrapeResults: Record<string, string>[] = await page.evaluate((s) => window.scrape(s ?? null), selector);
+                //   await this.callWithTimeout(
+                //     () => this.options.serializableCallback(scrapeResults),
+                //     30000,
+                //     'serializableCallback (scrape)'
+                //   );
+                // },
                 scrapeSchema: (schema_1, ...args_1) => __awaiter(this, [schema_1, ...args_1], void 0, function* (schema, actionName = "") {
                     var _a;
                     if (this.isAborted) {
@@ -922,14 +927,60 @@ class Interpreter extends events_1.EventEmitter {
                                     metadata: Object.assign(Object.assign({}, allMetadata), { title, language: document.documentElement.lang || '', favicon: ((_b = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]')) === null || _b === void 0 ? void 0 : _b.href) || '', statusCode: 200 })
                                 };
                             });
-                            return {
+                            const result = {
                                 metadata: Object.assign(Object.assign({}, pageData.metadata), { url: url, sourceURL: url }),
-                                html: pageData.html,
-                                text: pageData.text,
-                                links: pageData.links,
-                                wordCount: pageData.wordCount,
                                 scrapedAt: new Date().toISOString()
                             };
+                            const formats = crawlConfig.outputFormats || [];
+                            if (formats.includes('text')) {
+                                result.text = pageData.text;
+                                result.wordCount = pageData.wordCount;
+                            }
+                            if (formats.includes('html')) {
+                                result.html = pageData.html;
+                                result.links = pageData.links;
+                            }
+                            if (formats.includes('markdown')) {
+                                try {
+                                    const markdown = yield (0, markdown_1.parseMarkdown)(pageData.html, url);
+                                    result.markdown = markdown;
+                                }
+                                catch (err) {
+                                    this.log(`Markdown conversion failed for ${url}: ${err.message}`, logger_1.Level.WARN);
+                                    result.markdown = '';
+                                }
+                            }
+                            if (formats.includes('screenshot-visible')) {
+                                try {
+                                    const screenshotBuffer = yield page.screenshot({ fullPage: false });
+                                    const screenshotName = `Crawl - ${crawlResults.length} - Visible`;
+                                    yield this.options.binaryCallback({
+                                        name: screenshotName,
+                                        data: screenshotBuffer,
+                                        mimeType: 'image/png'
+                                    }, 'image/png');
+                                    result.screenshotVisible = screenshotName;
+                                }
+                                catch (err) {
+                                    this.log(`Screenshot-visible failed for ${url}: ${err.message}`, logger_1.Level.WARN);
+                                }
+                            }
+                            if (formats.includes('screenshot-fullpage')) {
+                                try {
+                                    const screenshotBuffer = yield page.screenshot({ fullPage: true });
+                                    const screenshotName = `Crawl - ${crawlResults.length} - Full Page`;
+                                    yield this.options.binaryCallback({
+                                        name: screenshotName,
+                                        data: screenshotBuffer,
+                                        mimeType: 'image/png'
+                                    }, 'image/png');
+                                    result.screenshotFullpage = screenshotName;
+                                }
+                                catch (err) {
+                                    this.log(`Screenshot-fullpage failed for ${url}: ${err.message}`, logger_1.Level.WARN);
+                                }
+                            }
+                            return result;
                         });
                         const visitedUrls = new Set();
                         const crawlResults = [];
@@ -1036,15 +1087,28 @@ class Interpreter extends events_1.EventEmitter {
                                     yield new Promise(resolve => setTimeout(resolve, robotRules.crawlDelay));
                                 }
                                 yield page.goto(url, {
-                                    waitUntil: 'domcontentloaded',
-                                    timeout: 30000
+                                    waitUntil: 'load',
+                                    timeout: 60000
                                 }).catch((err) => {
                                     throw new Error(`Navigation failed: ${err.message}`);
                                 });
-                                yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => { });
+                                yield page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => { });
                                 const pageResult = yield scrapePageContent(url);
                                 pageResult.metadata.depth = depth;
                                 crawlResults.push(pageResult);
+                                const actionType = "crawl";
+                                const actionName = "Crawl Results";
+                                if (!this.serializableDataByType[actionType]) {
+                                    this.serializableDataByType[actionType] = {};
+                                }
+                                this.serializableDataByType[actionType][actionName] = [...crawlResults];
+                                yield this.options.serializableCallback({
+                                    crawl: this.serializableDataByType.crawl
+                                });
+                                if (this.isAborted) {
+                                    this.log(`Run aborted after scraping ${url}, stopping crawl`, logger_1.Level.WARN);
+                                    break;
+                                }
                                 this.log(`✓ Scraped ${url} (${pageResult.wordCount} words, depth ${depth})`, logger_1.Level.LOG);
                                 if (crawlConfig.followLinks && depth < crawlConfig.maxDepth) {
                                     const newLinks = yield extractLinksFromPage();
@@ -1308,6 +1372,7 @@ class Interpreter extends events_1.EventEmitter {
                                 filters: searchConfig.filters || {},
                                 resultsCount: searchResults.length,
                                 results: searchResults,
+                                mode: searchConfig.mode,
                                 searchedAt: new Date().toISOString()
                             };
                             this.serializableDataByType[actionType][actionName] = searchData;
@@ -1323,16 +1388,25 @@ class Interpreter extends events_1.EventEmitter {
                         this.log(`Starting to scrape content from ${searchResults.length} search results...`, logger_1.Level.LOG);
                         const scrapedResults = [];
                         for (let i = 0; i < searchResults.length; i++) {
+                            if (this.isAborted) {
+                                this.log(`Run aborted, stopping search scraping at result ${i + 1}/${searchResults.length}`, logger_1.Level.WARN);
+                                break;
+                            }
                             const result = searchResults[i];
                             try {
                                 this.log(`[${i + 1}/${searchResults.length}] Scraping: ${result.url}`, logger_1.Level.LOG);
+                                let navigationFailed = false;
                                 yield page.goto(result.url, {
-                                    waitUntil: 'domcontentloaded',
-                                    timeout: 30000
+                                    waitUntil: 'load',
+                                    timeout: 60000
                                 }).catch(() => {
                                     this.log(`Failed to navigate to ${result.url}, skipping...`, logger_1.Level.WARN);
+                                    navigationFailed = true;
                                 });
-                                yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => { });
+                                if (navigationFailed) {
+                                    continue;
+                                }
+                                yield page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => { });
                                 const pageData = yield page.evaluate(() => {
                                     var _a, _b;
                                     const getMeta = (name) => {
@@ -1368,7 +1442,7 @@ class Interpreter extends events_1.EventEmitter {
                                         metadata: Object.assign(Object.assign({}, allMetadata), { title, language: document.documentElement.lang || '', favicon: ((_b = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]')) === null || _b === void 0 ? void 0 : _b.href) || '', statusCode: 200 })
                                     };
                                 });
-                                scrapedResults.push({
+                                const scrapedResult = {
                                     searchResult: {
                                         query: searchConfig.query,
                                         position: result.position,
@@ -1376,12 +1450,79 @@ class Interpreter extends events_1.EventEmitter {
                                         searchDescription: result.description,
                                     },
                                     metadata: Object.assign(Object.assign({}, pageData.metadata), { url: result.url, sourceURL: result.url }),
-                                    html: pageData.html,
-                                    text: pageData.text,
-                                    links: pageData.links,
-                                    wordCount: pageData.wordCount,
                                     scrapedAt: new Date().toISOString()
+                                };
+                                const formats = searchConfig.outputFormats || [];
+                                if (formats.includes('text')) {
+                                    scrapedResult.text = pageData.text;
+                                    scrapedResult.wordCount = pageData.wordCount;
+                                }
+                                if (formats.includes('html')) {
+                                    scrapedResult.html = pageData.html;
+                                    scrapedResult.links = pageData.links;
+                                }
+                                if (formats.includes('markdown')) {
+                                    try {
+                                        const markdown = yield (0, markdown_1.parseMarkdown)(pageData.html, result.url);
+                                        scrapedResult.markdown = markdown;
+                                    }
+                                    catch (err) {
+                                        this.log(`Markdown conversion failed for ${result.url}: ${err.message}`, logger_1.Level.WARN);
+                                        scrapedResult.markdown = '';
+                                    }
+                                }
+                                if (formats.includes('screenshot-visible')) {
+                                    try {
+                                        const screenshotBuffer = yield page.screenshot({ fullPage: false });
+                                        const screenshotName = `Search - ${i} - Visible`;
+                                        yield this.options.binaryCallback({
+                                            name: screenshotName,
+                                            data: screenshotBuffer,
+                                            mimeType: 'image/png'
+                                        }, 'image/png');
+                                        scrapedResult.screenshotVisible = screenshotName;
+                                    }
+                                    catch (err) {
+                                        this.log(`Screenshot-visible failed for ${result.url}: ${err.message}`, logger_1.Level.WARN);
+                                    }
+                                }
+                                if (formats.includes('screenshot-fullpage')) {
+                                    try {
+                                        const screenshotBuffer = yield page.screenshot({ fullPage: true });
+                                        const screenshotName = `Search - ${i} - Full Page`;
+                                        yield this.options.binaryCallback({
+                                            name: screenshotName,
+                                            data: screenshotBuffer,
+                                            mimeType: 'image/png'
+                                        }, 'image/png');
+                                        scrapedResult.screenshotFullpage = screenshotName;
+                                    }
+                                    catch (err) {
+                                        this.log(`Screenshot-fullpage failed for ${result.url}: ${err.message}`, logger_1.Level.WARN);
+                                    }
+                                }
+                                scrapedResults.push(scrapedResult);
+                                const actionType = "search";
+                                const actionName = "Search Results";
+                                if (!this.serializableDataByType[actionType]) {
+                                    this.serializableDataByType[actionType] = {};
+                                }
+                                this.serializableDataByType[actionType][actionName] = {
+                                    query: searchConfig.query,
+                                    provider: searchConfig.provider,
+                                    filters: searchConfig.filters || {},
+                                    resultsCount: scrapedResults.length,
+                                    results: scrapedResults,
+                                    mode: searchConfig.mode,
+                                    searchedAt: new Date().toISOString()
+                                };
+                                yield this.options.serializableCallback({
+                                    search: this.serializableDataByType.search
                                 });
+                                if (this.isAborted) {
+                                    this.log(`Run aborted after scraping ${result.url}, stopping search`, logger_1.Level.WARN);
+                                    break;
+                                }
                                 this.log(`✓ Scraped ${result.url} (${pageData.wordCount} words)`, logger_1.Level.LOG);
                             }
                             catch (error) {
@@ -1430,6 +1571,169 @@ class Interpreter extends events_1.EventEmitter {
                         throw new Error(`Search execution error: ${error.message}`);
                     }
                 }),
+                /**
+                 * scrape action: Converts a webpage to text, markdown, HTML, and/or screenshots.
+                 * This is the workflow action for scrape robots.
+                 */
+                scrape: (scrapeConfig) => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if (this.isAborted) {
+                        this.log('Workflow aborted, stopping scrape', logger_1.Level.WARN);
+                        return;
+                    }
+                    if ((_a = this.options.debugChannel) === null || _a === void 0 ? void 0 : _a.setActionType) {
+                        this.options.debugChannel.setActionType('scrape');
+                    }
+                    this.log(`Starting scrape for URL: ${scrapeConfig.url}`, logger_1.Level.LOG);
+                    try {
+                        const formats = scrapeConfig.formats || ['markdown', 'html', 'text'];
+                        const url = scrapeConfig.url;
+                        if (!url) {
+                            throw new Error('No URL specified for scrape action');
+                        }
+                        const currentUrl = page.url();
+                        if (currentUrl === 'about:blank' || currentUrl === '' || !currentUrl.includes(new URL(url).hostname)) {
+                            this.log(`Navigating to ${url}`, logger_1.Level.LOG);
+                            yield page.goto(url, { waitUntil: 'load', timeout: 60000 });
+                            yield page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => { });
+                        }
+                        const serializableOutput = {};
+                        const SCRAPE_TIMEOUT = 120000;
+                        if (formats.includes('text')) {
+                            try {
+                                const textPromise = page.evaluate(() => {
+                                    const body = document.body;
+                                    if (!body)
+                                        return '';
+                                    return body.innerText || body.textContent || '';
+                                });
+                                const timeoutPromise = new Promise((_, reject) => {
+                                    setTimeout(() => reject(new Error(`Text extraction timed out after ${SCRAPE_TIMEOUT / 1000}s`)), SCRAPE_TIMEOUT);
+                                });
+                                const text = yield Promise.race([textPromise, timeoutPromise]);
+                                if (text && text.trim().length > 0) {
+                                    serializableOutput.text = [{ content: text.trim() }];
+                                    this.log('Text extraction completed', logger_1.Level.LOG);
+                                }
+                            }
+                            catch (error) {
+                                this.log(`Text extraction failed: ${error.message}`, logger_1.Level.WARN);
+                            }
+                        }
+                        if (formats.includes('markdown')) {
+                            try {
+                                const html = yield page.evaluate(() => {
+                                    const selectors = [
+                                        "script", "style", "link[rel='stylesheet']", "noscript", "meta",
+                                        "svg", "img", "picture", "source", "video", "audio", "iframe", "object", "embed"
+                                    ];
+                                    selectors.forEach(sel => {
+                                        document.querySelectorAll(sel).forEach(e => e.remove());
+                                    });
+                                    const all = document.querySelectorAll("*");
+                                    all.forEach(el => {
+                                        [...el.attributes].forEach(attr => {
+                                            if (attr.name.startsWith("on") || attr.name === "data-mx-id") {
+                                                el.removeAttribute(attr.name);
+                                            }
+                                        });
+                                    });
+                                    return document.documentElement.outerHTML;
+                                });
+                                const markdownPromise = (0, markdown_1.parseMarkdown)(html, url);
+                                const timeoutPromise = new Promise((_, reject) => {
+                                    setTimeout(() => reject(new Error(`Markdown conversion timed out after ${SCRAPE_TIMEOUT / 1000}s`)), SCRAPE_TIMEOUT);
+                                });
+                                const markdown = yield Promise.race([markdownPromise, timeoutPromise]);
+                                if (markdown && markdown.trim().length > 0) {
+                                    serializableOutput.markdown = [{ content: markdown }];
+                                    this.log('Markdown conversion completed', logger_1.Level.LOG);
+                                }
+                            }
+                            catch (error) {
+                                this.log(`Markdown conversion failed: ${error.message}`, logger_1.Level.WARN);
+                            }
+                        }
+                        if (formats.includes('html')) {
+                            try {
+                                const htmlPromise = page.evaluate(() => {
+                                    const selectors = [
+                                        "script", "style", "link[rel='stylesheet']", "noscript", "meta",
+                                        "svg", "img", "picture", "source", "video", "audio", "iframe", "object", "embed"
+                                    ];
+                                    selectors.forEach(sel => {
+                                        document.querySelectorAll(sel).forEach(e => e.remove());
+                                    });
+                                    const all = document.querySelectorAll("*");
+                                    all.forEach(el => {
+                                        [...el.attributes].forEach(attr => {
+                                            if (attr.name.startsWith("on") || attr.name === "data-mx-id") {
+                                                el.removeAttribute(attr.name);
+                                            }
+                                        });
+                                    });
+                                    return document.documentElement.outerHTML;
+                                });
+                                const timeoutPromise = new Promise((_, reject) => {
+                                    setTimeout(() => reject(new Error(`HTML conversion timed out after ${SCRAPE_TIMEOUT / 1000}s`)), SCRAPE_TIMEOUT);
+                                });
+                                const html = yield Promise.race([htmlPromise, timeoutPromise]);
+                                if (html && html.trim().length > 0) {
+                                    serializableOutput.html = [{ content: html }];
+                                    this.log('HTML conversion completed', logger_1.Level.LOG);
+                                }
+                            }
+                            catch (error) {
+                                this.log(`HTML conversion failed: ${error.message}`, logger_1.Level.WARN);
+                            }
+                        }
+                        if (formats.includes('screenshot-visible')) {
+                            try {
+                                const screenshotBuffer = yield page.screenshot({ fullPage: false, type: 'png' });
+                                if (screenshotBuffer && screenshotBuffer.length > 0) {
+                                    yield this.options.binaryCallback({
+                                        name: 'screenshot-visible',
+                                        data: screenshotBuffer,
+                                        mimeType: 'image/png'
+                                    }, 'image/png');
+                                    this.log('Visible screenshot captured', logger_1.Level.LOG);
+                                }
+                            }
+                            catch (error) {
+                                this.log(`Screenshot-visible failed: ${error.message}`, logger_1.Level.WARN);
+                            }
+                        }
+                        if (formats.includes('screenshot-fullpage')) {
+                            try {
+                                const screenshotBuffer = yield page.screenshot({ fullPage: true, type: 'png' });
+                                if (screenshotBuffer && screenshotBuffer.length > 0) {
+                                    yield this.options.binaryCallback({
+                                        name: 'screenshot-fullpage',
+                                        data: screenshotBuffer,
+                                        mimeType: 'image/png'
+                                    }, 'image/png');
+                                    this.log('Full page screenshot captured', logger_1.Level.LOG);
+                                }
+                            }
+                            catch (error) {
+                                this.log(`Screenshot-fullpage failed: ${error.message}`, logger_1.Level.WARN);
+                            }
+                        }
+                        const hasSerializableOutput = Object.keys(serializableOutput).length > 0 &&
+                            Object.values(serializableOutput).some((arr) => Array.isArray(arr) && arr.length > 0);
+                        if (hasSerializableOutput) {
+                            yield this.options.serializableCallback({ scrape: serializableOutput });
+                            this.log(`scrape completed successfully for ${url}`, logger_1.Level.LOG);
+                        }
+                        else {
+                            this.log(`scrape completed but no content could be extracted from ${url}`, logger_1.Level.WARN);
+                        }
+                    }
+                    catch (error) {
+                        this.log(`scrape action failed: ${error.message}`, logger_1.Level.ERROR);
+                        throw new Error(`scrape execution error: ${error.message}`);
+                    }
+                }),
             };
             const executeAction = (invokee, methodName, args) => __awaiter(this, void 0, void 0, function* () {
                 console.log("Executing action:", methodName, args);

package/build/utils/markdown.d.ts

@@ -0,0 +1 @@
+export declare function parseMarkdown(html: string | null | undefined, baseUrl?: string | null): Promise<string>;

package/build/utils/markdown.js

@@ -0,0 +1,153 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseMarkdown = parseMarkdown;
+function parseMarkdown(html, baseUrl) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const TurndownService = require("turndown");
+        const { gfm } = require("joplin-turndown-plugin-gfm");
+        const cheerio = require("cheerio");
+        const { URL } = require("url");
+        if (!html)
+            return "";
+        const tidiedHtml = tidyHtml(html);
+        const t = new TurndownService({
+            headingStyle: "atx", // ensures #### instead of ------
+            codeBlockStyle: "fenced",
+        });
+        // ---------------------------------------------
+        // Proper ATX headings #### instead of underline-style
+        // ---------------------------------------------
+        t.addRule("forceAtxHeadings", {
+            filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
+            replacement: (content, node) => {
+                const level = Number(node.nodeName.charAt(1));
+                const clean = content.trim();
+                return `\n${"#".repeat(level)} ${clean}\n`;
+            },
+        });
+        // ---------------------------------------------
+        // Remove SVGs
+        // ---------------------------------------------
+        t.addRule("truncate-svg", {
+            filter: "svg",
+            replacement: () => "",
+        });
+        // ---------------------------------------------
+        // Improved paragraph cleanup
+        // ---------------------------------------------
+        t.addRule("improved-paragraph", {
+            filter: "p",
+            replacement: (innerText) => {
+                const trimmed = innerText.trim();
+                if (!trimmed)
+                    return "";
+                return `${trimmed.replace(/\n{3,}/g, "\n\n")}\n\n`;
+            },
+        });
+        // ---------------------------------------------
+        // Inline link with fallback text
+        // ---------------------------------------------
+        t.addRule("inlineLink", {
+            filter: (node, opts) => node.nodeName === "A" && node.getAttribute("href"),
+            replacement: (content, node) => {
+                var _a, _b;
+                let text = content.trim();
+                // Fallback: aria-label → title → domain
+                if (!text) {
+                    text =
+                        ((_a = node.getAttribute("aria-label")) === null || _a === void 0 ? void 0 : _a.trim()) ||
+                            ((_b = node.getAttribute("title")) === null || _b === void 0 ? void 0 : _b.trim()) ||
+                            getDomainFromUrl(node.getAttribute("href")) ||
+                            "link";
+                }
+                let href = node.getAttribute("href").trim();
+                // relative → absolute
+                if (baseUrl && isRelativeUrl(href)) {
+                    try {
+                        const u = new URL(href, baseUrl);
+                        href = u.toString();
+                    }
+                    catch (_c) { }
+                }
+                href = cleanUrl(href);
+                return `[${text}](${href})`;
+            },
+        });
+        t.use(gfm);
+        // Convert HTML → Markdown
+        try {
+            let out = yield t.turndown(tidiedHtml);
+            out = fixBrokenLinks(out);
+            out = stripSkipLinks(out);
+            return out.trim();
+        }
+        catch (err) {
+            console.error("HTML→Markdown failed", { err });
+            return "";
+        }
+    });
+}
+// -----------------------------------------------------
+// Helpers
+// -----------------------------------------------------
+function isRelativeUrl(url) {
+    return !url.includes("://") && !url.startsWith("mailto:") && !url.startsWith("tel:");
+}
+function getDomainFromUrl(url) {
+    try {
+        const u = new URL(url);
+        return u.hostname.replace("www.", "");
+    }
+    catch (_a) {
+        return null;
+    }
+}
+function cleanUrl(u) {
+    return u;
+}
+function cleanAttribute(attr) {
+    return attr ? attr.replace(/(\n+\s*)+/g, "\n") : "";
+}
+function tidyHtml(html) {
+    const cheerio = require("cheerio");
+    const $ = cheerio.load(html);
+    const manuallyCleanedElements = [
+        "script",
+        "style",
+        "iframe",
+        "noscript",
+        "meta",
+        "link",
+        "object",
+        "embed",
+        "canvas",
+        "audio",
+        "video",
+    ];
+    manuallyCleanedElements.forEach((tag) => $(tag).remove());
+    return $("body").html();
+}
+function fixBrokenLinks(md) {
+    let depth = 0;
+    let result = "";
+    for (const ch of md) {
+        if (ch === "[")
+            depth++;
+        if (ch === "]")
+            depth = Math.max(0, depth - 1);
+        result += depth > 0 && ch === "\n" ? "\\\n" : ch;
+    }
+    return result;
+}
+function stripSkipLinks(md) {
+    return md.replace(/\[Skip to Content\]\(#[^\)]*\)/gi, "");
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.28",
+  "version": "0.0.31",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",
@@ -17,9 +17,16 @@
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@cliqz/adblocker-playwright": "^1.31.3",
+    "cheerio": "^1.1.2",
     "cross-fetch": "^4.0.0",
     "joi": "^17.6.0",
+    "joplin-turndown-plugin-gfm": "^1.0.12",
     "nodemailer": "^6.10.0",
-    "playwright-core": "^1.57.0"
+    "playwright-core": "^1.57.0",
+    "rimraf": "^6.1.2",
+    "turndown": "^7.2.0"
+  },
+  "devDependencies": {
+    "@types/turndown": "^5.0.6"
   }
 }