mx-cloud 0.0.28 → 0.0.30

package/build/interpret.js

@@ -54,6 +54,7 @@ const concurrency_1 = __importDefault(require("./utils/concurrency"));
 const preprocessor_1 = __importDefault(require("./preprocessor"));
 const logger_1 = __importStar(require("./utils/logger"));
 const selector_1 = require("./selector");
+const markdown_1 = require("./utils/markdown");
 /**
  * Class for running the Smart Workflows.
  */
@@ -922,14 +923,60 @@ class Interpreter extends events_1.EventEmitter {
                                     metadata: Object.assign(Object.assign({}, allMetadata), { title, language: document.documentElement.lang || '', favicon: ((_b = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]')) === null || _b === void 0 ? void 0 : _b.href) || '', statusCode: 200 })
                                 };
                             });
-                            return {
+                            const result = {
                                 metadata: Object.assign(Object.assign({}, pageData.metadata), { url: url, sourceURL: url }),
-                                html: pageData.html,
-                                text: pageData.text,
-                                links: pageData.links,
-                                wordCount: pageData.wordCount,
                                 scrapedAt: new Date().toISOString()
                             };
+                            const formats = crawlConfig.outputFormats || [];
+                            if (formats.includes('text')) {
+                                result.text = pageData.text;
+                                result.wordCount = pageData.wordCount;
+                            }
+                            if (formats.includes('html')) {
+                                result.html = pageData.html;
+                                result.links = pageData.links;
+                            }
+                            if (formats.includes('markdown')) {
+                                try {
+                                    const markdown = yield (0, markdown_1.parseMarkdown)(pageData.html, url);
+                                    result.markdown = markdown;
+                                }
+                                catch (err) {
+                                    this.log(`Markdown conversion failed for ${url}: ${err.message}`, logger_1.Level.WARN);
+                                    result.markdown = '';
+                                }
+                            }
+                            if (formats.includes('screenshot-visible')) {
+                                try {
+                                    const screenshotBuffer = yield page.screenshot({ fullPage: false });
+                                    const screenshotName = `Crawl - ${crawlResults.length} - Visible`;
+                                    yield this.options.binaryCallback({
+                                        name: screenshotName,
+                                        data: screenshotBuffer,
+                                        mimeType: 'image/png'
+                                    }, 'image/png');
+                                    result.screenshotVisible = screenshotName;
+                                }
+                                catch (err) {
+                                    this.log(`Screenshot-visible failed for ${url}: ${err.message}`, logger_1.Level.WARN);
+                                }
+                            }
+                            if (formats.includes('screenshot-fullpage')) {
+                                try {
+                                    const screenshotBuffer = yield page.screenshot({ fullPage: true });
+                                    const screenshotName = `Crawl - ${crawlResults.length} - Full Page`;
+                                    yield this.options.binaryCallback({
+                                        name: screenshotName,
+                                        data: screenshotBuffer,
+                                        mimeType: 'image/png'
+                                    }, 'image/png');
+                                    result.screenshotFullpage = screenshotName;
+                                }
+                                catch (err) {
+                                    this.log(`Screenshot-fullpage failed for ${url}: ${err.message}`, logger_1.Level.WARN);
+                                }
+                            }
+                            return result;
                         });
                         const visitedUrls = new Set();
                         const crawlResults = [];
@@ -1036,15 +1083,28 @@ class Interpreter extends events_1.EventEmitter {
                                     yield new Promise(resolve => setTimeout(resolve, robotRules.crawlDelay));
                                 }
                                 yield page.goto(url, {
-                                    waitUntil: 'domcontentloaded',
-                                    timeout: 30000
+                                    waitUntil: 'load',
+                                    timeout: 60000
                                 }).catch((err) => {
                                     throw new Error(`Navigation failed: ${err.message}`);
                                 });
-                                yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => { });
+                                yield page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => { });
                                 const pageResult = yield scrapePageContent(url);
                                 pageResult.metadata.depth = depth;
                                 crawlResults.push(pageResult);
+                                const actionType = "crawl";
+                                const actionName = "Crawl Results";
+                                if (!this.serializableDataByType[actionType]) {
+                                    this.serializableDataByType[actionType] = {};
+                                }
+                                this.serializableDataByType[actionType][actionName] = [...crawlResults];
+                                yield this.options.serializableCallback({
+                                    crawl: this.serializableDataByType.crawl
+                                });
+                                if (this.isAborted) {
+                                    this.log(`Run aborted after scraping ${url}, stopping crawl`, logger_1.Level.WARN);
+                                    break;
+                                }
                                 this.log(`✓ Scraped ${url} (${pageResult.wordCount} words, depth ${depth})`, logger_1.Level.LOG);
                                 if (crawlConfig.followLinks && depth < crawlConfig.maxDepth) {
                                     const newLinks = yield extractLinksFromPage();
@@ -1308,6 +1368,7 @@ class Interpreter extends events_1.EventEmitter {
                                 filters: searchConfig.filters || {},
                                 resultsCount: searchResults.length,
                                 results: searchResults,
+                                mode: searchConfig.mode,
                                 searchedAt: new Date().toISOString()
                             };
                             this.serializableDataByType[actionType][actionName] = searchData;
@@ -1323,16 +1384,25 @@ class Interpreter extends events_1.EventEmitter {
                         this.log(`Starting to scrape content from ${searchResults.length} search results...`, logger_1.Level.LOG);
                         const scrapedResults = [];
                         for (let i = 0; i < searchResults.length; i++) {
+                            if (this.isAborted) {
+                                this.log(`Run aborted, stopping search scraping at result ${i + 1}/${searchResults.length}`, logger_1.Level.WARN);
+                                break;
+                            }
                             const result = searchResults[i];
                             try {
                                 this.log(`[${i + 1}/${searchResults.length}] Scraping: ${result.url}`, logger_1.Level.LOG);
+                                let navigationFailed = false;
                                 yield page.goto(result.url, {
-                                    waitUntil: 'domcontentloaded',
-                                    timeout: 30000
+                                    waitUntil: 'load',
+                                    timeout: 60000
                                 }).catch(() => {
                                     this.log(`Failed to navigate to ${result.url}, skipping...`, logger_1.Level.WARN);
+                                    navigationFailed = true;
                                 });
-                                yield page.waitForLoadState('load', { timeout: 10000 }).catch(() => { });
+                                if (navigationFailed) {
+                                    continue;
+                                }
+                                yield page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => { });
                                 const pageData = yield page.evaluate(() => {
                                     var _a, _b;
                                     const getMeta = (name) => {
@@ -1368,7 +1438,7 @@ class Interpreter extends events_1.EventEmitter {
                                         metadata: Object.assign(Object.assign({}, allMetadata), { title, language: document.documentElement.lang || '', favicon: ((_b = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]')) === null || _b === void 0 ? void 0 : _b.href) || '', statusCode: 200 })
                                     };
                                 });
-                                scrapedResults.push({
+                                const scrapedResult = {
                                     searchResult: {
                                         query: searchConfig.query,
                                         position: result.position,
@@ -1376,12 +1446,79 @@ class Interpreter extends events_1.EventEmitter {
                                         searchDescription: result.description,
                                     },
                                     metadata: Object.assign(Object.assign({}, pageData.metadata), { url: result.url, sourceURL: result.url }),
-                                    html: pageData.html,
-                                    text: pageData.text,
-                                    links: pageData.links,
-                                    wordCount: pageData.wordCount,
                                     scrapedAt: new Date().toISOString()
+                                };
+                                const formats = searchConfig.outputFormats || [];
+                                if (formats.includes('text')) {
+                                    scrapedResult.text = pageData.text;
+                                    scrapedResult.wordCount = pageData.wordCount;
+                                }
+                                if (formats.includes('html')) {
+                                    scrapedResult.html = pageData.html;
+                                    scrapedResult.links = pageData.links;
+                                }
+                                if (formats.includes('markdown')) {
+                                    try {
+                                        const markdown = yield (0, markdown_1.parseMarkdown)(pageData.html, result.url);
+                                        scrapedResult.markdown = markdown;
+                                    }
+                                    catch (err) {
+                                        this.log(`Markdown conversion failed for ${result.url}: ${err.message}`, logger_1.Level.WARN);
+                                        scrapedResult.markdown = '';
+                                    }
+                                }
+                                if (formats.includes('screenshot-visible')) {
+                                    try {
+                                        const screenshotBuffer = yield page.screenshot({ fullPage: false });
+                                        const screenshotName = `Search - ${i} - Visible`;
+                                        yield this.options.binaryCallback({
+                                            name: screenshotName,
+                                            data: screenshotBuffer,
+                                            mimeType: 'image/png'
+                                        }, 'image/png');
+                                        scrapedResult.screenshotVisible = screenshotName;
+                                    }
+                                    catch (err) {
+                                        this.log(`Screenshot-visible failed for ${result.url}: ${err.message}`, logger_1.Level.WARN);
+                                    }
+                                }
+                                if (formats.includes('screenshot-fullpage')) {
+                                    try {
+                                        const screenshotBuffer = yield page.screenshot({ fullPage: true });
+                                        const screenshotName = `Search - ${i} - Full Page`;
+                                        yield this.options.binaryCallback({
+                                            name: screenshotName,
+                                            data: screenshotBuffer,
+                                            mimeType: 'image/png'
+                                        }, 'image/png');
+                                        scrapedResult.screenshotFullpage = screenshotName;
+                                    }
+                                    catch (err) {
+                                        this.log(`Screenshot-fullpage failed for ${result.url}: ${err.message}`, logger_1.Level.WARN);
+                                    }
+                                }
+                                scrapedResults.push(scrapedResult);
+                                const actionType = "search";
+                                const actionName = "Search Results";
+                                if (!this.serializableDataByType[actionType]) {
+                                    this.serializableDataByType[actionType] = {};
+                                }
+                                this.serializableDataByType[actionType][actionName] = {
+                                    query: searchConfig.query,
+                                    provider: searchConfig.provider,
+                                    filters: searchConfig.filters || {},
+                                    resultsCount: scrapedResults.length,
+                                    results: scrapedResults,
+                                    mode: searchConfig.mode,
+                                    searchedAt: new Date().toISOString()
+                                };
+                                yield this.options.serializableCallback({
+                                    search: this.serializableDataByType.search
                                 });
+                                if (this.isAborted) {
+                                    this.log(`Run aborted after scraping ${result.url}, stopping search`, logger_1.Level.WARN);
+                                    break;
+                                }
                                 this.log(`✓ Scraped ${result.url} (${pageData.wordCount} words)`, logger_1.Level.LOG);
                             }
                             catch (error) {

package/build/utils/markdown.d.ts

@@ -0,0 +1 @@
+export declare function parseMarkdown(html: string | null | undefined, baseUrl?: string | null): Promise<string>;

package/build/utils/markdown.js

@@ -0,0 +1,153 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseMarkdown = parseMarkdown;
+function parseMarkdown(html, baseUrl) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const TurndownService = require("turndown");
+        const { gfm } = require("joplin-turndown-plugin-gfm");
+        const cheerio = require("cheerio");
+        const { URL } = require("url");
+        if (!html)
+            return "";
+        const tidiedHtml = tidyHtml(html);
+        const t = new TurndownService({
+            headingStyle: "atx", // ensures #### instead of ------
+            codeBlockStyle: "fenced",
+        });
+        // ---------------------------------------------
+        // Proper ATX headings #### instead of underline-style
+        // ---------------------------------------------
+        t.addRule("forceAtxHeadings", {
+            filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
+            replacement: (content, node) => {
+                const level = Number(node.nodeName.charAt(1));
+                const clean = content.trim();
+                return `\n${"#".repeat(level)} ${clean}\n`;
+            },
+        });
+        // ---------------------------------------------
+        // Remove SVGs
+        // ---------------------------------------------
+        t.addRule("truncate-svg", {
+            filter: "svg",
+            replacement: () => "",
+        });
+        // ---------------------------------------------
+        // Improved paragraph cleanup
+        // ---------------------------------------------
+        t.addRule("improved-paragraph", {
+            filter: "p",
+            replacement: (innerText) => {
+                const trimmed = innerText.trim();
+                if (!trimmed)
+                    return "";
+                return `${trimmed.replace(/\n{3,}/g, "\n\n")}\n\n`;
+            },
+        });
+        // ---------------------------------------------
+        // Inline link with fallback text
+        // ---------------------------------------------
+        t.addRule("inlineLink", {
+            filter: (node, opts) => node.nodeName === "A" && node.getAttribute("href"),
+            replacement: (content, node) => {
+                var _a, _b;
+                let text = content.trim();
+                // Fallback: aria-label → title → domain
+                if (!text) {
+                    text =
+                        ((_a = node.getAttribute("aria-label")) === null || _a === void 0 ? void 0 : _a.trim()) ||
+                            ((_b = node.getAttribute("title")) === null || _b === void 0 ? void 0 : _b.trim()) ||
+                            getDomainFromUrl(node.getAttribute("href")) ||
+                            "link";
+                }
+                let href = node.getAttribute("href").trim();
+                // relative → absolute
+                if (baseUrl && isRelativeUrl(href)) {
+                    try {
+                        const u = new URL(href, baseUrl);
+                        href = u.toString();
+                    }
+                    catch (_c) { }
+                }
+                href = cleanUrl(href);
+                return `[${text}](${href})`;
+            },
+        });
+        t.use(gfm);
+        // Convert HTML → Markdown
+        try {
+            let out = yield t.turndown(tidiedHtml);
+            out = fixBrokenLinks(out);
+            out = stripSkipLinks(out);
+            return out.trim();
+        }
+        catch (err) {
+            console.error("HTML→Markdown failed", { err });
+            return "";
+        }
+    });
+}
+// -----------------------------------------------------
+// Helpers
+// -----------------------------------------------------
+function isRelativeUrl(url) {
+    return !url.includes("://") && !url.startsWith("mailto:") && !url.startsWith("tel:");
+}
+function getDomainFromUrl(url) {
+    try {
+        const u = new URL(url);
+        return u.hostname.replace("www.", "");
+    }
+    catch (_a) {
+        return null;
+    }
+}
+function cleanUrl(u) {
+    return u;
+}
+function cleanAttribute(attr) {
+    return attr ? attr.replace(/(\n+\s*)+/g, "\n") : "";
+}
+function tidyHtml(html) {
+    const cheerio = require("cheerio");
+    const $ = cheerio.load(html);
+    const manuallyCleanedElements = [
+        "script",
+        "style",
+        "iframe",
+        "noscript",
+        "meta",
+        "link",
+        "object",
+        "embed",
+        "canvas",
+        "audio",
+        "video",
+    ];
+    manuallyCleanedElements.forEach((tag) => $(tag).remove());
+    return $("body").html();
+}
+function fixBrokenLinks(md) {
+    let depth = 0;
+    let result = "";
+    for (const ch of md) {
+        if (ch === "[")
+            depth++;
+        if (ch === "]")
+            depth = Math.max(0, depth - 1);
+        result += depth > 0 && ch === "\n" ? "\\\n" : ch;
+    }
+    return result;
+}
+function stripSkipLinks(md) {
+    return md.replace(/\[Skip to Content\]\(#[^\)]*\)/gi, "");
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.28",
+  "version": "0.0.30",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",
@@ -17,9 +17,16 @@
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@cliqz/adblocker-playwright": "^1.31.3",
+    "cheerio": "^1.1.2",
     "cross-fetch": "^4.0.0",
     "joi": "^17.6.0",
+    "joplin-turndown-plugin-gfm": "^1.0.12",
     "nodemailer": "^6.10.0",
-    "playwright-core": "^1.57.0"
+    "playwright-core": "^1.57.0",
+    "rimraf": "^6.1.2",
+    "turndown": "^7.2.0"
+  },
+  "devDependencies": {
+    "@types/turndown": "^5.0.6"
   }
 }