mx-cloud 0.0.31 → 0.0.33

package/build/interpret.js

@@ -1599,6 +1599,38 @@ class Interpreter extends events_1.EventEmitter {
                         }
                         const serializableOutput = {};
                         const SCRAPE_TIMEOUT = 120000;
+                        if (formats.includes('screenshot-visible')) {
+                            try {
+                                const screenshotBuffer = yield page.screenshot({ fullPage: false, type: 'png' });
+                                if (screenshotBuffer && screenshotBuffer.length > 0) {
+                                    yield this.options.binaryCallback({
+                                        name: 'screenshot-visible',
+                                        data: screenshotBuffer,
+                                        mimeType: 'image/png'
+                                    }, 'image/png');
+                                    this.log('Visible screenshot captured', logger_1.Level.LOG);
+                                }
+                            }
+                            catch (error) {
+                                this.log(`Screenshot-visible failed: ${error.message}`, logger_1.Level.WARN);
+                            }
+                        }
+                        if (formats.includes('screenshot-fullpage')) {
+                            try {
+                                const screenshotBuffer = yield page.screenshot({ fullPage: true, type: 'png' });
+                                if (screenshotBuffer && screenshotBuffer.length > 0) {
+                                    yield this.options.binaryCallback({
+                                        name: 'screenshot-fullpage',
+                                        data: screenshotBuffer,
+                                        mimeType: 'image/png'
+                                    }, 'image/png');
+                                    this.log('Full page screenshot captured', logger_1.Level.LOG);
+                                }
+                            }
+                            catch (error) {
+                                this.log(`Screenshot-fullpage failed: ${error.message}`, logger_1.Level.WARN);
+                            }
+                        }
                         if (formats.includes('text')) {
                             try {
                                 const textPromise = page.evaluate(() => {
@@ -1625,15 +1657,16 @@ class Interpreter extends events_1.EventEmitter {
                                 const html = yield page.evaluate(() => {
                                     const selectors = [
                                         "script", "style", "link[rel='stylesheet']", "noscript", "meta",
-                                        "svg", "img", "picture", "source", "video", "audio", "iframe", "object", "embed"
+                                        "iframe", "object", "embed"
                                     ];
                                     selectors.forEach(sel => {
                                         document.querySelectorAll(sel).forEach(e => e.remove());
                                     });
                                     const all = document.querySelectorAll("*");
                                     all.forEach(el => {
-                                        [...el.attributes].forEach(attr => {
-                                            if (attr.name.startsWith("on") || attr.name === "data-mx-id") {
+                                        Array.from(el.attributes).forEach(attr => {
+                                            const name = attr.name.toLowerCase();
+                                            if (name.startsWith("on") || name === "data-mx-id" || name === "jsaction" || name === "jsname") {
                                                 el.removeAttribute(attr.name);
                                             }
                                         });
@@ -1659,15 +1692,16 @@ class Interpreter extends events_1.EventEmitter {
                                 const htmlPromise = page.evaluate(() => {
                                     const selectors = [
                                         "script", "style", "link[rel='stylesheet']", "noscript", "meta",
-                                        "svg", "img", "picture", "source", "video", "audio", "iframe", "object", "embed"
+                                        "iframe", "object", "embed"
                                     ];
                                     selectors.forEach(sel => {
                                         document.querySelectorAll(sel).forEach(e => e.remove());
                                     });
                                     const all = document.querySelectorAll("*");
                                     all.forEach(el => {
-                                        [...el.attributes].forEach(attr => {
-                                            if (attr.name.startsWith("on") || attr.name === "data-mx-id") {
+                                        Array.from(el.attributes).forEach(attr => {
+                                            const name = attr.name.toLowerCase();
+                                            if (name.startsWith("on") || name === "data-mx-id" || name === "jsaction" || name === "jsname") {
                                                 el.removeAttribute(attr.name);
                                             }
                                         });
@@ -1687,38 +1721,6 @@ class Interpreter extends events_1.EventEmitter {
                                 this.log(`HTML conversion failed: ${error.message}`, logger_1.Level.WARN);
                             }
                         }
-                        if (formats.includes('screenshot-visible')) {
-                            try {
-                                const screenshotBuffer = yield page.screenshot({ fullPage: false, type: 'png' });
-                                if (screenshotBuffer && screenshotBuffer.length > 0) {
-                                    yield this.options.binaryCallback({
-                                        name: 'screenshot-visible',
-                                        data: screenshotBuffer,
-                                        mimeType: 'image/png'
-                                    }, 'image/png');
-                                    this.log('Visible screenshot captured', logger_1.Level.LOG);
-                                }
-                            }
-                            catch (error) {
-                                this.log(`Screenshot-visible failed: ${error.message}`, logger_1.Level.WARN);
-                            }
-                        }
-                        if (formats.includes('screenshot-fullpage')) {
-                            try {
-                                const screenshotBuffer = yield page.screenshot({ fullPage: true, type: 'png' });
-                                if (screenshotBuffer && screenshotBuffer.length > 0) {
-                                    yield this.options.binaryCallback({
-                                        name: 'screenshot-fullpage',
-                                        data: screenshotBuffer,
-                                        mimeType: 'image/png'
-                                    }, 'image/png');
-                                    this.log('Full page screenshot captured', logger_1.Level.LOG);
-                                }
-                            }
-                            catch (error) {
-                                this.log(`Screenshot-fullpage failed: ${error.message}`, logger_1.Level.WARN);
-                            }
-                        }
                         const hasSerializableOutput = Object.keys(serializableOutput).length > 0 &&
                             Object.values(serializableOutput).some((arr) => Array.isArray(arr) && arr.length > 0);
                         if (hasSerializableOutput) {

package/build/utils/markdown.js

@@ -1,4 +1,37 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -8,20 +41,25 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseMarkdown = parseMarkdown;
+const turndown_1 = __importDefault(require("turndown"));
+// @ts-ignore
+const joplin_turndown_plugin_gfm_1 = require("joplin-turndown-plugin-gfm");
+const cheerio = __importStar(require("cheerio"));
+const url_1 = require("url");
 function parseMarkdown(html, baseUrl) {
     return __awaiter(this, void 0, void 0, function* () {
-        const TurndownService = require("turndown");
-        const { gfm } = require("joplin-turndown-plugin-gfm");
-        const cheerio = require("cheerio");
-        const { URL } = require("url");
         if (!html)
             return "";
         const tidiedHtml = tidyHtml(html);
-        const t = new TurndownService({
+        const t = new turndown_1.default({
             headingStyle: "atx", // ensures #### instead of ------
             codeBlockStyle: "fenced",
+            bulletListMarker: "-",
         });
         // ---------------------------------------------
         // Proper ATX headings #### instead of underline-style
@@ -31,18 +69,20 @@ function parseMarkdown(html, baseUrl) {
             replacement: (content, node) => {
                 const level = Number(node.nodeName.charAt(1));
                 const clean = content.trim();
+                if (!clean)
+                    return "";
                 return `\n${"#".repeat(level)} ${clean}\n`;
             },
         });
         // ---------------------------------------------
-        // Remove SVGs
+        // Remove SVGs and other typically noisy elements
         // ---------------------------------------------
         t.addRule("truncate-svg", {
-            filter: "svg",
+            filter: (node) => node.nodeName.toLowerCase() === "svg",
             replacement: () => "",
         });
         // ---------------------------------------------
-        // Improved paragraph cleanup
+        // Paragraph cleanup
         // ---------------------------------------------
         t.addRule("improved-paragraph", {
             filter: "p",
@@ -50,17 +90,17 @@ function parseMarkdown(html, baseUrl) {
                 const trimmed = innerText.trim();
                 if (!trimmed)
                     return "";
-                return `${trimmed.replace(/\n{3,}/g, "\n\n")}\n\n`;
+                return `\n\n${trimmed.replace(/\n{3,}/g, "\n\n")}\n\n`;
             },
         });
         // ---------------------------------------------
         // Inline link with fallback text
         // ---------------------------------------------
         t.addRule("inlineLink", {
-            filter: (node, opts) => node.nodeName === "A" && node.getAttribute("href"),
+            filter: (node) => node.nodeName === "A" && node.getAttribute("href"),
             replacement: (content, node) => {
                 var _a, _b;
-                let text = content.trim();
+                let text = content.trim().replace(/\n+/g, " ");
                 // Fallback: aria-label → title → domain
                 if (!text) {
                     text =
@@ -73,7 +113,7 @@ function parseMarkdown(html, baseUrl) {
                 // relative → absolute
                 if (baseUrl && isRelativeUrl(href)) {
                     try {
-                        const u = new URL(href, baseUrl);
+                        const u = new url_1.URL(href, baseUrl);
                         href = u.toString();
                     }
                     catch (_c) { }
@@ -82,12 +122,33 @@ function parseMarkdown(html, baseUrl) {
                 return `[${text}](${href})`;
             },
         });
-        t.use(gfm);
+        // ---------------------------------------------
+        // Image handling
+        // ---------------------------------------------
+        t.addRule("images", {
+            filter: "img",
+            replacement: (_content, node) => {
+                var _a, _b, _c;
+                const alt = ((_a = node.getAttribute("alt")) === null || _a === void 0 ? void 0 : _a.trim()) || ((_b = node.getAttribute("title")) === null || _b === void 0 ? void 0 : _b.trim()) || "";
+                let src = ((_c = node.getAttribute("src")) === null || _c === void 0 ? void 0 : _c.trim()) || "";
+                if (!src)
+                    return "";
+                if (baseUrl && isRelativeUrl(src)) {
+                    try {
+                        src = new url_1.URL(src, baseUrl).toString();
+                    }
+                    catch (_d) { }
+                }
+                return alt ? `![${alt}](${src})` : `[Image](${src})`;
+            },
+        });
+        t.use(joplin_turndown_plugin_gfm_1.gfm);
         // Convert HTML → Markdown
         try {
-            let out = yield t.turndown(tidiedHtml);
+            let out = t.turndown(tidiedHtml);
             out = fixBrokenLinks(out);
             out = stripSkipLinks(out);
+            out = cleanupExtraWhitespace(out);
             return out.trim();
         }
         catch (err) {
@@ -100,11 +161,13 @@ function parseMarkdown(html, baseUrl) {
 // Helpers
 // -----------------------------------------------------
 function isRelativeUrl(url) {
-    return !url.includes("://") && !url.startsWith("mailto:") && !url.startsWith("tel:");
+    if (!url)
+        return false;
+    return !url.includes("://") && !url.startsWith("mailto:") && !url.startsWith("tel:") && !url.startsWith("data:");
 }
 function getDomainFromUrl(url) {
     try {
-        const u = new URL(url);
+        const u = new url_1.URL(url);
         return u.hostname.replace("www.", "");
     }
     catch (_a) {
@@ -112,29 +175,86 @@ function getDomainFromUrl(url) {
     }
 }
 function cleanUrl(u) {
-    return u;
-}
-function cleanAttribute(attr) {
-    return attr ? attr.replace(/(\n+\s*)+/g, "\n") : "";
+    return u.split("#")[0];
 }
 function tidyHtml(html) {
-    const cheerio = require("cheerio");
     const $ = cheerio.load(html);
-    const manuallyCleanedElements = [
-        "script",
-        "style",
-        "iframe",
-        "noscript",
-        "meta",
-        "link",
-        "object",
-        "embed",
-        "canvas",
-        "audio",
-        "video",
+    const technicalElements = [
+        "script", "style", "iframe", "noscript", "meta", "link", "object",
+        "embed", "canvas", "audio", "video", "svg", "map", "area"
+    ];
+    technicalElements.forEach((tag) => $(tag).remove());
+    const noiseSelectors = [
+        "nav", "header", "footer", "aside",
+        ".nav", ".header", ".footer", ".sidebar", ".menu", ".ads", ".ad", ".advertisement",
+        "#nav", "#header", "#footer", "#sidebar", ".breadcrumb", ".social-share",
+        ".comments", ".popup", ".modal", ".cookie-banner", ".location-widget",
+        ".keyboard-shortcuts", ".skip-link", ".banner", ".top-bar", ".nav-bar",
+        '[role="navigation"]', '[role="banner"]', '[role="contentinfo"]',
+        '[role="complementary"]', "#shortcut-menu", ".nav-sprite", ".a-header", ".a-footer",
+        ".gb_wa", ".gb_xa",
+        "#nav-belt", "#nav-main", "#nav-footer"
     ];
-    manuallyCleanedElements.forEach((tag) => $(tag).remove());
-    return $("body").html();
+    noiseSelectors.forEach((sel) => $(sel).remove());
+    const uiArtifacts = ["Undo", "Done", "Edit", "Viewed categories", "Dismiss", "Close", "View detail", "View more"];
+    $("button, span, a, div").each((_i, el) => {
+        const text = $(el).text().trim();
+        if (uiArtifacts.includes(text) && $(el).children().length === 0) {
+            $(el).remove();
+        }
+    });
+    const mainSelectors = ["main", "article", "#main-content", "#content", ".main", ".content", ".article", ".post-content", "[role='main']"];
+    let bestContent = null;
+    for (const selector of mainSelectors) {
+        const el = $(selector);
+        if (el.length > 0) {
+            let candidate = el.first();
+            let maxLen = candidate.text().length;
+            el.each((_idx, elem) => {
+                const len = $(elem).text().length;
+                if (len > maxLen) {
+                    maxLen = len;
+                    candidate = $(elem);
+                }
+            });
+            if (maxLen > 100) {
+                bestContent = candidate;
+                break;
+            }
+        }
+    }
+    let contentToProcess = bestContent || $("body");
+    contentToProcess.find("div, ul, section").each((_i, el) => {
+        const $el = $(el);
+        const children = $el.children();
+        if (children.length > 10) {
+            const tagCounts = {};
+            children.each((_idx, child) => {
+                const tag = child.tagName || child.name;
+                if (tag) {
+                    tagCounts[tag] = (tagCounts[tag] || 0) + 1;
+                }
+            });
+            const dominantTag = Object.keys(tagCounts).find(tag => tagCounts[tag] > 15);
+            if (dominantTag && $el.text().length / children.length < 30) {
+                $el.remove();
+            }
+        }
+    });
+    contentToProcess.find("ul, ol").each((_i, el) => {
+        const $el = $(el);
+        const items = $el.children("li");
+        if (items.length > 40) {
+            items.slice(40).remove();
+            $el.append("<li>... (further items truncated for readability)</li>");
+        }
+    });
+    const title = $("title").text().trim() || $("h1").first().text().trim();
+    let resultHtml = contentToProcess.html() || "";
+    if (title && !resultHtml.includes(title)) {
+        resultHtml = `<h1>${title}</h1>\n${resultHtml}`;
+    }
+    return resultHtml;
 }
 function fixBrokenLinks(md) {
     let depth = 0;
@@ -151,3 +271,9 @@ function fixBrokenLinks(md) {
 function stripSkipLinks(md) {
     return md.replace(/\[Skip to Content\]\(#[^\)]*\)/gi, "");
 }
+function cleanupExtraWhitespace(md) {
+    return md
+        .replace(/\n{3,}/g, "\n\n")
+        .replace(/[ \t]+\n/g, "\n")
+        .replace(/\n[ \t]+/g, "\n");
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mx-cloud",
-  "version": "0.0.31",
+  "version": "0.0.33",
   "description": "mx cloud",
   "main": "build/index.js",
   "typings": "build/index.d.ts",
@@ -27,6 +27,7 @@
     "turndown": "^7.2.0"
   },
   "devDependencies": {
+    "@types/node": "^25.2.0",
     "@types/turndown": "^5.0.6"
   }
 }