decant-core 1.0.0 → 1.1.0

package/package.json

@@ -1,16 +1,20 @@
 {
   "name": "decant-core",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Shared AI chat platform parsers and detection for Covai browser extensions.",
   "type": "module",
   "main": "ai/index.js",
   "exports": {
     ".": "./ai/index.js",
     "./ai/*": "./ai/*.js",
+    "./web": "./web/index.js",
+    "./web/*": "./web/*.js",
+    "./article": "./web/article.js",
     "./detection/*": "./detection/*.js"
   },
   "files": [
     "ai",
+    "web",
     "detection",
     "lib",
     "utils",
@@ -26,6 +30,7 @@
   },
   "homepage": "https://github.com/Covai-Labs/decant-core#readme",
   "scripts": {
+    "test": "node --test tests/*.test.js",
     "lint": "eslint .",
     "format": "prettier --write .",
     "format:check": "prettier --check ."
@@ -41,16 +46,30 @@
     "copilot",
     "browser-extension"
   ],
-  "author": "Rat-S",
+  "author": "deadrat-in",
   "license": "AGPL-3.0-only",
+  "publishConfig": {
+    "access": "public"
+  },
   "dependencies": {
-    "turndown": "^7.2.4"
+    "@extractus/article-extractor": "^9.0.0",
+    "@mozilla/readability": "^0.6.0",
+    "defuddle": "^0.19.2",
+    "turndown": "^7.2.4",
+    "turndown-plugin-gfm": "^1.0.2"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
+    "@semantic-release/changelog": "^7.0.0",
+    "@semantic-release/commit-analyzer": "^13.0.1",
+    "@semantic-release/git": "^11.0.1",
+    "@semantic-release/github": "^12.0.9",
+    "@semantic-release/npm": "^13.1.5",
+    "@semantic-release/release-notes-generator": "^14.1.1",
     "eslint": "^10.8.0",
     "globals": "^17.9.0",
-    "prettier": "^3.9.6"
+    "linkedom": "^0.18.13",
+    "prettier": "^3.9.6",
+    "semantic-release": "^25.0.9"
   }
 }
-

package/utils/html-to-markdown.js

@@ -12,7 +12,8 @@
  * @param {Object} options - Optional configuration for the conversion
  * @returns {string} Markdown formatted text
  */
-import TurndownService from '../lib/turndown.js';
+import TurndownService from "../lib/turndown.js";
+import { strikethrough, taskListItems } from "turndown-plugin-gfm";
 
 /**
  * Convert an HTML element to markdown
@@ -23,177 +24,197 @@ import TurndownService from '../lib/turndown.js';
 export function convertToMarkdown(htmlContent, options = {}) {
   // Configure Turndown service
   const turndownService = new TurndownService({
-    headingStyle: 'atx', // Use # for headings (not underline style)
-    hr: '* * *', // Horizontal rules
-    bulletListMarker: '*', // Use * for bullet lists
-    codeBlockStyle: 'fenced', // Use ``` for code blocks
-    fence: '```', // Code block fence characters
-    emDelimiter: '*', // Use * for emphasis
-    strongDelimiter: '**', // Use ** for strong
-    linkStyle: 'inlined', // [text](url) format
-    linkReferenceStyle: 'full', // Full reference style for links
+    headingStyle: "atx", // Use # for headings (not underline style)
+    hr: "* * *", // Horizontal rules
+    bulletListMarker: "*", // Use * for bullet lists
+    codeBlockStyle: "fenced", // Use ``` for code blocks
+    fence: "```", // Code block fence characters
+    emDelimiter: "*", // Use * for emphasis
+    strongDelimiter: "**", // Use ** for strong
+    linkStyle: "inlined", // [text](url) format
+    linkReferenceStyle: "full", // Full reference style for links
     ...options,
   });
 
+  // Enable GFM plugins (strikethrough and task list checkboxes)
+  turndownService.use([strikethrough, taskListItems]);
+
   // Add custom rules for better conversion
 
   // Preserve line breaks
-  turndownService.addRule('lineBreak', {
-    filter: ['br'],
-    replacement: () => '  \n',
+  turndownService.addRule("lineBreak", {
+    filter: ["br"],
+    replacement: () => "  \n",
   });
 
   // Handle Qwen's Monaco editor code blocks
-  turndownService.addRule('qwenCodeBlock', {
+  turndownService.addRule("qwenCodeBlock", {
     filter: function (node) {
-      return node.nodeName === 'PRE' && node.classList.contains('qwen-markdown-code');
+      return (
+        node.nodeName === "PRE" && node.classList.contains("qwen-markdown-code")
+      );
     },
     replacement: function (content, node) {
       // Extract language from header
-      let language = '';
-      const headerDiv = node.querySelector('.qwen-markdown-code-header div');
+      let language = "";
+      const headerDiv = node.querySelector(".qwen-markdown-code-header div");
       if (headerDiv) {
         language = headerDiv.textContent.trim();
       }
 
       // Extract code from Monaco editor view-lines
-      const viewLines = node.querySelectorAll('.view-line');
+      const viewLines = node.querySelectorAll(".view-line");
       const codeLines = [];
       viewLines.forEach((line) => {
         // Get text content, preserving structure
-        let lineText = '';
-        const spans = line.querySelectorAll('span');
+        let lineText = "";
+        const spans = line.querySelectorAll("span");
         spans.forEach((span) => {
           // Replace   with spaces and get text
-          const text = span.textContent.replace(/\u00A0/g, ' ');
+          const text = span.textContent.replace(/\u00A0/g, " ");
           lineText += text;
         });
         codeLines.push(lineText);
       });
 
-      const code = codeLines.join('\n');
-      if (!code.trim()) return '';
+      const code = codeLines.join("\n");
+      if (!code.trim()) return "";
 
-      return '\n```' + language + '\n' + code + '\n```\n';
+      return "\n```" + language + "\n" + code + "\n```\n";
     },
   });
 
   // Add table conversion rule
-  turndownService.addRule('tables', {
-    filter: 'table',
+  turndownService.addRule("tables", {
+    filter: "table",
     replacement: function (content, node) {
       // Convert HTML table to markdown table
-      const rows = Array.from(node.querySelectorAll('tr'));
+      const rows = Array.from(node.querySelectorAll("tr"));
       if (rows.length === 0) return content;
 
       // Create a separate Turndown service for cell content to avoid recursion/state issues
       // and incorrectly stripping or double-processing content
       const cellTurndown = new TurndownService({
-        emDelimiter: '*',
-        strongDelimiter: '**',
+        emDelimiter: "*",
+        strongDelimiter: "**",
       });
 
       // Handle breaks in cells
       // Convert BR to newlines first so Turndown sees them as breaks (or handle directly)
       // Actually Turndown by default drops BRs or converts to newline.
       // We want to force them to <br> string for table cells.
-      cellTurndown.addRule('lineBreak', {
-        filter: ['br'],
-        replacement: () => '<br>',
+      cellTurndown.addRule("lineBreak", {
+        filter: ["br"],
+        replacement: () => "<br>",
       });
 
-      let markdown = '\n';
+      let markdown = "\n";
       rows.forEach((row, rowIndex) => {
-        const cells = Array.from(row.querySelectorAll('th, td'));
+        const cells = Array.from(row.querySelectorAll("th, td"));
 
         // Convert each cell's HTML to markdown using the isolated service
         const cellContents = cells.map((cell) => {
-          let cellMarkdown = cellTurndown.turndown(cell.innerHTML);
+          let cellMarkdown = cellTurndown.turndown(cell);
           // Replace any actual newlines that Turndown generated (e.g. from P tags) with <br>
+
           // as tables cannot have literal newlines in GFM.
-          return cellMarkdown.trim().replace(/\n/g, '<br>');
+          return cellMarkdown.trim().replace(/\n/g, "<br>");
         });
 
         // Add row
-        markdown += '| ' + cellContents.join(' | ') + ' |\n';
+        markdown += "| " + cellContents.join(" | ") + " |\n";
 
         // Add separator after header row
         if (rowIndex === 0) {
-          markdown += '| ' + cells.map(() => '---').join(' | ') + ' |\n';
+          markdown += "| " + cells.map(() => "---").join(" | ") + " |\n";
         }
       });
 
-      return markdown + '\n';
+      return markdown + "\n";
     },
   });
 
   // Convert the HTML
   let html;
-  if (typeof htmlContent === 'string') {
+  if (typeof htmlContent === "string") {
     html = htmlContent;
   } else if (
     htmlContent &&
-    (htmlContent instanceof HTMLElement ||
+    ((typeof HTMLElement !== "undefined" &&
+      htmlContent instanceof HTMLElement) ||
       htmlContent.nodeType === 1 ||
       htmlContent.nodeType === 9 ||
-      typeof htmlContent.querySelectorAll === 'function')
+      typeof htmlContent.querySelectorAll === "function")
   ) {
     // Clone the element to avoid modifying the original
     const clone = htmlContent.cloneNode(true);
 
     // Convert math equations (KaTeX and data-math) to standard markdown math blocks
     // 1. Process block display KaTeX
-    clone.querySelectorAll('.katex-display').forEach((el) => {
-      const annotation = el.querySelector('annotation[encoding="application/x-tex"]');
+    clone.querySelectorAll(".katex-display").forEach((el) => {
+      const annotation = el.querySelector(
+        'annotation[encoding="application/x-tex"]',
+      );
       if (annotation) {
         const latex = annotation.textContent.trim();
-        const textNode = clone.ownerDocument.createTextNode(`\n\n$$${latex}$$\n\n`);
+        const textNode = clone.ownerDocument.createTextNode(
+          `\n\n$$${latex}$$\n\n`,
+        );
         el.parentNode.replaceChild(textNode, el);
       } else {
-        const textNode = clone.ownerDocument.createTextNode(`\n\n$$${el.textContent.trim()}$$\n\n`);
+        const textNode = clone.ownerDocument.createTextNode(
+          `\n\n$$${el.textContent.trim()}$$\n\n`,
+        );
         el.parentNode.replaceChild(textNode, el);
       }
     });
 
     // 2. Process inline KaTeX
-    clone.querySelectorAll('.katex').forEach((el) => {
+    clone.querySelectorAll(".katex").forEach((el) => {
       if (!el.parentNode) return;
-      const annotation = el.querySelector('annotation[encoding="application/x-tex"]');
+      const annotation = el.querySelector(
+        'annotation[encoding="application/x-tex"]',
+      );
       if (annotation) {
         const latex = annotation.textContent.trim();
         const textNode = clone.ownerDocument.createTextNode(`$${latex}$`);
         el.parentNode.replaceChild(textNode, el);
       } else {
-        const textNode = clone.ownerDocument.createTextNode(`$${el.textContent.trim()}$`);
+        const textNode = clone.ownerDocument.createTextNode(
+          `$${el.textContent.trim()}$`,
+        );
         el.parentNode.replaceChild(textNode, el);
       }
     });
 
     // 3. Process Gemini-style data-math attributes
-    clone.querySelectorAll('[data-math]').forEach((el) => {
+    clone.querySelectorAll("[data-math]").forEach((el) => {
       if (!el.parentNode) return;
-      const latex = el.getAttribute('data-math');
-      const isBlock = el.classList.contains('math-block') || el.tagName === 'DIV';
+      const latex = el.getAttribute("data-math");
+      const isBlock =
+        el.classList.contains("math-block") || el.tagName === "DIV";
       const replacementText = isBlock ? `\n\n$$${latex}$$\n\n` : `$${latex}$`;
       const textNode = clone.ownerDocument.createTextNode(replacementText);
       el.parentNode.replaceChild(textNode, el);
     });
 
     // 4. Process Google Search SGE LaTeX images with [data-xpm-latex]
-    clone.querySelectorAll('[data-xpm-latex]').forEach((el) => {
-      const copyRoot = el.closest('[data-xpm-copy-root]');
+    clone.querySelectorAll("[data-xpm-latex]").forEach((el) => {
+      const copyRoot = el.closest("[data-xpm-copy-root]");
       if (!copyRoot) return;
-      const container = el.closest('.cPGBZb') || copyRoot;
+      const container = el.closest(".cPGBZb") || copyRoot;
       if (!container.parentNode) return;
 
-      const latex = el.getAttribute('data-xpm-latex');
+      const latex = el.getAttribute("data-xpm-latex");
 
       // Determine if it is block math
       let isBlock = false;
       const parent = container.parentNode;
       if (parent) {
-        const parentText = parent.textContent.replace(container.textContent, '').trim();
-        isBlock = parentText === '';
+        const parentText = parent.textContent
+          .replace(container.textContent, "")
+          .trim();
+        isBlock = parentText === "";
       }
 
       const replacementText = isBlock ? `\n\n$$${latex}$$\n\n` : `$${latex}$`;
@@ -202,18 +223,18 @@ export function convertToMarkdown(htmlContent, options = {}) {
     });
 
     // Preprocess code blocks (pre elements) to normalize formatting and language tags
-    clone.querySelectorAll('pre').forEach((pre) => {
+    clone.querySelectorAll("pre").forEach((pre) => {
       // Guard against processing detached/already-replaced pre elements
       if (!clone.contains(pre)) return;
 
-      const innerPre = pre.querySelector('pre');
-      const codeElement = pre.querySelector('code');
+      const innerPre = pre.querySelector("pre");
+      const codeElement = pre.querySelector("code");
 
       if (innerPre && codeElement) {
         // --- ChatGPT-style wrapped code block ---
         // Find the language label from the header (usually next to an SVG icon)
-        let language = '';
-        const svg = pre.querySelector('svg');
+        let language = "";
+        const svg = pre.querySelector("svg");
         if (svg && svg.parentElement) {
           const parent = svg.parentElement;
           svg.remove();
@@ -221,9 +242,9 @@ export function convertToMarkdown(htmlContent, options = {}) {
         }
 
         // Replace all <br> elements inside the code block with actual newlines
-        codeElement.querySelectorAll('br').forEach((br) => {
+        codeElement.querySelectorAll("br").forEach((br) => {
           if (br.parentNode) {
-            const newline = clone.ownerDocument.createTextNode('\n');
+            const newline = clone.ownerDocument.createTextNode("\n");
             br.parentNode.replaceChild(newline, br);
           }
         });
@@ -231,8 +252,8 @@ export function convertToMarkdown(htmlContent, options = {}) {
         const codeText = codeElement.textContent;
 
         // Create a new clean pre and code element structure
-        const newPre = clone.ownerDocument.createElement('pre');
-        const newCode = clone.ownerDocument.createElement('code');
+        const newPre = clone.ownerDocument.createElement("pre");
+        const newCode = clone.ownerDocument.createElement("code");
         if (language) {
           newCode.className = `language-${language}`;
         }
@@ -246,17 +267,19 @@ export function convertToMarkdown(htmlContent, options = {}) {
       } else {
         // --- Standard/SGE-style code block ---
         // Extract language label if available (e.g. from Google Search SGE containers)
-        const code = pre.querySelector('code');
+        const code = pre.querySelector("code");
         const hasLanguage =
           code &&
           Array.from(code.classList).some(
-            (cls) => cls.startsWith('language-') && cls !== 'language-',
+            (cls) => cls.startsWith("language-") && cls !== "language-",
           );
         if (!hasLanguage) {
-          const container = pre.closest('.pHpOfb') || pre.parentElement?.parentElement;
+          const container =
+            pre.closest(".pHpOfb") || pre.parentElement?.parentElement;
           if (container) {
             const langEl =
-              container.querySelector('.vVRw1d') || container.firstElementChild?.firstElementChild;
+              container.querySelector(".vVRw1d") ||
+              container.firstElementChild?.firstElementChild;
             if (langEl && langEl !== pre) {
               const language = langEl.textContent.trim().toLowerCase();
               if (code) {
@@ -268,10 +291,10 @@ export function convertToMarkdown(htmlContent, options = {}) {
         }
 
         // Replace all <br> elements inside the code block with actual newlines
-        const codeEl = pre.querySelector('code') || pre;
-        codeEl.querySelectorAll('br').forEach((br) => {
+        const codeEl = pre.querySelector("code") || pre;
+        codeEl.querySelectorAll("br").forEach((br) => {
           if (br.parentNode) {
-            const newline = clone.ownerDocument.createTextNode('\n');
+            const newline = clone.ownerDocument.createTextNode("\n");
             br.parentNode.replaceChild(newline, br);
           }
         });
@@ -279,33 +302,37 @@ export function convertToMarkdown(htmlContent, options = {}) {
     });
 
     // Remove SGE "Use code with caution" text blocks
-    clone.querySelectorAll('*').forEach((el) => {
-      if (el.textContent.trim() === 'Use code with caution.') {
+    clone.querySelectorAll("*").forEach((el) => {
+      if (el.textContent.trim() === "Use code with caution.") {
         el.remove();
       }
     });
 
     // Pre-process button-wrapped images (e.g., ChatGPT image carousels) before noise button removal
-    clone.querySelectorAll('button').forEach((button) => {
-      const img = button.querySelector('img');
+    clone.querySelectorAll("button").forEach((button) => {
+      const img = button.querySelector("img");
       if (!img) return;
 
-      let caption = 'Image';
-      const ariaLabel = button.getAttribute('aria-label') || '';
-      if (ariaLabel.toLowerCase().includes('open image details for')) {
-        caption = ariaLabel.replace(/^Open image details for\s*/i, '').trim();
-      } else if (img.getAttribute('alt') && !img.getAttribute('alt').startsWith('http')) {
-        caption = img.getAttribute('alt').trim();
+      let caption = "Image";
+      const ariaLabel = button.getAttribute("aria-label") || "";
+      if (ariaLabel.toLowerCase().includes("open image details for")) {
+        caption = ariaLabel.replace(/^Open image details for\s*/i, "").trim();
+      } else if (
+        img.getAttribute("alt") &&
+        !img.getAttribute("alt").startsWith("http")
+      ) {
+        caption = img.getAttribute("alt").trim();
       }
 
-      const alt = img.getAttribute('alt') || '';
-      const src = img.getAttribute('src') || '';
-      const imageUrl = alt.startsWith('http://') || alt.startsWith('https://') ? alt : src;
+      const alt = img.getAttribute("alt") || "";
+      const src = img.getAttribute("src") || "";
+      const imageUrl =
+        alt.startsWith("http://") || alt.startsWith("https://") ? alt : src;
 
       if (imageUrl) {
-        const newImg = clone.ownerDocument.createElement('img');
-        newImg.setAttribute('src', imageUrl);
-        newImg.setAttribute('alt', caption);
+        const newImg = clone.ownerDocument.createElement("img");
+        newImg.setAttribute("src", imageUrl);
+        newImg.setAttribute("alt", caption);
         button.parentNode.replaceChild(newImg, button);
       }
     });
@@ -313,24 +340,24 @@ export function convertToMarkdown(htmlContent, options = {}) {
     // Remove noise elements (buttons, icons, etc.)
     // Note: Exclude elements inside qwen-markdown-code to preserve Monaco editor content
     const noiseSelectors = [
-      'button:not(.qwen-markdown-code *)',
-      '.copy-button',
+      "button:not(.qwen-markdown-code *)",
+      ".copy-button",
       '[role="button"]:not(.qwen-markdown-code *)',
-      '.sr-only',
-      'svg:not(.qwen-markdown-code *)',
-      '.icon:not(.qwen-markdown-code *)',
+      ".sr-only",
+      "svg:not(.qwen-markdown-code *)",
+      ".icon:not(.qwen-markdown-code *)",
       '[aria-hidden="true"]:not(.qwen-markdown-code *)',
       '[id^="shrproxy"]',
       '[id^="fbproxy"]',
-      'div.HvurC',
-      '.DBd2Wb',
-      '.oLpkLe',
-      '.OUQe0e',
-      '.UrecDd',
+      "div.HvurC",
+      ".DBd2Wb",
+      ".oLpkLe",
+      ".OUQe0e",
+      ".UrecDd",
       '[aria-label="Share public link"]',
-      'aside.L9AUvd',
-      'aside.UL0w9b',
-      'div.qacuz',
+      "aside.L9AUvd",
+      "aside.UL0w9b",
+      "div.qacuz",
     ];
 
     noiseSelectors.forEach((selector) => {
@@ -339,19 +366,28 @@ export function convertToMarkdown(htmlContent, options = {}) {
 
     html = clone;
   } else {
-    console.warn('Invalid input to convertToMarkdown:', htmlContent);
-    return '';
+    console.warn("Invalid input to convertToMarkdown:", htmlContent);
+    return "";
   }
 
   try {
     const markdown = turndownService.turndown(html);
     return markdown.trim();
   } catch (error) {
-    console.error('Error converting HTML to markdown:', error);
+    console.error("Error converting HTML to markdown:", error);
     // Fallback to plain text
-    const parser = new DOMParser();
-    const doc = parser.parseFromString(html, 'text/html');
-    return doc.body.innerText || doc.body.textContent || '';
+    if (typeof DOMParser !== "undefined") {
+      const parser = new DOMParser();
+      const doc = parser.parseFromString(html, "text/html");
+      return doc.body?.innerText || doc.body?.textContent || "";
+    }
+    if (typeof html === "string") {
+      return html
+        .replace(/<[^>]+>/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+    }
+    return html?.textContent || "";
   }
 }
 
@@ -364,11 +400,11 @@ export function cleanMarkdown(markdown) {
   return (
     markdown
       // Remove excessive blank lines (max 2 consecutive)
-      .replace(/\n{3,}/g, '\n\n')
+      .replace(/\n{3,}/g, "\n\n")
       // Remove trailing whitespace from lines
-      .replace(/[ \t]+$/gm, '')
+      .replace(/[ \t]+$/gm, "")
       // Normalize horizontal rules
-      .replace(/^(-{3,}|\*{3,}|_{3,})$/gm, '* * *')
+      .replace(/^(-{3,}|\*{3,}|_{3,})$/gm, "* * *")
       .trim()
   );
 }