decant-core 1.2.3 → 1.2.5

package/ai/chatgpt.js

@@ -1,5 +1,6 @@
 import { ChatParser } from "./base.js";
 import { convertToMarkdown } from "../utils/html-to-markdown.js";
+import { normalizeLatexMath } from "../utils/latex-math.js";
 import {
   collectMountedTurnMessages,
   findChatGPTScrollRoot,
@@ -534,6 +535,8 @@ function cleanMarkdownFromApi(text, citeMap, imageGroupMap) {
     },
   );
 
+  text = normalizeLatexMath(text);
+
   return text;
 }
 

package/ai/google_search_ai.js

@@ -1,6 +1,70 @@
 import { ChatParser } from "./base.js";
 import { convertToMarkdown } from "../utils/html-to-markdown.js";
 
+export function sanitizeResponseContainer(container) {
+  if (!container) return container;
+  const clone =
+    typeof container.cloneNode === "function"
+      ? container.cloneNode(true)
+      : container;
+
+  // 1. Remove script, style, and noscript elements to prevent inline script leakage
+  const unwanted = clone.querySelectorAll
+    ? clone.querySelectorAll("script, style, noscript")
+    : [];
+  unwanted.forEach((el) => el.remove());
+
+  // 2. Remove base64 inline images to prevent megabyte-scale text walls in markdown exports
+  const images = clone.querySelectorAll ? clone.querySelectorAll("img") : [];
+  images.forEach((img) => {
+    const src = img.getAttribute("src") || "";
+    if (src.startsWith("data:image/")) {
+      img.remove();
+    }
+  });
+
+  // 3. Unwrap Google tracking redirects (/goto?url=..., /url?q=...) to direct URLs
+  const links = clone.querySelectorAll ? clone.querySelectorAll("a[href]") : [];
+  links.forEach((a) => {
+    const href = a.getAttribute("href") || "";
+    if (
+      href.startsWith("/goto?") ||
+      href.startsWith("/url?") ||
+      href.includes("google.com/url?") ||
+      href.includes("google.com/goto?")
+    ) {
+      try {
+        const parsed = new URL(href, "https://www.google.com");
+        const target =
+          parsed.searchParams.get("url") || parsed.searchParams.get("q");
+        if (
+          target &&
+          (target.startsWith("http://") || target.startsWith("https://"))
+        ) {
+          a.setAttribute("href", target);
+        }
+      } catch {
+        // Keep original href if URL parsing fails
+      }
+    }
+
+    // 4. Remove empty link shells left behind by stripped images/tracking icons
+    if (!a.textContent.trim() && !a.querySelector("img, svg")) {
+      a.remove();
+    }
+  });
+
+  return clone;
+}
+
+export function cleanMarkdownSpacing(markdown) {
+  if (!markdown) return "";
+  return markdown
+    .replace(/[ \t]+$/gm, "")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
 export class GoogleSearchAIParser extends ChatParser {
   name = "Google Search AI";
   isAvailable(url) {
@@ -94,9 +158,10 @@ export class GoogleSearchAIParser extends ChatParser {
     const minLength = Math.min(queries.length, responseContainers.length);
     for (let i = 0; i < minLength; i++) {
       messages.push({ role: "User", content: queries[i].trim() });
-      const text = convertToMarkdown(responseContainers[i]);
-      if (text.trim()) {
-        messages.push({ role: "Model", content: text.trim() });
+      const cleanContainer = sanitizeResponseContainer(responseContainers[i]);
+      const text = cleanMarkdownSpacing(convertToMarkdown(cleanContainer));
+      if (text) {
+        messages.push({ role: "Model", content: text });
       }
     }
 

package/ai/index.js

@@ -30,6 +30,7 @@ export { ChubParser } from "./chub.js";
 
 // Utilities
 export { convertToMarkdown, cleanMarkdown } from "../utils/html-to-markdown.js";
+export { normalizeLatexMath, cleanLatexMath } from "../utils/latex-math.js";
 
 // Detection
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "decant-core",
-  "version": "1.2.3",
+  "version": "1.2.5",
   "description": "Shared AI chat platform parsers and detection for Covai browser extensions.",
   "type": "module",
   "main": "ai/index.js",

package/utils/latex-math.js

@@ -0,0 +1,81 @@
+/**
+ * Clean up LaTeX math formulas by unescaping common markdown parser artifacts.
+ * @param {string} latex - Raw LaTeX math string
+ * @returns {string} Cleaned LaTeX math
+ */
+export function cleanLatexMath(latex) {
+  if (!latex || typeof latex !== "string") return "";
+  return latex
+    .replace(/\\\\([a-zA-Z]+)/g, "\\$1")
+    .replace(/\\([_\][*])/g, "$1");
+}
+
+/**
+ * Standardize LaTeX math delimiters in markdown text.
+ * Converts LaTeX bracket delimiters (\[...\] and \(...\)) to standard markdown
+ * math delimiters ($$...$$ and $...$) while protecting code blocks and inline code.
+ *
+ * @param {string} text - Input text/markdown
+ * @returns {string} Text with normalized LaTeX math delimiters
+ */
+export function normalizeLatexMath(text) {
+  if (!text || typeof text !== "string") return "";
+
+  const placeholders = [];
+  let tokenCounter = 0;
+
+  // 1. Protect fenced code blocks (``` ... ``` or ~~~ ... ~~~)
+  let processed = text.replace(/(```[\s\S]*?```|~~~[\s\S]*?~~~)/g, (match) => {
+    const id = `@@MATH_CODE_BLOCK_${tokenCounter++}@@`;
+    placeholders.push({ id, content: match });
+    return id;
+  });
+
+  // 2. Protect inline code (`...`)
+  processed = processed.replace(/`([^`\n]+?)`/g, (match) => {
+    const id = `@@MATH_INLINE_CODE_${tokenCounter++}@@`;
+    placeholders.push({ id, content: match });
+    return id;
+  });
+
+  // 3. Protect existing display math ($$ ... $$) and clean any escaped LaTeX syntax
+  processed = processed.replace(/\$\$([\s\S]*?)\$\$/g, (match, math) => {
+    const id = `@@MATH_DISPLAY_${tokenCounter++}@@`;
+    placeholders.push({ id, content: `$$${cleanLatexMath(math)}$$` });
+    return id;
+  });
+
+  // 4. Protect existing inline math ($ ... $) and clean any escaped LaTeX syntax
+  processed = processed.replace(/\$([^$\n]+?)\$/g, (match, math) => {
+    const id = `@@MATH_INLINE_${tokenCounter++}@@`;
+    placeholders.push({ id, content: `$${cleanLatexMath(math)}$` });
+    return id;
+  });
+
+  // 5. Convert display math: \[ ... \] or \\[ ... \\]
+  processed = processed.replace(
+    /(?:\\{1,2}\[)([\s\S]+?)(?:\\{1,2}\])/g,
+    (match, math) => {
+      return `$$${cleanLatexMath(math).trim()}$$`;
+    },
+  );
+
+  // 6. Convert inline math: \( ... \) or \\( ... \\)
+  processed = processed.replace(
+    /(?:\\{1,2}\()([\s\S]+?)(?:\\{1,2}\))/g,
+    (match, math) => {
+      return `$${cleanLatexMath(math).trim()}$`;
+    },
+  );
+
+  // 7. Collapse excessive blank lines outside protected code
+  processed = processed.replace(/\n{3,}/g, "\n\n");
+
+  // 8. Restore protected items in reverse order
+  for (let i = placeholders.length - 1; i >= 0; i--) {
+    const { id, content } = placeholders[i];
+    processed = processed.replace(id, () => content);
+  }
+
+  return processed;
+}