decant-core 1.0.1 → 1.1.0

package/ai/index.js

@@ -38,3 +38,11 @@ export {
   parsers,
 } from "../detection/detect-platform.js";
 export { AI_CHAT_DOMAINS } from "../detection/domains.js";
+
+// Web & Article Extraction
+export {
+  extractArticle,
+  extractArticleIntelligent,
+  ArticleParser,
+} from "../web/article.js";
+export { scoreContent, cleanTitle } from "../web/scoring.js";

package/package.json

@@ -1,16 +1,20 @@
 {
   "name": "decant-core",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "Shared AI chat platform parsers and detection for Covai browser extensions.",
   "type": "module",
   "main": "ai/index.js",
   "exports": {
     ".": "./ai/index.js",
     "./ai/*": "./ai/*.js",
+    "./web": "./web/index.js",
+    "./web/*": "./web/*.js",
+    "./article": "./web/article.js",
     "./detection/*": "./detection/*.js"
   },
   "files": [
     "ai",
+    "web",
     "detection",
     "lib",
     "utils",
@@ -26,6 +30,7 @@
   },
   "homepage": "https://github.com/Covai-Labs/decant-core#readme",
   "scripts": {
+    "test": "node --test tests/*.test.js",
     "lint": "eslint .",
     "format": "prettier --write .",
     "format:check": "prettier --check ."
@@ -47,7 +52,11 @@
     "access": "public"
   },
   "dependencies": {
-    "turndown": "^7.2.4"
+    "@extractus/article-extractor": "^9.0.0",
+    "@mozilla/readability": "^0.6.0",
+    "defuddle": "^0.19.2",
+    "turndown": "^7.2.4",
+    "turndown-plugin-gfm": "^1.0.2"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
@@ -59,6 +68,7 @@
     "@semantic-release/release-notes-generator": "^14.1.1",
     "eslint": "^10.8.0",
     "globals": "^17.9.0",
+    "linkedom": "^0.18.13",
     "prettier": "^3.9.6",
     "semantic-release": "^25.0.9"
   }

package/utils/html-to-markdown.js

@@ -13,6 +13,7 @@
  * @returns {string} Markdown formatted text
  */
 import TurndownService from "../lib/turndown.js";
+import { strikethrough, taskListItems } from "turndown-plugin-gfm";
 
 /**
  * Convert an HTML element to markdown
@@ -35,6 +36,9 @@ export function convertToMarkdown(htmlContent, options = {}) {
     ...options,
   });
 
+  // Enable GFM plugins (strikethrough and task list checkboxes)
+  turndownService.use([strikethrough, taskListItems]);
+
   // Add custom rules for better conversion
 
   // Preserve line breaks
@@ -110,8 +114,9 @@ export function convertToMarkdown(htmlContent, options = {}) {
 
         // Convert each cell's HTML to markdown using the isolated service
         const cellContents = cells.map((cell) => {
-          let cellMarkdown = cellTurndown.turndown(cell.innerHTML);
+          let cellMarkdown = cellTurndown.turndown(cell);
           // Replace any actual newlines that Turndown generated (e.g. from P tags) with <br>
+
           // as tables cannot have literal newlines in GFM.
           return cellMarkdown.trim().replace(/\n/g, "<br>");
         });
@@ -135,7 +140,8 @@ export function convertToMarkdown(htmlContent, options = {}) {
     html = htmlContent;
   } else if (
     htmlContent &&
-    (htmlContent instanceof HTMLElement ||
+    ((typeof HTMLElement !== "undefined" &&
+      htmlContent instanceof HTMLElement) ||
       htmlContent.nodeType === 1 ||
       htmlContent.nodeType === 9 ||
       typeof htmlContent.querySelectorAll === "function")
@@ -370,9 +376,18 @@ export function convertToMarkdown(htmlContent, options = {}) {
   } catch (error) {
     console.error("Error converting HTML to markdown:", error);
     // Fallback to plain text
-    const parser = new DOMParser();
-    const doc = parser.parseFromString(html, "text/html");
-    return doc.body.innerText || doc.body.textContent || "";
+    if (typeof DOMParser !== "undefined") {
+      const parser = new DOMParser();
+      const doc = parser.parseFromString(html, "text/html");
+      return doc.body?.innerText || doc.body?.textContent || "";
+    }
+    if (typeof html === "string") {
+      return html
+        .replace(/<[^>]+>/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+    }
+    return html?.textContent || "";
   }
 }
 

package/web/article.js

@@ -0,0 +1,286 @@
+import { Readability } from "@mozilla/readability";
+import Defuddle from "defuddle";
+import { extractFromHtml } from "@extractus/article-extractor";
+import { convertToMarkdown } from "../utils/html-to-markdown.js";
+import { scoreContent, cleanTitle } from "./scoring.js";
+import { ChatParser } from "../ai/base.js";
+
+/**
+ * Resolves a Document object from a document or an HTML string.
+ * Works seamlessly in browser DOM and Node/testing environments (via linkedom).
+ *
+ * @param {Document|string} docOrHtml
+ * @param {string} [url]
+ * @returns {Promise<{ doc: Document, rawHtml: string, url: string }>}
+ */
+async function resolveDocument(docOrHtml, url = "") {
+  let doc = null;
+  let rawHtml = "";
+  let finalUrl = url;
+
+  if (typeof docOrHtml === "string") {
+    rawHtml = docOrHtml;
+    if (typeof DOMParser !== "undefined") {
+      doc = new DOMParser().parseFromString(docOrHtml, "text/html");
+    } else {
+      const { parseHTML } = await import("linkedom");
+      const parsed = parseHTML(docOrHtml);
+      doc = parsed.document;
+    }
+  } else if (
+    docOrHtml &&
+    typeof docOrHtml === "object" &&
+    docOrHtml.documentElement
+  ) {
+    doc = docOrHtml;
+    rawHtml = docOrHtml.documentElement.innerHTML || "";
+    if (
+      !finalUrl &&
+      docOrHtml.location &&
+      docOrHtml.location.href &&
+      docOrHtml.location.href !== "about:blank"
+    ) {
+      finalUrl = docOrHtml.location.href;
+    }
+  } else if (typeof document !== "undefined") {
+    doc = document;
+    rawHtml = document.documentElement.innerHTML || "";
+    if (!finalUrl && typeof window !== "undefined" && window.location) {
+      finalUrl = window.location.href;
+    }
+  }
+
+  if (!finalUrl) {
+    finalUrl = "https://localhost/article";
+  }
+
+  return { doc, rawHtml, url: finalUrl };
+}
+
+/**
+ * Extracts content from any web page running Readability, Defuddle,
+ * and Article-Extractor in parallel, with intelligent scoring and arbitration.
+ *
+ * @param {Document|string} [docOrHtml] - Document or HTML string
+ * @param {Object} [options] - Options { url?: string, turndownOptions?: object }
+ * @returns {Promise<{
+ *   title: string,
+ *   author: string,
+ *   published: string,
+ *   siteName: string,
+ *   description: string,
+ *   image: string,
+ *   url: string,
+ *   content: string,
+ *   htmlContent: string,
+ *   markdown: string,
+ *   engine: 'readability' | 'defuddle' | 'raw',
+ *   wordCount: number
+ * }>}
+ */
+export async function extractArticleIntelligent(docOrHtml, options = {}) {
+  const { doc, rawHtml, url } = await resolveDocument(docOrHtml, options.url);
+
+  if (!doc) {
+    throw new Error(
+      "Unable to resolve a valid Document object for extraction.",
+    );
+  }
+
+  // 1. Run all 3 extractors concurrently
+  const [readabilitySettled, defuddleSettled, extractusSettled] =
+    await Promise.allSettled([
+      // Mozilla Readability
+      Promise.resolve().then(() => {
+        try {
+          const clone = doc.cloneNode(true);
+          const reader = new Readability(clone);
+          return reader.parse();
+        } catch {
+          return null;
+        }
+      }),
+      // Defuddle
+      Promise.resolve().then(() => {
+        try {
+          const defuddle = new Defuddle(doc, { url });
+          return defuddle.parse();
+        } catch {
+          return null;
+        }
+      }),
+      // Article-Extractor (Metadata & JSON-LD)
+      Promise.resolve().then(async () => {
+        try {
+          if (!rawHtml) return null;
+          return await extractFromHtml(rawHtml, url);
+        } catch {
+          return null;
+        }
+      }),
+    ]);
+
+  const rData =
+    readabilitySettled.status === "fulfilled" ? readabilitySettled.value : null;
+  const dData =
+    defuddleSettled.status === "fulfilled" ? defuddleSettled.value : null;
+  const mData =
+    extractusSettled.status === "fulfilled" ? extractusSettled.value : null;
+
+  // 2. Best-of-breed Metadata Resolution
+  const ogSiteName = doc
+    .querySelector?.('meta[property="og:site_name"], meta[name="og:site_name"]')
+    ?.getAttribute?.("content");
+  const metaAuthor = doc
+    .querySelector?.('meta[name="author"], meta[property="article:author"]')
+    ?.getAttribute?.("content");
+  const siteName =
+    ogSiteName || mData?.source || rData?.siteName || dData?.site || "";
+
+  const articleH1 = doc
+    .querySelector?.("article h1, main h1")
+    ?.textContent?.trim();
+  const rawTitle =
+    articleH1 && articleH1.length > 3
+      ? articleH1
+      : mData?.title || rData?.title || dData?.title || doc.title || "Untitled";
+  const title = cleanTitle(rawTitle, siteName);
+
+  const author =
+    metaAuthor || mData?.author || rData?.byline || dData?.author || "";
+  const published =
+    mData?.published || rData?.publishedTime || dData?.published || "";
+  const description =
+    mData?.description || rData?.excerpt || dData?.description || "";
+  const image = mData?.image || dData?.image || "";
+
+  // 3. Content Quality Scoring & Arbitration
+  const rHtml = rData?.content || "";
+  const dHtml = dData?.content || "";
+
+  const rScore = scoreContent(rHtml);
+  const dScore = scoreContent(dHtml);
+
+  let bestHtml;
+  let winningEngine;
+
+  if (rScore === 0 && dScore === 0) {
+    // Fallback: take main/article or body
+    const mainEl = doc.querySelector(
+      "main, article, #content, .content, .post",
+    );
+    bestHtml = mainEl
+      ? mainEl.innerHTML
+      : doc.body
+        ? doc.body.innerHTML
+        : rawHtml;
+    winningEngine = "raw";
+  } else if (rScore >= dScore) {
+    bestHtml = rHtml || dHtml;
+    winningEngine = "readability";
+  } else {
+    bestHtml = dHtml || rHtml;
+    winningEngine = "defuddle";
+  }
+
+  // 4. Convert to Markdown using decant-core convertToMarkdown
+  let markdownBody;
+  try {
+    const container = doc.createElement("div");
+    container.innerHTML = bestHtml;
+    markdownBody = convertToMarkdown(container, options.turndownOptions || {});
+  } catch {
+    markdownBody = convertToMarkdown(bestHtml, options.turndownOptions || {});
+  }
+  const cleanMarkdown = markdownBody ? markdownBody.trim() : "";
+  const fullMarkdown = title ? `# ${title}\n\n${cleanMarkdown}` : cleanMarkdown;
+
+  // Approximate word count
+  const rawText = bestHtml
+    .replace(/<[^>]+>/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+  const wordCount = rawText ? rawText.split(" ").length : 0;
+
+  return {
+    title,
+    author,
+    published,
+    siteName,
+    description,
+    image,
+    url,
+    content: cleanMarkdown,
+    htmlContent: bestHtml,
+    markdown: fullMarkdown,
+    engine: winningEngine,
+    wordCount,
+  };
+}
+
+/**
+ * Standard alias for extractArticleIntelligent.
+ */
+export const extractArticle = extractArticleIntelligent;
+
+/**
+ * ArticleParser conforms to the decant-core Parser interface,
+ * allowing extensions to treat generic web pages uniformly alongside AI chat parsers.
+ */
+export class ArticleParser extends ChatParser {
+  name = "WebArticle";
+
+  getPlatformName() {
+    return "Web Article";
+  }
+
+  /**
+   * Always available for standard HTTP/HTTPS URLs.
+   * @param {string} url
+   * @returns {boolean}
+   */
+  isAvailable(url) {
+    if (!url || typeof url !== "string") return false;
+    return (
+      url.startsWith("http://") ||
+      url.startsWith("https://") ||
+      url.startsWith("file://")
+    );
+  }
+
+  /**
+   * Parses the current document into a standardized format.
+   * @param {Object} [options]
+   * @returns {Promise<{
+   *   title: string,
+   *   messages: Array<{ role: string, content: string }>,
+   *   metadata: Record<string, string>,
+   *   url: string
+   * }>}
+   */
+  async parse(options = {}) {
+    const article = await extractArticleIntelligent(
+      typeof document !== "undefined" ? document : null,
+      options,
+    );
+
+    return {
+      title: article.title,
+      messages: [
+        {
+          role: "Assistant",
+          content: article.content,
+        },
+      ],
+      metadata: {
+        Source: article.siteName || "Web Article",
+        Author: article.author || "",
+        Date: article.published || new Date().toISOString().split("T")[0],
+        Description: article.description || "",
+        Engine: article.engine,
+      },
+      url: article.url,
+      rawArticle: article,
+    };
+  }
+}

package/web/index.js

@@ -0,0 +1,6 @@
+export {
+  extractArticle,
+  extractArticleIntelligent,
+  ArticleParser,
+} from "./article.js";
+export { scoreContent, cleanTitle } from "./scoring.js";

package/web/scoring.js

@@ -0,0 +1,76 @@
+/**
+ * Evaluates the quality and richness of extracted HTML content.
+ * Higher scores indicate cleaner article body with good formatting (headings, code, tables)
+ * and low link/navigation noise.
+ *
+ * @param {string} html
+ * @returns {number} Quality score >= 0
+ */
+export function scoreContent(html) {
+  if (!html || typeof html !== "string") return 0;
+
+  // Strip HTML tags to get raw visible text
+  const rawText = html.replace(/<[^>]+>/g, "").trim();
+  const textLength = rawText.length;
+  if (textLength < 30) return 0;
+
+  // 1. Base score scaled logarithmically by text volume
+  let score = Math.log10(textLength) * 25;
+
+  // 2. Bonus for structural elements (paragraphs, headings)
+  const paragraphs = (html.match(/<p[\s>]/gi) || []).length;
+  const headings = (html.match(/<h[1-6][\s>]/gi) || []).length;
+  score += paragraphs * 2 + headings * 4;
+
+  // 3. High bonus for code blocks & tables (technical articles/documentation)
+  const codeBlocks = (html.match(/<pre[\s>]/gi) || []).length;
+  const tables = (html.match(/<table[\s>]/gi) || []).length;
+  const listItems = (html.match(/<li[\s>]/gi) || []).length;
+  score += codeBlocks * 20 + tables * 15 + Math.min(listItems, 20) * 1.5;
+
+  // 4. Heavy penalty for anchor density (detects sidebar link farms / navigation headers)
+  const linkMatches = html.match(/<a[^>]*>(.*?)<\/a>/gi) || [];
+  const linkTextLength = linkMatches.reduce((acc, tag) => {
+    return acc + tag.replace(/<[^>]+>/g, "").length;
+  }, 0);
+
+  const anchorDensity = textLength > 0 ? linkTextLength / textLength : 0;
+  if (anchorDensity > 0.3) {
+    score -= (anchorDensity - 0.3) * 150;
+  }
+
+  return Math.max(0, Math.round(score * 100) / 100);
+}
+
+/**
+ * Normalizes an article title by trimming brand/site suffixes (e.g. "Article Title | Medium" -> "Article Title").
+ *
+ * @param {string} title
+ * @param {string} [siteName]
+ * @returns {string} Cleaned title
+ */
+export function cleanTitle(title, siteName = "") {
+  if (!title || typeof title !== "string") return "Untitled";
+  let cleaned = title.trim();
+
+  // Strip known siteName if provided at end
+  if (siteName && siteName.trim()) {
+    const escapedSite = siteName.trim().replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const siteRegex = new RegExp(
+      `\\s*[\\|\\-–—:•·]\\s*${escapedSite}\\s*$`,
+      "i",
+    );
+    cleaned = cleaned.replace(siteRegex, "");
+  }
+
+  // Strip general brand suffix if short (e.g., " | BrandName", " - The Verge")
+  cleaned = cleaned
+    .replace(/\s*[|\-–—•·]\s*[^|\-–—•·]+$/, (match) => {
+      // If the trailing segment is relatively short (brand name), remove it
+      const segment = match.replace(/^[\s|\-–—•·]+/, "").trim();
+      return segment.length > 0 && segment.length <= 35 ? "" : match;
+    })
+    .trim();
+
+  return cleaned || title.trim() || "Untitled";
+}