@jenslys/curldown 1.0.4 → 1.0.6

package/dist/constants.js

@@ -1,4 +1,4 @@
-export const VERSION = "1.0.1";
+export const VERSION = "1.0.5";
 export const DEFAULT_STATIC_TIMEOUT_MS = 15_000;
 export const DEFAULT_DYNAMIC_TIMEOUT_MS = 30_000;
 export const DEFAULT_REMOVE_SELECTORS = [
@@ -8,5 +8,13 @@ export const DEFAULT_REMOVE_SELECTORS = [
     "template",
     "svg",
     "canvas",
-    "iframe"
+    "iframe",
+    "wbr",
+    "button",
+    "input",
+    "select",
+    "textarea",
+    "[role='button']",
+    "[role='toolbar']",
+    "[role='separator']"
 ];

package/dist/transform.js

@@ -1,6 +1,6 @@
 import { Readability } from "@mozilla/readability";
 import { load } from "cheerio";
-import { JSDOM } from "jsdom";
+import { DOMParser } from "linkedom";
 import { createRequire } from "node:module";
 import TurndownService from "turndown";
 import { DEFAULT_REMOVE_SELECTORS } from "./constants.js";
@@ -14,15 +14,50 @@ const turndown = new TurndownService({
     emDelimiter: "_"
 });
 turndown.use(turndownPluginGfm.gfm);
+const tableCellTurndown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+    bulletListMarker: "-",
+    emDelimiter: "_"
+});
+tableCellTurndown.use(turndownPluginGfm.gfm);
 const FALLBACK_BASE_URL = "https://curldown.local/";
 const PRIMARY_CONTENT_SELECTOR = "main, article, [role='main']";
 const MIN_PRIMARY_CONTENT_TEXT_LENGTH = 200;
+const COMPLEX_TABLE_CELL_SELECTOR = "ul, ol, blockquote, pre, h1, h2, h3, h4, h5, h6, hr";
 function getNormalizedTextLength(value) {
     return value?.replace(/\s+/g, " ").trim().length ?? 0;
 }
-function cleanupFragmentHtml(html) {
+function resolveUrl(value, baseUrl) {
+    if (!value) {
+        return undefined;
+    }
+    if (!baseUrl) {
+        return value;
+    }
+    try {
+        return new URL(value, baseUrl).toString();
+    }
+    catch {
+        return value;
+    }
+}
+function cleanupFragmentHtml(html, baseUrl) {
     const $ = load(html);
     $(DEFAULT_REMOVE_SELECTORS.join(",")).remove();
+    normalizeComplexTables($);
+    $("img").each((_, element) => {
+        const src = resolveUrl($(element).attr("src"), baseUrl);
+        if (src) {
+            $(element).attr("src", src);
+        }
+    });
+    $("a").each((_, element) => {
+        const href = resolveUrl($(element).attr("href"), baseUrl);
+        if (href) {
+            $(element).attr("href", href);
+        }
+    });
     $("img").each((_, element) => {
         const alt = $(element).attr("alt")?.trim() ?? "";
         if (!alt) {
@@ -42,6 +77,22 @@ function cleanupFragmentHtml(html) {
     });
     return $.root().html() ?? "";
 }
+function normalizeComplexTables($) {
+    $("table")
+        .find(`th:has(${COMPLEX_TABLE_CELL_SELECTOR}), td:has(${COMPLEX_TABLE_CELL_SELECTOR})`)
+        .each((_, cell) => {
+        const markdown = normalizeTableCellMarkdown(tableCellTurndown.turndown($(cell).html() ?? ""));
+        $(cell).empty().text(markdown);
+    });
+}
+function normalizeTableCellMarkdown(markdown) {
+    return markdown
+        .replace(/\r\n/g, "\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .replace(/^#{1,6}\s+/gm, "")
+        .replace(/^\s{0,3}>\s?/gm, "")
+        .trim();
+}
 function extractBodyHtml(document) {
     return document.body?.innerHTML ?? document.documentElement?.innerHTML ?? "";
 }
@@ -66,11 +117,11 @@ function selectReadabilityHtml(document) {
     }
     return article.content ?? undefined;
 }
-function toMarkdownCandidate(html) {
+function toMarkdownCandidate(html, baseUrl) {
     if (!html) {
         return undefined;
     }
-    const cleanedHtml = cleanupFragmentHtml(html);
+    const cleanedHtml = cleanupFragmentHtml(html, baseUrl);
     if (cleanedHtml.trim().length === 0) {
         return undefined;
     }
@@ -86,6 +137,10 @@ function getFirstMeaningfulMarkdownLine(markdown) {
 function startsWithPrimaryHeading(markdown) {
     return /^#\s+\S/.test(getFirstMeaningfulMarkdownLine(markdown) ?? "");
 }
+function parseDocument(html) {
+    const normalizedHtml = /<html[\s>]/i.test(html) ? html : `<!doctype html><html>${html}</html>`;
+    return new DOMParser().parseFromString(normalizedHtml, "text/html");
+}
 /**
  * Convert fetched HTML into markdown.
  * The function prefers semantic primary-content containers, falls back to
@@ -93,13 +148,11 @@ function startsWithPrimaryHeading(markdown) {
  * no stronger content signal exists.
  */
 export function transformHtmlToMarkdown(input) {
-    const dom = new JSDOM(input.html, {
-        url: input.url ?? FALLBACK_BASE_URL
-    });
-    const { document } = dom.window;
-    const semanticMarkdown = toMarkdownCandidate(selectSemanticPrimaryHtml(document));
-    const readabilityMarkdown = toMarkdownCandidate(selectReadabilityHtml(new JSDOM(input.html, { url: input.url ?? FALLBACK_BASE_URL }).window.document));
-    const fallbackMarkdown = toMarkdownCandidate(extractBodyHtml(document));
+    const baseUrl = input.url ?? FALLBACK_BASE_URL;
+    const document = parseDocument(input.html);
+    const semanticMarkdown = toMarkdownCandidate(selectSemanticPrimaryHtml(document), baseUrl);
+    const readabilityMarkdown = toMarkdownCandidate(selectReadabilityHtml(parseDocument(input.html)), baseUrl);
+    const fallbackMarkdown = toMarkdownCandidate(extractBodyHtml(document), baseUrl);
     const markdown = semanticMarkdown && startsWithPrimaryHeading(semanticMarkdown) && !startsWithPrimaryHeading(readabilityMarkdown ?? "")
         ? semanticMarkdown
         : readabilityMarkdown ?? semanticMarkdown ?? fallbackMarkdown;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jenslys/curldown",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "description": "Fetch URL content and convert it to markdown.",
   "repository": {
     "type": "git",
@@ -33,12 +33,11 @@
     "@mozilla/readability": "^0.6.0",
     "cheerio": "^1.2.0",
     "commander": "^14.0.3",
-    "jsdom": "^29.0.0",
+    "linkedom": "^0.18.12",
     "playwright": "^1.58.2",
     "turndown": "^7.2.2"
   },
   "devDependencies": {
-    "@types/jsdom": "^28.0.1",
     "@types/node": "^25.3.3",
     "@types/turndown": "^5.0.6",
     "typescript": "^5.9.3",