@jenslys/curldown 1.0.2 → 1.0.4

package/README.md

@@ -7,6 +7,7 @@ Fetch a webpage and return clean Markdown for AI workflows.
 - Static mode: `fetch` HTML -> Cheerio cleanup -> Turndown markdown.
 - Dynamic mode: headless Chromium (Playwright) -> HTML -> markdown.
 - `--auto` tries static first and falls back to dynamic when static output is thin.
+- Direct markdown responses are passed through (including `.md` URLs served as `text/plain`).
 - `--format json` emits markdown plus metadata for agent pipelines.
 
 ## Install
@@ -82,7 +83,8 @@ Paste this into your `AGENTS.md` if you want agents to always use `curldown` for
 ```md
 ## Website Content Retrieval
 
-- Always use `curldown` to fetch web pages for agent workflows.
+- Use `curldown` for website/article page retrieval in agent workflows.
+- Do not use `curldown` for raw code files or repository file blobs (for those, fetch the file directly).
 - Default command: `curldown <url>`.
 - Prefer `curldown <url> --auto` when page rendering might be uncertain.
 - Use `curldown <url> --format json` when downstream steps need structured metadata.

package/dist/cli-args.js

@@ -0,0 +1,108 @@
+import { Command } from "commander";
+import { DEFAULT_DYNAMIC_TIMEOUT_MS, DEFAULT_STATIC_TIMEOUT_MS, VERSION } from "./constants.js";
+import { InputError } from "./errors.js";
+function collectRepeatable(value, previous = []) {
+    return [...previous, value];
+}
+export function buildProgram() {
+    return new Command()
+        .name("curldown")
+        .description("Fetch URL content and convert it to markdown.")
+        .version(VERSION)
+        .argument("<url>", "The URL to fetch")
+        .option("--dynamic", "Use headless Chromium (Playwright) to render the page")
+        .option("--auto", "Try static first and fallback to dynamic when static output is thin")
+        .option("--format <type>", "Output format: markdown|json", "markdown")
+        .option("-o, --output <path>", "Write output to a file instead of stdout")
+        .option("--timeout-ms <number>", "Timeout in milliseconds")
+        .option("--header <key:value>", "Set custom request header", collectRepeatable, [])
+        .showHelpAfterError()
+        .exitOverride();
+}
+function parseHeaders(rawHeaders) {
+    const headers = {};
+    for (const rawHeader of rawHeaders) {
+        const separatorIndex = rawHeader.indexOf(":");
+        if (separatorIndex <= 0 || separatorIndex === rawHeader.length - 1) {
+            throw new InputError(`Invalid --header value \"${rawHeader}\". Use key:value format.`);
+        }
+        const key = rawHeader.slice(0, separatorIndex).trim();
+        const value = rawHeader.slice(separatorIndex + 1).trim();
+        if (!key || !value) {
+            throw new InputError(`Invalid --header value \"${rawHeader}\". Header key and value are required.`);
+        }
+        headers[key] = value;
+    }
+    return headers;
+}
+function parseFormat(rawFormat) {
+    if (rawFormat === "markdown" || rawFormat === "json") {
+        return rawFormat;
+    }
+    throw new InputError(`Invalid --format value \"${rawFormat}\". Use \"markdown\" or \"json\".`);
+}
+function parseTimeouts(rawTimeout, dynamic, auto) {
+    if (rawTimeout === undefined) {
+        if (dynamic) {
+            return {
+                timeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS,
+                dynamicTimeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS
+            };
+        }
+        if (auto) {
+            return {
+                timeoutMs: DEFAULT_STATIC_TIMEOUT_MS,
+                dynamicTimeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS
+            };
+        }
+        return {
+            timeoutMs: DEFAULT_STATIC_TIMEOUT_MS,
+            dynamicTimeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS
+        };
+    }
+    const parsed = Number.parseInt(rawTimeout, 10);
+    if (!Number.isInteger(parsed) || parsed <= 0) {
+        throw new InputError(`Invalid --timeout-ms value \"${rawTimeout}\". Must be a positive integer.`);
+    }
+    return {
+        timeoutMs: parsed,
+        dynamicTimeoutMs: parsed
+    };
+}
+/**
+ * Validate and normalize parsed CLI arguments into the canonical runtime shape.
+ * Fails fast with {@link InputError} on malformed input.
+ */
+export function normalizeArgs(urlInput, options) {
+    if (!urlInput) {
+        throw new InputError("A URL argument is required.");
+    }
+    let parsedUrl;
+    try {
+        parsedUrl = new URL(urlInput);
+    }
+    catch (error) {
+        throw new InputError(`Invalid URL \"${urlInput}\".`, {
+            cause: error instanceof Error ? error : undefined
+        });
+    }
+    if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
+        throw new InputError(`Unsupported URL protocol \"${parsedUrl.protocol}\". Only http:// and https:// are supported.`);
+    }
+    const dynamic = options.dynamic ?? false;
+    const auto = options.auto ?? false;
+    if (dynamic && auto) {
+        throw new InputError("--dynamic and --auto cannot be used together.");
+    }
+    const timeouts = parseTimeouts(options.timeoutMs, dynamic, auto);
+    return {
+        url: parsedUrl.toString(),
+        auto,
+        dynamic,
+        format: parseFormat(options.format ?? "markdown"),
+        outputPath: options.output,
+        timeoutMs: timeouts.timeoutMs,
+        dynamicTimeoutMs: timeouts.dynamicTimeoutMs,
+        headers: parseHeaders(options.header ?? [])
+    };
+}

package/dist/cli-content.js

@@ -0,0 +1,119 @@
+import { createHash } from "node:crypto";
+import { ConversionError } from "./errors.js";
+import { extractHtmlTitle } from "./transform.js";
+const MARKDOWN_CONTENT_TYPES = new Set([
+    "text/markdown",
+    "text/x-markdown",
+    "application/markdown",
+    "application/x-markdown"
+]);
+const PLAINTEXT_CONTENT_TYPE = "text/plain";
+const MARKDOWN_FILE_EXTENSIONS = [
+    ".md",
+    ".markdown",
+    ".mdown",
+    ".mkd",
+    ".mkdn",
+    ".mdtxt",
+    ".mdx"
+];
+function normalizeMarkdown(markdown) {
+    const trimmed = markdown.trim();
+    if (!trimmed) {
+        throw new ConversionError("Content was fetched but markdown output is empty.");
+    }
+    return `${trimmed}\n`;
+}
+function inferTitleFromMarkdown(markdown) {
+    const firstHeading = markdown.match(/^#\s+(.+)$/m)?.[1]?.trim();
+    return firstHeading || undefined;
+}
+function isMarkdownContentType(contentType) {
+    if (!contentType) {
+        return false;
+    }
+    const normalized = contentType.toLowerCase().split(";")[0]?.trim() ?? "";
+    return MARKDOWN_CONTENT_TYPES.has(normalized);
+}
+function isPlainTextContentType(contentType) {
+    if (!contentType) {
+        return false;
+    }
+    const normalized = contentType.toLowerCase().split(";")[0]?.trim() ?? "";
+    return normalized === PLAINTEXT_CONTENT_TYPE;
+}
+function hasMarkdownFileExtension(urlValue) {
+    let pathname;
+    try {
+        pathname = new URL(urlValue).pathname;
+    }
+    catch {
+        return false;
+    }
+    const normalizedPath = pathname.toLowerCase();
+    return MARKDOWN_FILE_EXTENSIONS.some((extension) => normalizedPath.endsWith(extension));
+}
+function shouldTreatAsMarkdownPassthrough(result) {
+    if (isMarkdownContentType(result.contentType)) {
+        return true;
+    }
+    return isPlainTextContentType(result.contentType) && hasMarkdownFileExtension(result.finalUrl);
+}
+function countWords(value) {
+    const trimmed = value.trim();
+    if (!trimmed) {
+        return 0;
+    }
+    return trimmed.split(/\s+/).length;
+}
+export function shouldAutoFallback(markdown) {
+    const trimmed = markdown.trim();
+    if (!trimmed) {
+        return true;
+    }
+    const lower = trimmed.toLowerCase();
+    if (/enable javascript|javascript is required|checking your browser|just a moment|please wait/.test(lower)) {
+        return true;
+    }
+    const nonEmptyLines = trimmed.split(/\r?\n/).filter((line) => line.trim().length > 0).length;
+    return countWords(trimmed) < 30 && nonEmptyLines <= 2;
+}
+export function prepareContentFromFetchResult(result, deps) {
+    if (shouldTreatAsMarkdownPassthrough(result)) {
+        const markdown = normalizeMarkdown(result.body);
+        return {
+            markdown,
+            title: inferTitleFromMarkdown(markdown),
+            source: result,
+            passthrough: true
+        };
+    }
+    const markdown = deps.transformHtmlToMarkdown({
+        html: result.body,
+        url: result.finalUrl
+    });
+    return {
+        markdown,
+        title: extractHtmlTitle(result.body),
+        source: result,
+        passthrough: false
+    };
+}
+export function formatOutput(args, content, usedDynamic) {
+    if (args.format === "markdown") {
+        return content.markdown;
+    }
+    const payload = {
+        url: args.url,
+        final_url: content.source.finalUrl,
+        title: content.title ?? null,
+        markdown: content.markdown,
+        content_type: content.source.contentType ?? null,
+        status: content.source.status,
+        fetched_at: new Date().toISOString(),
+        word_count: countWords(content.markdown),
+        sha256: createHash("sha256").update(content.markdown).digest("hex"),
+        used_dynamic: usedDynamic
+    };
+    return `${JSON.stringify(payload, null, 2)}\n`;
+}

package/dist/cli-main-module.js

@@ -0,0 +1,22 @@
+import { realpathSync } from "node:fs";
+import { fileURLToPath, pathToFileURL } from "node:url";
+function resolvePathStrict(pathInput) {
+    return realpathSync(pathInput);
+}
+/**
+ * Determine whether `argvPath` points at the current module entrypoint.
+ * Resolves symlinks for both paths so global installs that expose a symlinked bin still execute.
+ */
+export function isMainModuleFor(moduleUrl, argvPath = process.argv[1]) {
+    if (argvPath === undefined) {
+        return false;
+    }
+    try {
+        const invokedPath = resolvePathStrict(argvPath);
+        const modulePath = resolvePathStrict(fileURLToPath(moduleUrl));
+        return invokedPath === modulePath;
+    }
+    catch {
+        return pathToFileURL(argvPath).href === moduleUrl;
+    }
+}

package/dist/cli.js

@@ -1,20 +1,13 @@
 #!/usr/bin/env node
-import { createHash } from "node:crypto";
-import { realpathSync } from "node:fs";
-import { Command, CommanderError } from "commander";
-import { fileURLToPath, pathToFileURL } from "node:url";
-import { DEFAULT_DYNAMIC_TIMEOUT_MS, DEFAULT_STATIC_TIMEOUT_MS, VERSION } from "./constants.js";
-import { asCurldownError, ConversionError, InputError } from "./errors.js";
+import { CommanderError } from "commander";
+import { buildProgram, normalizeArgs } from "./cli-args.js";
+import { formatOutput, prepareContentFromFetchResult, shouldAutoFallback } from "./cli-content.js";
+import { isMainModuleFor } from "./cli-main-module.js";
+import { asCurldownError } from "./errors.js";
 import { fetchDynamicHtml } from "./fetch-dynamic.js";
 import { fetchStaticHtml } from "./fetch-static.js";
 import { writeOutput } from "./output.js";
-import { extractHtmlTitle, transformHtmlToMarkdown } from "./transform.js";
-const MARKDOWN_CONTENT_TYPES = new Set([
-    "text/markdown",
-    "text/x-markdown",
-    "application/markdown",
-    "application/x-markdown"
-]);
+import { transformHtmlToMarkdown } from "./transform.js";
 const defaultDependencies = {
     fetchStatic: fetchStaticHtml,
     fetchDynamic: fetchDynamicHtml,
@@ -22,184 +15,6 @@ const defaultDependencies = {
     writeOutput,
     stderrWrite: (message) => process.stderr.write(message)
 };
-function collectRepeatable(value, previous = []) {
-    return [...previous, value];
-}
-function buildProgram() {
-    return new Command()
-        .name("curldown")
-        .description("Fetch URL content and convert it to markdown.")
-        .version(VERSION)
-        .argument("<url>", "The URL to fetch")
-        .option("--dynamic", "Use headless Chromium (Playwright) to render the page")
-        .option("--auto", "Try static first and fallback to dynamic when static output is thin")
-        .option("--format <type>", "Output format: markdown|json", "markdown")
-        .option("-o, --output <path>", "Write output to a file instead of stdout")
-        .option("--timeout-ms <number>", "Timeout in milliseconds")
-        .option("--header <key:value>", "Set custom request header", collectRepeatable, [])
-        .showHelpAfterError()
-        .exitOverride();
-}
-function parseHeaders(rawHeaders) {
-    const headers = {};
-    for (const rawHeader of rawHeaders) {
-        const separatorIndex = rawHeader.indexOf(":");
-        if (separatorIndex <= 0 || separatorIndex === rawHeader.length - 1) {
-            throw new InputError(`Invalid --header value \"${rawHeader}\". Use key:value format.`);
-        }
-        const key = rawHeader.slice(0, separatorIndex).trim();
-        const value = rawHeader.slice(separatorIndex + 1).trim();
-        if (!key || !value) {
-            throw new InputError(`Invalid --header value \"${rawHeader}\". Header key and value are required.`);
-        }
-        headers[key] = value;
-    }
-    return headers;
-}
-function parseFormat(rawFormat) {
-    if (rawFormat === "markdown" || rawFormat === "json") {
-        return rawFormat;
-    }
-    throw new InputError(`Invalid --format value \"${rawFormat}\". Use \"markdown\" or \"json\".`);
-}
-function parseTimeouts(rawTimeout, dynamic, auto) {
-    if (rawTimeout === undefined) {
-        if (dynamic) {
-            return {
-                timeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS,
-                dynamicTimeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS
-            };
-        }
-        if (auto) {
-            return {
-                timeoutMs: DEFAULT_STATIC_TIMEOUT_MS,
-                dynamicTimeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS
-            };
-        }
-        return {
-            timeoutMs: DEFAULT_STATIC_TIMEOUT_MS,
-            dynamicTimeoutMs: DEFAULT_DYNAMIC_TIMEOUT_MS
-        };
-    }
-    const parsed = Number.parseInt(rawTimeout, 10);
-    if (!Number.isInteger(parsed) || parsed <= 0) {
-        throw new InputError(`Invalid --timeout-ms value \"${rawTimeout}\". Must be a positive integer.`);
-    }
-    return {
-        timeoutMs: parsed,
-        dynamicTimeoutMs: parsed
-    };
-}
-function normalizeMarkdown(markdown) {
-    const trimmed = markdown.trim();
-    if (!trimmed) {
-        throw new ConversionError("Content was fetched but markdown output is empty.");
-    }
-    return `${trimmed}\n`;
-}
-function inferTitleFromMarkdown(markdown) {
-    const firstHeading = markdown.match(/^#\s+(.+)$/m)?.[1]?.trim();
-    return firstHeading || undefined;
-}
-function isMarkdownContentType(contentType) {
-    if (!contentType) {
-        return false;
-    }
-    const normalized = contentType.toLowerCase().split(";")[0]?.trim() ?? "";
-    return MARKDOWN_CONTENT_TYPES.has(normalized);
-}
-function countWords(value) {
-    const trimmed = value.trim();
-    if (!trimmed) {
-        return 0;
-    }
-    return trimmed.split(/\s+/).length;
-}
-function shouldAutoFallback(markdown) {
-    const trimmed = markdown.trim();
-    if (!trimmed) {
-        return true;
-    }
-    const lower = trimmed.toLowerCase();
-    if (/enable javascript|javascript is required|checking your browser|just a moment|please wait/.test(lower)) {
-        return true;
-    }
-    const nonEmptyLines = trimmed.split(/\r?\n/).filter((line) => line.trim().length > 0).length;
-    return countWords(trimmed) < 30 && nonEmptyLines <= 2;
-}
-/**
- * Validate and normalize parsed CLI arguments into the canonical runtime shape.
- * Fails fast with {@link InputError} on malformed input.
- */
-function normalizeArgs(urlInput, options) {
-    if (!urlInput) {
-        throw new InputError("A URL argument is required.");
-    }
-    let parsedUrl;
-    try {
-        parsedUrl = new URL(urlInput);
-    }
-    catch (error) {
-        throw new InputError(`Invalid URL \"${urlInput}\".`, {
-            cause: error instanceof Error ? error : undefined
-        });
-    }
-    if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
-        throw new InputError(`Unsupported URL protocol \"${parsedUrl.protocol}\". Only http:// and https:// are supported.`);
-    }
-    const dynamic = options.dynamic ?? false;
-    const auto = options.auto ?? false;
-    if (dynamic && auto) {
-        throw new InputError("--dynamic and --auto cannot be used together.");
-    }
-    const timeouts = parseTimeouts(options.timeoutMs, dynamic, auto);
-    return {
-        url: parsedUrl.toString(),
-        auto,
-        dynamic,
-        format: parseFormat(options.format ?? "markdown"),
-        outputPath: options.output,
-        timeoutMs: timeouts.timeoutMs,
-        dynamicTimeoutMs: timeouts.dynamicTimeoutMs,
-        headers: parseHeaders(options.header ?? [])
-    };
-}
-function prepareContentFromFetchResult(result, deps) {
-    if (isMarkdownContentType(result.contentType)) {
-        const markdown = normalizeMarkdown(result.body);
-        return {
-            markdown,
-            title: inferTitleFromMarkdown(markdown),
-            source: result,
-            passthrough: true
-        };
-    }
-    const markdown = deps.transformHtmlToMarkdown({ html: result.body });
-    return {
-        markdown,
-        title: extractHtmlTitle(result.body),
-        source: result,
-        passthrough: false
-    };
-}
-function formatOutput(args, content, usedDynamic) {
-    if (args.format === "markdown") {
-        return content.markdown;
-    }
-    const payload = {
-        url: args.url,
-        final_url: content.source.finalUrl,
-        title: content.title ?? null,
-        markdown: content.markdown,
-        content_type: content.source.contentType ?? null,
-        status: content.source.status,
-        fetched_at: new Date().toISOString(),
-        word_count: countWords(content.markdown),
-        sha256: createHash("sha256").update(content.markdown).digest("hex"),
-        used_dynamic: usedDynamic
-    };
-    return `${JSON.stringify(payload, null, 2)}\n`;
-}
 /**
  * Execute one curldown CLI invocation and return process exit code.
  * `argv` should not include the Node executable or script path.
@@ -276,28 +91,10 @@ export async function run(argv, deps = defaultDependencies) {
         return curldownError.exitCode;
     }
 }
-function resolvePathStrict(pathInput) {
-    return realpathSync(pathInput);
-}
-/**
- * Determine whether this module was invoked as the CLI entrypoint.
- * Resolves symlinks for both paths so global installs that expose a symlinked bin still execute.
- */
 export function isMainModule(argvPath = process.argv[1]) {
-    if (argvPath === undefined) {
-        return false;
-    }
-    try {
-        const invokedPath = resolvePathStrict(argvPath);
-        const modulePath = resolvePathStrict(fileURLToPath(import.meta.url));
-        return invokedPath === modulePath;
-    }
-    catch {
-        return pathToFileURL(argvPath).href === import.meta.url;
-    }
+    return isMainModuleFor(import.meta.url, argvPath);
 }
-const isMain = isMainModule();
-if (isMain) {
+if (isMainModule()) {
     void run(process.argv.slice(2)).then((exitCode) => {
         process.exitCode = exitCode;
     });

package/dist/transform.js

@@ -1,4 +1,6 @@
+import { Readability } from "@mozilla/readability";
 import { load } from "cheerio";
+import { JSDOM } from "jsdom";
 import { createRequire } from "node:module";
 import TurndownService from "turndown";
 import { DEFAULT_REMOVE_SELECTORS } from "./constants.js";
@@ -12,22 +14,98 @@ const turndown = new TurndownService({
     emDelimiter: "_"
 });
 turndown.use(turndownPluginGfm.gfm);
+const FALLBACK_BASE_URL = "https://curldown.local/";
+const PRIMARY_CONTENT_SELECTOR = "main, article, [role='main']";
+const MIN_PRIMARY_CONTENT_TEXT_LENGTH = 200;
+function getNormalizedTextLength(value) {
+    return value?.replace(/\s+/g, " ").trim().length ?? 0;
+}
+function cleanupFragmentHtml(html) {
+    const $ = load(html);
+    $(DEFAULT_REMOVE_SELECTORS.join(",")).remove();
+    $("img").each((_, element) => {
+        const alt = $(element).attr("alt")?.trim() ?? "";
+        if (!alt) {
+            $(element).remove();
+        }
+    });
+    $("a").each((_, element) => {
+        const link = $(element);
+        const textLength = getNormalizedTextLength(link.text());
+        const hasAltImage = link
+            .find("img")
+            .toArray()
+            .some((image) => getNormalizedTextLength($(image).attr("alt")) > 0);
+        if (textLength === 0 && !hasAltImage) {
+            link.remove();
+        }
+    });
+    return $.root().html() ?? "";
+}
+function extractBodyHtml(document) {
+    return document.body?.innerHTML ?? document.documentElement?.innerHTML ?? "";
+}
+function selectSemanticPrimaryHtml(document) {
+    const candidates = Array.from(document.querySelectorAll(PRIMARY_CONTENT_SELECTOR));
+    const bestCandidate = candidates
+        .map((element) => ({
+        html: element.innerHTML,
+        textLength: getNormalizedTextLength(element.textContent)
+    }))
+        .filter((candidate) => candidate.textLength > 0)
+        .sort((left, right) => right.textLength - left.textLength)[0];
+    if (!bestCandidate || bestCandidate.textLength < MIN_PRIMARY_CONTENT_TEXT_LENGTH) {
+        return undefined;
+    }
+    return bestCandidate.html;
+}
+function selectReadabilityHtml(document) {
+    const article = new Readability(document).parse();
+    if (!article || getNormalizedTextLength(article.textContent) === 0) {
+        return undefined;
+    }
+    return article.content ?? undefined;
+}
+function toMarkdownCandidate(html) {
+    if (!html) {
+        return undefined;
+    }
+    const cleanedHtml = cleanupFragmentHtml(html);
+    if (cleanedHtml.trim().length === 0) {
+        return undefined;
+    }
+    const markdown = turndown.turndown(cleanedHtml).trim();
+    return markdown.length > 0 ? markdown : undefined;
+}
+function getFirstMeaningfulMarkdownLine(markdown) {
+    return markdown
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .find((line) => line.length > 0);
+}
+function startsWithPrimaryHeading(markdown) {
+    return /^#\s+\S/.test(getFirstMeaningfulMarkdownLine(markdown) ?? "");
+}
 /**
  * Convert fetched HTML into markdown.
- * The function removes default non-content nodes before running Turndown
- * with GitHub Flavored Markdown extensions.
+ * The function prefers semantic primary-content containers, falls back to
+ * Readability for unstructured pages, and only converts the full body when
+ * no stronger content signal exists.
  */
 export function transformHtmlToMarkdown(input) {
-    const $ = load(input.html);
-    $(DEFAULT_REMOVE_SELECTORS.join(",")).remove();
-    const bodyHtml = $("body").length > 0 ? $("body").html() ?? "" : $.root().html() ?? "";
-    if (bodyHtml.trim().length === 0) {
+    const dom = new JSDOM(input.html, {
+        url: input.url ?? FALLBACK_BASE_URL
+    });
+    const { document } = dom.window;
+    const semanticMarkdown = toMarkdownCandidate(selectSemanticPrimaryHtml(document));
+    const readabilityMarkdown = toMarkdownCandidate(selectReadabilityHtml(new JSDOM(input.html, { url: input.url ?? FALLBACK_BASE_URL }).window.document));
+    const fallbackMarkdown = toMarkdownCandidate(extractBodyHtml(document));
+    const markdown = semanticMarkdown && startsWithPrimaryHeading(semanticMarkdown) && !startsWithPrimaryHeading(readabilityMarkdown ?? "")
+        ? semanticMarkdown
+        : readabilityMarkdown ?? semanticMarkdown ?? fallbackMarkdown;
+    if (!markdown) {
         throw new ConversionError("No HTML body content found to convert.");
     }
-    const markdown = turndown.turndown(bodyHtml).trim();
-    if (markdown.length === 0) {
-        throw new ConversionError("HTML was fetched but produced empty markdown output.");
-    }
     return `${markdown}\n`;
 }
 /** Extract document title from HTML head when available. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jenslys/curldown",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Fetch URL content and convert it to markdown.",
   "repository": {
     "type": "git",
@@ -30,12 +30,15 @@
   },
   "dependencies": {
     "@joplin/turndown-plugin-gfm": "^1.0.64",
+    "@mozilla/readability": "^0.6.0",
     "cheerio": "^1.2.0",
     "commander": "^14.0.3",
+    "jsdom": "^29.0.0",
     "playwright": "^1.58.2",
     "turndown": "^7.2.2"
   },
   "devDependencies": {
+    "@types/jsdom": "^28.0.1",
     "@types/node": "^25.3.3",
     "@types/turndown": "^5.0.6",
     "typescript": "^5.9.3",