@j0hanz/fetch-url-mcp 1.3.1 → 1.5.0

package/dist/transform/transform.js

@@ -3,15 +3,16 @@ import diagnosticsChannel from 'node:diagnostics_channel';
 import { performance } from 'node:perf_hooks';
 import { isProbablyReaderable, Readability } from '@mozilla/readability';
 import { parseHTML } from 'linkedom';
-import { NodeHtmlMarkdown, } from 'node-html-markdown';
-import { config } from '../config.js';
-import { removeNoiseFromHtml } from '../dom-noise-removal.js';
-import { FetchError, getErrorMessage } from '../errors.js';
-import { isRawTextContentUrl } from '../fetch.js';
-import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../language-detection.js';
-import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../markdown-cleanup.js';
-import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../observability.js';
-import { isLikeNode, isObject } from '../type-guards.js';
+import { removeNoiseFromHtml } from '../lib/content.js';
+import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/content.js';
+import { config } from '../lib/core.js';
+import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
+import { isRawTextContentUrl } from '../lib/http.js';
+import { createAbortError, throwIfAborted } from '../lib/utils.js';
+import { FetchError, getErrorMessage } from '../lib/utils.js';
+import { isObject } from '../lib/utils.js';
+import { translateHtmlFragmentToMarkdown } from './html-translators.js';
+import { extractMetadata, extractMetadataFromHead, mergeMetadata, } from './metadata.js';
 import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
 const utf8Decoder = new TextDecoder('utf-8');
 function decodeInput(input, encoding) {
@@ -30,43 +31,9 @@ function decodeInput(input, encoding) {
         return utf8Decoder.decode(input);
     }
 }
-function getTagName(node) {
-    if (!isLikeNode(node))
-        return '';
-    const raw = node.tagName;
-    return typeof raw === 'string' ? raw.toUpperCase() : '';
-}
 function asError(value) {
     return value instanceof Error ? value : undefined;
 }
-function getAbortReason(signal) {
-    const record = isObject(signal) ? signal : null;
-    return record && 'reason' in record ? record['reason'] : undefined;
-}
-function isTimeoutAbortReason(reason) {
-    return reason instanceof Error && reason.name === 'TimeoutError';
-}
-function throwIfAborted(signal, url, stage) {
-    if (!signal?.aborted)
-        return;
-    const reason = getAbortReason(signal);
-    if (isTimeoutAbortReason(reason)) {
-        throw new FetchError('Request timeout', url, 504, {
-            reason: 'timeout',
-            stage,
-        });
-    }
-    throw new FetchError('Request was canceled', url, 499, {
-        reason: 'aborted',
-        stage,
-    });
-}
-function createAbortError(url, stage) {
-    return new FetchError('Request was canceled', url, 499, {
-        reason: 'aborted',
-        stage,
-    });
-}
 const abortPolicy = { throwIfAborted, createAbortError };
 function buildTransformSignal(signal) {
     const { timeoutMs } = config.transform;
@@ -213,7 +180,22 @@ function trimUtf8Buffer(buffer, maxBytes) {
 function trimDanglingTagFragment(content) {
     const lastOpen = content.lastIndexOf('<');
     const lastClose = content.lastIndexOf('>');
-    return lastOpen > lastClose ? content.substring(0, lastOpen) : content;
+    if (lastOpen > lastClose) {
+        if (lastOpen === content.length - 1) {
+            return content.substring(0, lastOpen);
+        }
+        const code = content.codePointAt(lastOpen + 1);
+        if (code !== undefined &&
+            (code === 47 || // '/'
+                code === 33 || // '!'
+                code === 63 || // '?'
+                (code >= 65 && code <= 90) || // A-Z
+                (code >= 97 && code <= 122)) // a-z
+        ) {
+            return content.substring(0, lastOpen);
+        }
+    }
+    return content;
 }
 function truncateHtml(html, inputTruncated = false) {
     const maxSize = config.constants.maxHtmlSize;
@@ -241,184 +223,6 @@ function willTruncate(html) {
     const maxSize = config.constants.maxHtmlSize;
     return (maxSize > 0 && (html.length > maxSize || getUtf8ByteLength(html) > maxSize));
 }
-const HEAD_END_PATTERN = /<\/head\s*>|<body\b/i;
-const MAX_HEAD_SCAN_LENGTH = 50_000;
-function extractHeadSection(html) {
-    if (html.length <= MAX_HEAD_SCAN_LENGTH) {
-        const match = HEAD_END_PATTERN.exec(html);
-        return match ? html.substring(0, match.index) : null;
-    }
-    const searchText = html.substring(0, MAX_HEAD_SCAN_LENGTH);
-    const match = HEAD_END_PATTERN.exec(searchText);
-    if (!match)
-        return null;
-    return html.substring(0, match.index);
-}
-function extractMetadataFromHead(html, baseUrl) {
-    const headSection = extractHeadSection(html);
-    if (!headSection)
-        return null;
-    try {
-        const { document } = parseHTML(`<!DOCTYPE html><html>${headSection}</head><body></body></html>`);
-        return extractMetadata(document, baseUrl);
-    }
-    catch {
-        return null;
-    }
-}
-function mergeMetadata(early, late) {
-    if (!early)
-        return late;
-    const merged = {};
-    const title = late.title ?? early.title;
-    const description = late.description ?? early.description;
-    const author = late.author ?? early.author;
-    const image = late.image ?? early.image;
-    const publishedAt = late.publishedAt ?? early.publishedAt;
-    const modifiedAt = late.modifiedAt ?? early.modifiedAt;
-    if (title !== undefined)
-        merged.title = title;
-    if (description !== undefined)
-        merged.description = description;
-    if (author !== undefined)
-        merged.author = author;
-    if (image !== undefined)
-        merged.image = image;
-    if (publishedAt !== undefined)
-        merged.publishedAt = publishedAt;
-    if (modifiedAt !== undefined)
-        merged.modifiedAt = modifiedAt;
-    return merged;
-}
-const META_PROPERTY_HANDLERS = new Map([
-    [
-        'og:title',
-        (ctx, c) => {
-            ctx.title.og = c;
-        },
-    ],
-    [
-        'og:description',
-        (ctx, c) => {
-            ctx.description.og = c;
-        },
-    ],
-    [
-        'og:image',
-        (ctx, c) => {
-            ctx.image = c;
-        },
-    ],
-    [
-        'article:published_time',
-        (ctx, c) => {
-            ctx.publishedAt = c;
-        },
-    ],
-    [
-        'article:modified_time',
-        (ctx, c) => {
-            ctx.modifiedAt = c;
-        },
-    ],
-]);
-const META_NAME_HANDLERS = new Map([
-    [
-        'twitter:title',
-        (ctx, c) => {
-            ctx.title.twitter = c;
-        },
-    ],
-    [
-        'twitter:description',
-        (ctx, c) => {
-            ctx.description.twitter = c;
-        },
-    ],
-    [
-        'description',
-        (ctx, c) => {
-            ctx.description.standard = c;
-        },
-    ],
-    [
-        'author',
-        (ctx, c) => {
-            ctx.author = c;
-        },
-    ],
-]);
-function processMetaTag(ctx, tag) {
-    const content = tag.getAttribute('content')?.trim();
-    if (!content)
-        return;
-    const property = tag.getAttribute('property');
-    if (property)
-        META_PROPERTY_HANDLERS.get(property)?.(ctx, content);
-    const name = tag.getAttribute('name');
-    if (name)
-        META_NAME_HANDLERS.get(name)?.(ctx, content);
-}
-function buildMetaContext(document) {
-    const ctx = { title: {}, description: {} };
-    for (const tag of document.querySelectorAll('meta')) {
-        processMetaTag(ctx, tag);
-    }
-    const titleEl = document.querySelector('title');
-    if (!ctx.title.standard && titleEl?.textContent) {
-        ctx.title.standard = titleEl.textContent.trim();
-    }
-    return ctx;
-}
-function resolveMetadataFromContext(ctx) {
-    const metadata = {};
-    const resolvedTitle = ctx.title.og ?? ctx.title.twitter ?? ctx.title.standard;
-    const resolvedDesc = ctx.description.og ?? ctx.description.twitter ?? ctx.description.standard;
-    if (resolvedTitle)
-        metadata.title = resolvedTitle;
-    if (resolvedDesc)
-        metadata.description = resolvedDesc;
-    if (ctx.author)
-        metadata.author = ctx.author;
-    if (ctx.image)
-        metadata.image = ctx.image;
-    if (ctx.publishedAt)
-        metadata.publishedAt = ctx.publishedAt;
-    if (ctx.modifiedAt)
-        metadata.modifiedAt = ctx.modifiedAt;
-    return metadata;
-}
-function extractMetadata(document, baseUrl) {
-    const ctx = buildMetaContext(document);
-    const metadata = resolveMetadataFromContext(ctx);
-    if (baseUrl) {
-        const icon32 = document.querySelector('link[rel="icon"][sizes="32x32"]');
-        const href = icon32?.getAttribute('href');
-        if (href) {
-            const resolved = resolveFaviconUrl(href, baseUrl);
-            if (resolved)
-                metadata.favicon = resolved;
-        }
-    }
-    return metadata;
-}
-function resolveFaviconUrl(href, baseUrl) {
-    const trimmed = href.trim();
-    if (!trimmed)
-        return undefined;
-    if (trimmed.toLowerCase().startsWith('data:'))
-        return undefined;
-    try {
-        const resolved = new URL(trimmed, baseUrl);
-        if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:') {
-            return undefined;
-        }
-        return resolved.toString();
-    }
-    catch {
-        return undefined;
-    }
-}
 function isReadabilityCompatible(doc) {
     if (!isObject(doc))
         return false;
@@ -588,433 +392,6 @@ export function extractContent(html, url, options = {
     const result = extractContentContext(html, url, options);
     return { article: result.article, metadata: result.metadata };
 }
-const CODE_BLOCK = {
-    fence: '```',
-    format: (code, language = '') => `\`\`\`${language}\n${code}\n\`\`\``,
-};
-function buildInlineCode(content) {
-    const trimmed = content.trim();
-    if (!trimmed)
-        return '``';
-    let maxBackticks = 0;
-    let currentRun = 0;
-    for (const char of trimmed) {
-        if (char === '`')
-            currentRun += 1;
-        else {
-            if (currentRun > maxBackticks)
-                maxBackticks = currentRun;
-            currentRun = 0;
-        }
-    }
-    if (currentRun > maxBackticks)
-        maxBackticks = currentRun;
-    const delimiter = '`'.repeat(maxBackticks + 1);
-    const padding = trimmed.startsWith('`') || trimmed.endsWith('`') ? ' ' : '';
-    return `${delimiter}${padding}${trimmed}${padding}${delimiter}`;
-}
-function deriveAltFromImageUrl(src) {
-    if (!src)
-        return '';
-    try {
-        const isAbsolute = URL.canParse(src);
-        let parsed = null;
-        if (isAbsolute) {
-            parsed = new URL(src);
-        }
-        else if (URL.canParse(src, 'http://localhost')) {
-            parsed = new URL(src, 'http://localhost');
-        }
-        if (!parsed)
-            return '';
-        if (isAbsolute &&
-            parsed.protocol !== 'http:' &&
-            parsed.protocol !== 'https:') {
-            return '';
-        }
-        const { pathname } = parsed;
-        const segments = pathname.split('/');
-        const filename = segments.pop() ?? '';
-        if (!filename)
-            return '';
-        const dotIndex = filename.lastIndexOf('.');
-        const name = dotIndex > 0 ? filename.slice(0, dotIndex) : filename;
-        return name.replace(/[_-]+/g, ' ').trim();
-    }
-    catch {
-        return '';
-    }
-}
-function hasGetAttribute(value) {
-    return (isObject(value) &&
-        typeof value.getAttribute === 'function');
-}
-function isCodeBlock(parent) {
-    const tagName = getTagName(parent);
-    return tagName === 'PRE' || tagName === 'WRAPPED-PRE';
-}
-function isAnchor(node) {
-    return getTagName(node) === 'A';
-}
-function resolveAttributeLanguage(node) {
-    const getAttribute = hasGetAttribute(node)
-        ? node.getAttribute.bind(node)
-        : undefined;
-    const className = getAttribute?.('class') ?? '';
-    const dataLanguage = getAttribute?.('data-language') ?? '';
-    return resolveLanguageFromAttributes(className, dataLanguage);
-}
-function findLanguageFromCodeChild(node) {
-    if (!isLikeNode(node))
-        return undefined;
-    const childNodes = Array.from(node.childNodes ?? []);
-    for (const child of childNodes) {
-        if (!isLikeNode(child))
-            continue;
-        const raw = child.rawTagName;
-        const tagName = typeof raw === 'string' ? raw.toUpperCase() : '';
-        if (tagName === 'CODE')
-            return resolveAttributeLanguage(child);
-    }
-    return undefined;
-}
-function createCodeBlockPostprocessor(language) {
-    return ({ content }) => {
-        const trimmed = content.trim();
-        if (!trimmed)
-            return '';
-        const resolvedLanguage = language ?? detectLanguageFromCode(trimmed) ?? '';
-        return CODE_BLOCK.format(trimmed, resolvedLanguage);
-    };
-}
-function buildInlineCodeTranslator() {
-    return {
-        spaceIfRepeatingChar: true,
-        noEscape: true,
-        postprocess: ({ content }) => buildInlineCode(content),
-    };
-}
-function buildCodeTranslator(ctx) {
-    const inlineCodeTranslator = buildInlineCodeTranslator();
-    if (!isObject(ctx))
-        return inlineCodeTranslator;
-    const { parent } = ctx;
-    if (!isCodeBlock(parent))
-        return inlineCodeTranslator;
-    return { noEscape: true, preserveWhitespace: true };
-}
-function extractFirstSrcsetUrl(srcset) {
-    const first = srcset.split(',')[0];
-    if (!first)
-        return '';
-    return first.trim().split(/\s+/)[0] ?? '';
-}
-const LAZY_SRC_ATTRIBUTES = [
-    'data-src',
-    'data-lazy-src',
-    'data-original',
-    'data-srcset',
-];
-function isDataUri(value) {
-    return value.startsWith('data:');
-}
-function extractNonDataSrcsetUrl(value) {
-    const url = extractFirstSrcsetUrl(value);
-    return url && !isDataUri(url) ? url : undefined;
-}
-function resolveLazySrc(getAttribute) {
-    for (const attr of LAZY_SRC_ATTRIBUTES) {
-        const lazy = getAttribute(attr);
-        if (!lazy || isDataUri(lazy))
-            continue;
-        if (attr === 'data-srcset') {
-            const url = extractNonDataSrcsetUrl(lazy);
-            if (url)
-                return url;
-            continue;
-        }
-        return lazy;
-    }
-    return undefined;
-}
-function resolveImageSrc(getAttribute) {
-    if (!getAttribute)
-        return '';
-    const srcRaw = getAttribute('src') ?? '';
-    if (srcRaw && !isDataUri(srcRaw))
-        return srcRaw;
-    // First check common lazy-loading attributes that may contain non-data URLs before falling back to the native srcset, as some sites use data URIs in lazy attributes while still providing valid URLs in srcset.
-    const lazySrc = resolveLazySrc(getAttribute);
-    if (lazySrc)
-        return lazySrc;
-    // If the src is a data URI or missing, check srcset for a valid URL. Some sites use srcset with data URIs in src and actual URLs in srcset for responsive images.
-    const srcset = getAttribute('srcset');
-    if (srcset) {
-        const url = extractNonDataSrcsetUrl(srcset);
-        if (url)
-            return url;
-    }
-    // If the only available src is a data URI, we choose to omit it rather than include the raw data in the alt text or URL, as data URIs can be very long and are not useful in Markdown output.
-    if (isDataUri(srcRaw))
-        return '[data URI removed]';
-    return '';
-}
-function buildImageTranslator(ctx) {
-    if (!isObject(ctx))
-        return { content: '' };
-    const { node, parent } = ctx;
-    const getAttribute = hasGetAttribute(node)
-        ? node.getAttribute.bind(node)
-        : undefined;
-    const src = resolveImageSrc(getAttribute);
-    const existingAlt = getAttribute?.('alt') ?? '';
-    const alt = existingAlt.trim() || deriveAltFromImageUrl(src);
-    const markdown = `![${alt}](${src})`;
-    if (isAnchor(parent)) {
-        return { content: markdown };
-    }
-    return { content: `\n\n${markdown}\n\n` };
-}
-const GFM_ALERT_MAP = new Map([
-    ['note', 'NOTE'],
-    ['info', 'NOTE'],
-    ['tip', 'TIP'],
-    ['hint', 'TIP'],
-    ['warning', 'WARNING'],
-    ['warn', 'WARNING'],
-    ['caution', 'CAUTION'],
-    ['danger', 'CAUTION'],
-    ['important', 'IMPORTANT'],
-]);
-function resolveGfmAlertType(className) {
-    const lower = className.toLowerCase();
-    for (const [key, type] of GFM_ALERT_MAP) {
-        if (lower.includes(key))
-            return type;
-    }
-    return undefined;
-}
-function resolveDlNodeName(child) {
-    if (!isLikeNode(child))
-        return '';
-    const raw = child.nodeName;
-    return typeof raw === 'string' ? raw.toUpperCase() : '';
-}
-function resolveDlTextContent(child) {
-    if (!isLikeNode(child))
-        return '';
-    const raw = child.textContent;
-    return typeof raw === 'string' ? raw.trim() : '';
-}
-function buildDlChildFragment(child) {
-    const nodeName = resolveDlNodeName(child);
-    if (nodeName === 'DT')
-        return `**${resolveDlTextContent(child)}**\n`;
-    if (nodeName === 'DD')
-        return `: ${resolveDlTextContent(child)}\n`;
-    return null;
-}
-function hasComplexTableLayout(node) {
-    if (!isLikeNode(node))
-        return false;
-    const innerHTML = typeof node.innerHTML === 'string' ? node.innerHTML : '';
-    return /(?:colspan|rowspan)=["']?[2-9]/i.test(innerHTML);
-}
-function buildPreTranslator(ctx) {
-    if (!isObject(ctx))
-        return {};
-    const { node } = ctx;
-    const attributeLanguage = resolveAttributeLanguage(node) ?? findLanguageFromCodeChild(node);
-    return {
-        noEscape: true,
-        preserveWhitespace: true,
-        postprocess: createCodeBlockPostprocessor(attributeLanguage),
-    };
-}
-function createCustomTranslators() {
-    return {
-        code: (ctx) => buildCodeTranslator(ctx),
-        img: (ctx) => buildImageTranslator(ctx),
-        table: (ctx) => {
-            if (!isObject(ctx))
-                return {};
-            const { node } = ctx;
-            if (hasComplexTableLayout(node)) {
-                return {
-                    postprocess: ({ content }) => {
-                        const trimmed = content.trim();
-                        if (!trimmed)
-                            return '';
-                        return `\n\n${trimmed}\n\n`;
-                    },
-                };
-            }
-            return {};
-        },
-        dl: (ctx) => {
-            if (!isObject(ctx))
-                return { content: '' };
-            const { node } = ctx;
-            if (!isLikeNode(node))
-                return { content: '' };
-            const childNodes = Array.from(node.childNodes ?? []);
-            let items = '';
-            for (const child of childNodes) {
-                const fragment = buildDlChildFragment(child);
-                if (fragment !== null)
-                    items += fragment;
-            }
-            return { content: items ? `\n${items}\n` : '' };
-        },
-        div: (ctx) => {
-            if (!isObject(ctx))
-                return {};
-            const { node } = ctx;
-            if (!isLikeNode(node))
-                return {};
-            const getAttribute = typeof node.getAttribute === 'function'
-                ? node.getAttribute.bind(node)
-                : undefined;
-            const className = getAttribute?.('class') ?? '';
-            if (className.includes('mermaid')) {
-                return {
-                    noEscape: true,
-                    preserveWhitespace: true,
-                    postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
-                };
-            }
-            const isAdmonition = className.includes('admonition') ||
-                className.includes('callout') ||
-                className.includes('custom-block') ||
-                getAttribute?.('role') === 'alert' ||
-                /\b(note|tip|info|warning|danger|caution|important)\b/i.test(className);
-            if (isAdmonition) {
-                return {
-                    postprocess: ({ content }) => {
-                        const alertType = resolveGfmAlertType(className);
-                        const lines = content.trim().split('\n');
-                        const header = alertType ? `> [!${alertType}]\n` : '';
-                        return `\n\n${header}> ${lines.join('\n> ')}\n\n`;
-                    },
-                };
-            }
-            if (!className.includes('type'))
-                return {};
-            return {
-                postprocess: ({ content }) => {
-                    const lines = content.split('\n');
-                    const separated = [];
-                    for (let i = 0; i < lines.length; i++) {
-                        const line = lines[i] ?? '';
-                        const nextLine = i < lines.length - 1 ? (lines[i + 1] ?? '') : '';
-                        separated.push(line);
-                        if (line.trim() &&
-                            nextLine.trim() &&
-                            line.includes(':') &&
-                            nextLine.includes(':') &&
-                            !line.startsWith(' ') &&
-                            !nextLine.startsWith(' ')) {
-                            separated.push('');
-                        }
-                    }
-                    return separated.join('\n');
-                },
-            };
-        },
-        kbd: () => ({
-            postprocess: ({ content }) => `\`${content}\``,
-        }),
-        mark: () => ({
-            postprocess: ({ content }) => `==${content}==`,
-        }),
-        sub: () => ({
-            postprocess: ({ content }) => `~${content}~`,
-        }),
-        sup: () => ({
-            postprocess: ({ content }) => `^${content}^`,
-        }),
-        section: (ctx) => {
-            if (isObject(ctx)) {
-                const { node } = ctx;
-                if (isLikeNode(node)) {
-                    const getAttribute = typeof node.getAttribute === 'function'
-                        ? node.getAttribute.bind(node)
-                        : undefined;
-                    if (getAttribute?.('class')?.includes('tsd-member')) {
-                        return {
-                            postprocess: ({ content }) => `\n\n&nbsp;\n\n${content}\n\n`,
-                        };
-                    }
-                }
-            }
-            return {
-                postprocess: ({ content }) => `\n\n${content}\n\n`,
-            };
-        },
-        details: () => ({
-            postprocess: ({ content }) => {
-                const trimmed = content.trim();
-                if (!trimmed)
-                    return '';
-                return `\n\n${trimmed}\n\n`;
-            },
-        }),
-        summary: () => ({
-            postprocess: ({ content }) => `${content.trim()}\n\n`,
-        }),
-        span: (ctx) => {
-            if (!isObject(ctx))
-                return {};
-            const { node } = ctx;
-            if (!isLikeNode(node))
-                return {};
-            const getAttribute = typeof node.getAttribute === 'function'
-                ? node.getAttribute.bind(node)
-                : undefined;
-            const dataAs = getAttribute?.('data-as') ?? '';
-            if (dataAs === 'p') {
-                return {
-                    postprocess: ({ content }) => `\n\n${content.trim()}\n\n`,
-                };
-            }
-            return {};
-        },
-        pre: (ctx) => {
-            if (!isObject(ctx))
-                return buildPreTranslator(ctx);
-            const { node } = ctx;
-            if (!isLikeNode(node)) {
-                return buildPreTranslator(ctx);
-            }
-            const getAttribute = typeof node.getAttribute === 'function'
-                ? node.getAttribute.bind(node)
-                : undefined;
-            const className = getAttribute?.('class') ?? '';
-            if (className.includes('mermaid')) {
-                return {
-                    noEscape: true,
-                    preserveWhitespace: true,
-                    postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
-                };
-            }
-            return buildPreTranslator(ctx);
-        },
-    };
-}
-let markdownConverter = null;
-function getMarkdownConverter() {
-    markdownConverter ??= new NodeHtmlMarkdown({
-        codeFence: CODE_BLOCK.fence,
-        codeBlockStyle: 'fenced',
-        emDelimiter: '_',
-        bulletMarker: '-',
-        globalEscape: [/[\\`*_~]/gm, '\\$&'],
-    }, createCustomTranslators());
-    return markdownConverter;
-}
-function translateHtmlFragmentToMarkdown(html) {
-    return getMarkdownConverter().translate(html).trim();
-}
 function isWhitespaceChar(code) {
     return code === 9 || code === 10 || code === 12 || code === 13 || code === 32;
 }
@@ -1127,8 +504,8 @@ function resolveRelativeUrls(markdown, baseUrl, signal) {
     let lastIndex = 0;
     let lineCount = 0;
     while (lastIndex < len) {
-        if (++lineCount % 500 === 0 && signal?.aborted) {
-            throw new Error('Transform aborted during URL resolution');
+        if (++lineCount % 500 === 0) {
+            abortPolicy.throwIfAborted(signal, baseUrl, 'markdown:resolve-urls');
         }
         let nextIndex = markdown.indexOf('\n', lastIndex);
         let line;
@@ -1315,7 +692,12 @@ function getTextContentSkippingHidden(node, parts) {
     }
     if (nodeType !== 1)
         return;
-    const { tagName } = node;
+    const element = node;
+    if (element.hasAttribute('hidden') ||
+        element.getAttribute('aria-hidden') === 'true') {
+        return;
+    }
+    const { tagName } = element;
     if (tagName === 'SCRIPT' || tagName === 'STYLE' || tagName === 'NOSCRIPT')
         return;
     const { childNodes } = node;
@@ -1479,19 +861,21 @@ function shouldUseArticleContent(article, originalHtmlOrDocument) {
 function buildContentSource(params) {
     const { html, url, article, extractedMeta, includeMetadata, useArticleContent, document, truncated, skipNoiseRemoval, signal, } = params;
     const metadata = createContentMetadataBlock(url, article, extractedMeta, useArticleContent, includeMetadata);
+    const base = {
+        favicon: extractedMeta.favicon,
+        metadata,
+        extractedMetadata: extractedMeta,
+        truncated,
+    };
     if (useArticleContent && article) {
-        // Readability output can still be noisy (unless user requested skip).
         const cleanedArticleHtml = skipNoiseRemoval
             ? article.content
             : removeNoiseFromHtml(article.content, undefined, url, signal);
         return {
+            ...base,
             sourceHtml: cleanedArticleHtml,
             title: article.title,
-            favicon: extractedMeta.favicon,
-            metadata,
-            extractedMetadata: extractedMeta,
             skipNoiseRemoval: true,
-            truncated,
         };
     }
     if (document) {
@@ -1499,36 +883,18 @@ function buildContentSource(params) {
             ? html
             : removeNoiseFromHtml(html, document, url, signal);
         const contentRoot = findContentRoot(document);
-        if (contentRoot) {
-            return {
-                sourceHtml: contentRoot,
-                title: extractedMeta.title,
-                favicon: extractedMeta.favicon,
-                metadata,
-                extractedMetadata: extractedMeta,
-                skipNoiseRemoval: true,
-                document,
-                truncated,
-            };
-        }
         return {
-            sourceHtml: cleanedHtml,
+            ...base,
+            sourceHtml: contentRoot ?? cleanedHtml,
             title: extractedMeta.title,
-            favicon: extractedMeta.favicon,
-            metadata,
-            extractedMetadata: extractedMeta,
             skipNoiseRemoval: true,
             document,
-            truncated,
         };
     }
     return {
+        ...base,
         sourceHtml: html,
         title: extractedMeta.title,
-        favicon: extractedMeta.favicon,
-        metadata,
-        extractedMetadata: extractedMeta,
-        truncated,
     };
 }
 function resolveContentSource(params) {
@@ -1719,4 +1085,3 @@ export async function transformHtmlToMarkdown(html, url, options) {
 export async function transformBufferToMarkdown(htmlBuffer, url, options) {
     return transformInputToMarkdown(htmlBuffer, url, options);
 }
-//# sourceMappingURL=transform.js.map