@j0hanz/fetch-url-mcp 1.9.4 → 1.9.5

package/dist/lib/code-lang.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"code-lang.d.ts","sourceRoot":"","sources":["../../src/lib/code-lang.ts"],"names":[],"mappings":"AA6QA,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAuBpB;AAqBD,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAqBvE"}
+{"version":3,"file":"code-lang.d.ts","sourceRoot":"","sources":["../../src/lib/code-lang.ts"],"names":[],"mappings":"AAgRA,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAuBpB;AAqBD,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAqBvE"}

package/dist/lib/code-lang.js

@@ -82,7 +82,7 @@ const RUST_REGEX = /\b(?:fn|impl|struct|enum)\b/;
 const JS_REGEX = /\b(?:const|let|var|function|class|async|await|export|import)\b/;
 const PYTHON_UNIQUE_REGEX = /\b(?:def |elif |except |finally:|yield |lambda |raise |pass$)/m;
 const JS_SIGNAL_REGEX = /\b(?:const |let |var |function |require\(|=>|===|!==|console\.)/;
-const CSS_REGEX = /@media|@import|@keyframes/;
+const CSS_REGEX = /@media|@import|@keyframes|@theme\b|@utility\b|@layer\b|@apply\b|@variant\b|@custom-variant\b|@reference\b|@source\b/;
 const CSS_PROPERTY_REGEX = /^\s*[a-z][\w-]*\s*:/;
 function containsJsxTag(code) {
     const len = code.length;
@@ -195,6 +195,8 @@ function hasJsSignals(lowerCode) {
         lowerCode.includes("from '"));
 }
 function matchPython(ctx) {
+    if (matchHtml(ctx))
+        return false;
     const l = ctx.lower;
     if (l.includes('print(') || l.includes('__name__'))
         return true;
@@ -229,6 +231,7 @@ const LANGUAGES = [
     { lang: 'jsx', weight: 22, match: matchJsx },
     { lang: 'typescript', weight: 20, match: matchTypeScript },
     { lang: 'sql', weight: 20, match: matchSql },
+    { lang: 'html', weight: 19, match: matchHtml },
     { lang: 'python', weight: 18, match: matchPython },
     {
         lang: 'css',
@@ -238,7 +241,6 @@ const LANGUAGES = [
     { lang: 'bash', weight: 15, match: (ctx) => detectBashIndicators(ctx.lines) },
     { lang: 'yaml', weight: 15, match: (ctx) => detectYamlStructure(ctx.lines) },
     { lang: 'javascript', weight: 15, match: (ctx) => JS_REGEX.test(ctx.lower) },
-    { lang: 'html', weight: 12, match: matchHtml },
     {
         lang: 'json',
         weight: 10,

package/dist/lib/dom-prep.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dom-prep.d.ts","sourceRoot":"","sources":["../../src/lib/dom-prep.ts"],"names":[],"mappings":"AA8hBA,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CAQR;AA4DD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,IAAI,CAiBN;AA+DD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,MAAM,CAcR"}
+{"version":3,"file":"dom-prep.d.ts","sourceRoot":"","sources":["../../src/lib/dom-prep.ts"],"names":[],"mappings":"AAwlBA,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CAQR;AAmPD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,IAAI,CAkBN;AAiED,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,MAAM,CAcR"}

package/dist/lib/dom-prep.js

@@ -17,6 +17,8 @@ const NOISE_PATTERNS = [
 ];
 const HEADER_NOISE_PATTERN = /\b(site-header|masthead|topbar|navbar|nav(?:bar)?|menu|header-nav)\b/i;
 const FIXED_OR_HIGH_Z_PATTERN = /\b(?:fixed|sticky|z-(?:4\d|50)|isolate)\b/;
+const HEADING_PERMALINK_TEXT_PATTERN = /^(?:#|¶|§|¤|🔗)$/u;
+const HEADING_PERMALINK_CLASS_PATTERN = /\b(?:mark|permalink|hash-link|anchor(?:js)?-?link|header-?link|heading-anchor|deep-link)\b/i;
 const SKIP_URL_PREFIXES = [
     '#',
     'javascript:',
@@ -50,6 +52,9 @@ const NAVIGATION_ROLES = new Set([
     'alertdialog',
     'search',
 ]);
+const INLINE_DEMO_INSTRUCTION_MAX_CHARS = 160;
+const REDUNDANT_PREVIEW_SEGMENT_MAX_CHARS = 60;
+const REDUNDANT_PREVIEW_MAX_SEGMENTS = 12;
 const INTERACTIVE_CONTENT_ROLES = new Set([
     'tabpanel',
     'tab',
@@ -362,11 +367,8 @@ function cleanHeadings(document) {
             const a = anchors[j];
             if (!a?.parentNode)
                 continue;
-            const href = a.getAttribute('href') ?? '';
-            const txt = (a.textContent || '').replace(/[\u200B\s]/g, '');
-            if (href.startsWith('#') && txt.length === 0) {
+            if (isHeadingPermalinkAnchor(a))
                 a.remove();
-            }
         }
         // Strip zero-width spaces from text nodes
         const walker = document.createTreeWalker(h, NODE_FILTER_SHOW_TEXT);
@@ -378,6 +380,47 @@ function cleanHeadings(document) {
         }
     }
 }
+function getCollapsedHeadingAnchorText(anchor) {
+    return (anchor.textContent || '').replace(/[\u200B\s]/g, '');
+}
+function isHeadingPermalinkAnchor(anchor) {
+    const href = anchor.getAttribute('href') ?? '';
+    if (!href.startsWith('#'))
+        return false;
+    const text = getCollapsedHeadingAnchorText(anchor);
+    if (text.length === 0 || HEADING_PERMALINK_TEXT_PATTERN.test(text)) {
+        return true;
+    }
+    const className = anchor.getAttribute('class') ?? '';
+    if (HEADING_PERMALINK_CLASS_PATTERN.test(className) && text.length <= 2) {
+        return true;
+    }
+    const ariaHidden = anchor.getAttribute('aria-hidden');
+    const tabindex = anchor.getAttribute('tabindex');
+    return (ariaHidden === 'true' || tabindex === '-1') && text.length <= 2;
+}
+function getDirectRows(section) {
+    return Array.from(section.children).filter((child) => child.tagName === 'TR');
+}
+function getDirectCells(row) {
+    return Array.from(row.children).filter((child) => child.tagName === 'TH' || child.tagName === 'TD');
+}
+function hoistNestedRows(table) {
+    const sections = Array.from(table.querySelectorAll('thead,tbody,tfoot'));
+    for (const section of sections) {
+        const rows = getDirectRows(section);
+        for (const row of rows) {
+            let insertAfter = row;
+            for (const cell of getDirectCells(row)) {
+                const nestedRows = Array.from(cell.querySelectorAll('tr')).filter((nested) => nested.closest('table') === table);
+                for (const nestedRow of nestedRows) {
+                    insertAfter.after(nestedRow);
+                    insertAfter = nestedRow;
+                }
+            }
+        }
+    }
+}
 function stripNoise(document, context, signal) {
     cleanHeadings(document);
     // Structural Removal
@@ -506,6 +549,130 @@ function escapeTableCellPipes(document) {
         }
     }
 }
+function normalizeWhitespace(value) {
+    return value.replace(/\s+/g, ' ').trim();
+}
+function hasDirectPreDescendant(element) {
+    return (element.tagName === 'PRE' ||
+        Array.from(element.children).some((child) => child.tagName === 'PRE' || child.querySelector('pre') !== null));
+}
+function collectLeafTextSegments(element) {
+    const seen = new Set();
+    const segments = [];
+    const candidates = element.querySelectorAll('p,li,div,span');
+    for (const candidate of candidates) {
+        if (candidate.children.length > 0 ||
+            candidate.querySelector('pre,code,table,ul,ol,blockquote,figure') !== null) {
+            continue;
+        }
+        const text = normalizeWhitespace(candidate.textContent || '');
+        if (text.length === 0 ||
+            text.length > REDUNDANT_PREVIEW_SEGMENT_MAX_CHARS ||
+            seen.has(text)) {
+            continue;
+        }
+        seen.add(text);
+        segments.push(text);
+    }
+    if (segments.length > 0)
+        return segments;
+    const fallback = normalizeWhitespace(element.textContent || '');
+    return fallback ? [fallback] : [];
+}
+function isHostnameLike(value) {
+    return /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(value);
+}
+function hasPreviewMedia(element) {
+    return element.querySelector('svg,canvas') !== null;
+}
+function isRedundantCodePreview(preview, codeContainer) {
+    if (preview.tagName === 'FIGCAPTION' ||
+        preview.querySelector('a[href],button,input,select,textarea,form,video,audio,iframe,table,ul,ol,blockquote') !== null) {
+        return false;
+    }
+    const segments = collectLeafTextSegments(preview);
+    if (segments.length === 0 ||
+        segments.length > REDUNDANT_PREVIEW_MAX_SEGMENTS ||
+        segments.some((segment) => segment.length > REDUNDANT_PREVIEW_SEGMENT_MAX_CHARS)) {
+        return false;
+    }
+    const codeText = normalizeWhitespace(codeContainer.textContent || '');
+    if (!codeText)
+        return false;
+    const matchingSegments = segments.filter((segment) => codeText.includes(segment));
+    if (matchingSegments.length === segments.length)
+        return true;
+    return ((hasPreviewMedia(preview) ||
+        segments.some((segment) => isHostnameLike(segment))) &&
+        matchingSegments.length > 0 &&
+        segments.every((segment) => segment.length <= REDUNDANT_PREVIEW_SEGMENT_MAX_CHARS));
+}
+function pruneFigurePreviewPanes(document) {
+    for (const figure of document.querySelectorAll('figure')) {
+        const directChildren = Array.from(figure.children);
+        const codeChild = directChildren.find((child) => hasDirectPreDescendant(child));
+        if (!codeChild)
+            continue;
+        for (const child of directChildren) {
+            if (child === codeChild || child.tagName === 'FIGCAPTION')
+                continue;
+            if (isRedundantCodePreview(child, codeChild))
+                child.remove();
+        }
+    }
+}
+function isDemoInstructionBlock(element) {
+    if (element.querySelector('a[href],pre,code,table,ul,ol,blockquote,figure,h1,h2,h3,h4,h5,h6') !== null) {
+        return false;
+    }
+    const text = normalizeWhitespace(element.textContent || '');
+    if (text.length === 0 ||
+        text.length > INLINE_DEMO_INSTRUCTION_MAX_CHARS ||
+        /[.!?]$/.test(text)) {
+        return false;
+    }
+    return collectLeafTextSegments(element).length <= 3;
+}
+function pruneDemoInstructionBlocks(document) {
+    for (const container of document.querySelectorAll('div,section,article')) {
+        const children = Array.from(container.children);
+        const figureIndex = children.findIndex((child) => child.tagName === 'FIGURE' && child.querySelector('pre') !== null);
+        if (figureIndex <= 0)
+            continue;
+        for (let i = 0; i < figureIndex; i++) {
+            const child = children[i];
+            if (child && isDemoInstructionBlock(child))
+                child.remove();
+        }
+    }
+}
+function normalizeHighlightedCodeLines(document) {
+    for (const code of document.querySelectorAll('pre > code')) {
+        const directChildren = Array.from(code.children);
+        if (directChildren.length < 2)
+            continue;
+        const directSpans = directChildren.filter((child) => child.tagName === 'SPAN');
+        if (directSpans.length !== directChildren.length)
+            continue;
+        const hasLineClass = directSpans.some((child) => (child.getAttribute('class') ?? '').split(/\s+/).includes('line'));
+        const hasNewlineNode = Array.from(code.childNodes).some((node) => node.nodeType === 3 && /[\r\n]/.test(node.textContent ?? ''));
+        if (hasNewlineNode || !hasLineClass)
+            continue;
+        for (let i = 0; i < directSpans.length - 1; i++) {
+            const current = directSpans[i];
+            const next = current?.nextSibling;
+            if (next?.nodeType === 3 && (next.textContent ?? '').startsWith('\n')) {
+                continue;
+            }
+            current?.after(document.createTextNode('\n'));
+        }
+    }
+}
+function cleanCodeExamples(document) {
+    pruneFigurePreviewPanes(document);
+    pruneDemoInstructionBlocks(document);
+    normalizeHighlightedCodeLines(document);
+}
 function separateAdjacentInlineElements(document) {
     const badges = document.querySelectorAll('span.chakra-badge, [data-scope="badge"], [class*="badge"]');
     for (const badge of badges) {
@@ -524,6 +691,7 @@ export function prepareDocumentForMarkdown(document, baseUrl, signal) {
     }
     stripNoise(document, context, signal);
     stripTabTriggers(document);
+    cleanCodeExamples(document);
     separateAdjacentInlineElements(document);
     flattenTableCellBreaks(document);
     escapeTableCellPipes(document);
@@ -552,6 +720,7 @@ function normalizeTableStructure(document) {
                 }
             }
         }
+        hoistNestedRows(table);
     }
 }
 function flattenTableCellBreaks(document) {

package/dist/lib/md-cleanup.d.ts

@@ -1,4 +1,5 @@
 interface CleanupOptions {
+    preserveEmptyHeadings?: boolean;
     signal?: AbortSignal;
     url?: string;
 }

package/dist/lib/md-cleanup.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"md-cleanup.d.ts","sourceRoot":"","sources":["../../src/lib/md-cleanup.ts"],"names":[],"mappings":"AAmEA,UAAU,cAAc;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AA6VD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,EACf,kBAAkB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAC3C,MAAM,CAuCR;AAWD,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,cAAc,GACvB,MAAM,CAUR"}
+{"version":3,"file":"md-cleanup.d.ts","sourceRoot":"","sources":["../../src/lib/md-cleanup.ts"],"names":[],"mappings":"AAwEA,UAAU,cAAc;IACtB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AA6cD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,EACf,kBAAkB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAC3C,MAAM,CAuCR;AAWD,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,cAAc,GACvB,MAAM,CAmBR"}

package/dist/lib/md-cleanup.js

@@ -22,6 +22,7 @@ const REGEX = {
     HEADING_STRICT: /^#{1,6}\s+/m,
     EMPTY_HEADING_LINE: /^#{1,6}[ \t\u00A0]*$/,
     ANCHOR_ONLY_HEADING: /^#{1,6}\s+\[[^\]]+\]\(#[^)]+\)\s*$/,
+    HEADING_TRAILING_PERMALINK: /^(#{1,6}\s+.+?)\s*\[(?:#|¶|§|¤|🔗)\]\(#[^)]+\)\s*$/gmu,
     FENCE_START: FENCE_PATTERN,
     LIST_MARKER: /^(?:[-*+])\s/m,
     TOC_LINK: /^- \[[^\]]+\]\(#[^)]+\)\s*$/,
@@ -36,7 +37,10 @@ const REGEX = {
     HEADING_CODE_BLOCK: /(^#{1,6}\s+\w+)```/gm,
     SPACING_LINK_FIX: /\]\(([^)]+)\)\[/g,
     SPACING_ADJ_COMBINED: /(?:\]\([^)]+\)|`[^`]+`)(?=[A-Za-z0-9])/g,
+    SPACING_CODE_PAD_BEFORE: /(\S)[ \t]{2,}(?=`[^`\n]+`)/g,
+    SPACING_CODE_PAD_AFTER: /(`[^`\n]+`)[ \t]{2,}(?=\S)/g,
     SPACING_CODE_DASH: /(`[^`]+`)\s*\\-\s*/g,
+    SPACING_ESCAPED_DASH: /(?<=[\w)\]`])\s*\\-\s*(?=[A-Za-z0-9([])/g,
     SPACING_ESCAPES: /\\([[\].])/g,
     SPACING_LIST_NUM_COMBINED: /^((?![-*+] |\d+\. |[ \t]).+)\n((?:[-*+]|\d+\.) )/gm,
     PUNCT_ONLY_LIST_ARTIFACT: /^(?:[-*+]|\d+\.)\s*(?:\\[-*+|/]|[-*+|/])(?:\s+(?:\\[-*+|/]|[-*+|/]))*\s*$/gm,
@@ -72,6 +76,14 @@ function hasFollowingContent(lines, startIndex) {
     }
     return false;
 }
+function findNextNonBlankLine(lines, startIndex) {
+    for (let i = startIndex + 1; i < Math.min(lines.length, startIndex + HAS_FOLLOWING_LOOKAHEAD); i++) {
+        const line = lines[i];
+        if (!isBlank(line))
+            return line?.trim();
+    }
+    return undefined;
+}
 function stripAnchorOnlyHeading(line) {
     return line.replace(/^(#{1,6})\s+\[([^\]]+)\]\(#[^)]+\)\s*$/, '$1 $2');
 }
@@ -191,6 +203,11 @@ function tryPromoteOrphan(lines, i, trimmed) {
     const isSpecialPrefix = SPECIAL_PREFIXES.test(trimmed);
     if (!isSpecialPrefix && !hasFollowingContent(lines, i))
         return null;
+    if (!isSpecialPrefix) {
+        const nextLine = findNextNonBlankLine(lines, i);
+        if (nextLine && REGEX.HEADING_MARKER.test(nextLine))
+            return null;
+    }
     return `${prefix}${trimmed}`;
 }
 function shouldSkipAsToc(lines, i, trimmed, removeToc, options) {
@@ -205,13 +222,16 @@ function shouldSkipAsToc(lines, i, trimmed, removeToc, options) {
     throwIfAborted(options?.signal, options?.url ?? '', 'markdown:cleanup:toc');
     return skipTocLines(lines, i + 1);
 }
-function normalizePreprocessLine(lines, i, trimmed, line) {
+function normalizePreprocessLine(lines, i, trimmed, line, options) {
     if (REGEX.EMPTY_HEADING_LINE.test(trimmed))
         return null;
     if (!REGEX.ANCHOR_ONLY_HEADING.test(trimmed))
         return line;
-    if (!hasFollowingContent(lines, i))
-        return null;
+    if (!hasFollowingContent(lines, i)) {
+        return options?.preserveEmptyHeadings
+            ? stripAnchorOnlyHeading(trimmed)
+            : null;
+    }
     return stripAnchorOnlyHeading(trimmed);
 }
 function maybeSkipTocBlock(lines, i, trimmed, options) {
@@ -235,7 +255,7 @@ function preprocessLines(lines, options) {
         if (currentLine === undefined)
             continue;
         const trimmed = currentLine.trim();
-        const normalizedLine = normalizePreprocessLine(lines, i, trimmed, currentLine);
+        const normalizedLine = normalizePreprocessLine(lines, i, trimmed, currentLine, options);
         if (normalizedLine === null)
             continue;
         const tocSkip = maybeSkipTocBlock(lines, i, trimmed, options);
@@ -269,21 +289,91 @@ function removeSkipLinks(text) {
 function normalizeInlineCodeTokens(text) {
     return text.replace(/`([^`\n]+)`/g, (match, inner) => {
         const trimmed = inner.trim();
-        if (trimmed === inner)
-            return match;
         if (!/[A-Za-z0-9]/.test(trimmed))
             return match;
         const parts = /^(\s*)(.*?)(\s*)$/.exec(inner);
         if (!parts)
             return match;
-        return `${parts[1] ?? ''}\`${parts[2] ?? ''}\`${parts[3] ?? ''}`;
+        const normalized = collapseQualifiedIdentifierSpacing(parts[2] ?? '');
+        if (trimmed === inner && normalized === inner)
+            return match;
+        return `${parts[1] ?? ''}\`${normalized}\`${parts[3] ?? ''}`;
     });
 }
+function collapseQualifiedIdentifierSpacing(text) {
+    let result = text;
+    for (let i = 0; i < PROPERTY_FIX_MAX_PASSES; i++) {
+        const next = result.replace(/\b([A-Za-z_$][\w$]*)\.\s+(?=[A-Za-z_$<])/g, '$1.');
+        if (next === result)
+            break;
+        result = next;
+    }
+    return result;
+}
+function normalizeMarkdownLinkText(text) {
+    const normalized = collapseQualifiedIdentifierSpacing(text.replace(/\\`/g, '`').replace(/\\</g, '<').replace(/\\>/g, '>'));
+    return normalized.replace(/</g, '\\<').replace(/>/g, '\\>');
+}
+function normalizeMarkdownLinkLabels(text) {
+    return text.replace(/\[([^\]]+)\]\(([^)]+)\)/g, (_match, linkText, url) => `[${normalizeMarkdownLinkText(linkText)}](${url})`);
+}
+function collapseInlineCodePadding(text) {
+    return text
+        .replace(/(\S)[ \t]{2,}(?=`[^`\n]+`)/g, '$1 ')
+        .replace(/(`[^`\n]+`)[ \t]{2,}(?=\S)/g, '$1 ');
+}
+function escapeAngleBracketsInMarkdownTables(text) {
+    return text.replace(/^(?!\|\s*[-: ]+\|)(\|.*\|)\s*$/gm, (line) => line
+        .replace(/<\/([A-Za-z][A-Za-z0-9-]*)>/g, '\\</$1\\>')
+        .replace(/<([A-Za-z][A-Za-z0-9-]*)>/g, '\\<$1\\>'));
+}
+function stripTrailingHeadingPermalinks(text) {
+    return text
+        .replace(REGEX.HEADING_TRAILING_PERMALINK, '$1')
+        .replace(/^(#{1,6})\s{2,}/gm, '$1 ')
+        .replace(/^(#{1,6}\s+.*?)[ \t]+$/gm, '$1');
+}
+function getHeadingInfo(line) {
+    const match = /^(#{1,6})\s+/.exec(line.trim());
+    if (!match)
+        return null;
+    return { level: match[1]?.length ?? 0 };
+}
+function removeEmptyHeadingSections(text) {
+    const lines = text.split('\n');
+    const kept = [];
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i] ?? '';
+        const heading = getHeadingInfo(line);
+        if (!heading) {
+            kept.push(line);
+            continue;
+        }
+        let nextIndex = i + 1;
+        while (nextIndex < lines.length && isBlank(lines[nextIndex])) {
+            nextIndex += 1;
+        }
+        const nextLine = lines[nextIndex];
+        if (nextLine === undefined) {
+            kept.push(line);
+            continue;
+        }
+        const nextHeading = getHeadingInfo(nextLine);
+        if (nextHeading && nextHeading.level <= heading.level) {
+            continue;
+        }
+        kept.push(line);
+    }
+    return kept.join('\n').replace(REGEX.DOUBLE_NEWLINE_REDUCER, '\n\n');
+}
 function normalizeMarkdownSpacing(text) {
     let result = text
-        .replace(REGEX.SPACING_LINK_FIX, ']($1)\n\n[')
+        .replace(REGEX.SPACING_LINK_FIX, ']($1) [')
         .replace(REGEX.SPACING_ADJ_COMBINED, '$& ')
+        .replace(REGEX.SPACING_CODE_PAD_BEFORE, '$1 ')
+        .replace(REGEX.SPACING_CODE_PAD_AFTER, '$1 ')
         .replace(REGEX.SPACING_CODE_DASH, '$1 - ')
+        .replace(REGEX.SPACING_ESCAPED_DASH, ' - ')
         .replace(REGEX.SPACING_ESCAPES, '$1')
         .replace(REGEX.SPACING_LIST_NUM_COMBINED, '$1\n\n$2')
         .replace(REGEX.PUNCT_ONLY_LIST_ARTIFACT, '')
@@ -292,9 +382,9 @@ function normalizeMarkdownSpacing(text) {
     result = result.replace(/([.!?:;])([A-Z])/g, '$1 $2');
     // Trim whitespace around token-like inline code spans.
     result = normalizeInlineCodeTokens(result);
-    // Unescape backticks inside markdown link text
-    result = result.replace(/\[([^\]]*\\`[^\]]*)\]\(([^)]+)\)/g, (_match, linkText, url) => `[${linkText.replace(/\\`/g, '`')}](${url})`);
-    result = result.replace(/\[([^\]]*<[^\]]*)\]\(([^)]+)\)/g, (_match, linkText, url) => `[${linkText.replace(/</g, '\\<').replace(/>/g, '\\>')}](${url})`);
+    result = collapseInlineCodePadding(result);
+    result = normalizeMarkdownLinkLabels(result);
+    result = escapeAngleBracketsInMarkdownTables(result);
     return normalizeNestedListIndentation(result);
 }
 function fixConcatenatedProperties(text) {
@@ -325,7 +415,8 @@ function applyGlobalRegexes(text, options) {
     checkAbort('markdown:cleanup:spacing');
     result = normalizeMarkdownSpacing(result);
     checkAbort('markdown:cleanup:properties');
-    return fixConcatenatedProperties(result);
+    result = fixConcatenatedProperties(result);
+    return stripTrailingHeadingPermalinks(result);
 }
 function normalizeNestedListIndentation(text) {
     return text.replace(REGEX.NESTED_LIST_INDENT, (match, spaces, marker) => {
@@ -386,6 +477,10 @@ export function cleanupMarkdownArtifacts(content, options) {
     if (!content)
         return '';
     throwIfAborted(options?.signal, options?.url ?? '', 'markdown:cleanup:begin');
-    const result = processFencedContent(content, (text) => processTextBuffer(text.split('\n'), options)).trim();
+    let result = processFencedContent(content, (text) => processTextBuffer(text.split('\n'), options)).trim();
+    if (!options?.preserveEmptyHeadings) {
+        throwIfAborted(options?.signal, options?.url ?? '', 'markdown:cleanup:empty-headings');
+        result = removeEmptyHeadingSections(result);
+    }
     return stripLeadingBreadcrumbNoise(result);
 }

package/dist/transform/transform.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAgDA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAqJD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAwYD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AA8KD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAuJD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAiED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AAuCD,iBAAS,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAc/D;AAED,iBAAS,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAYlE;AA6CD,iBAAS,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAcvD;AAED,eAAO,MAAM,mBAAmB;;;;CAItB,CAAC;AA2vBX,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAqCzB;AAaD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAkH1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}
+{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAgDA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAqJD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAwYD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AAgLD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAuJD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAiED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AA4DD,iBAAS,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAc/D;AAED,iBAAS,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAyBlE;AA6CD,iBAAS,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAcvD;AAED,eAAO,MAAM,mBAAmB;;;;CAItB,CAAC;AA8xBX,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAqCzB;AAaD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAkH1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}

package/dist/transform/transform.js

@@ -553,7 +553,9 @@ function translateHtmlToMarkdown(params) {
     throwIfAborted(signal, url, 'markdown:cleaned');
     const content = stageTracker.run(url, 'markdown:translate', () => translateHtmlFragmentToMarkdown(cleanedHtml));
     throwIfAborted(signal, url, 'markdown:translated');
-    const cleaned = cleanupMarkdownArtifacts(content, signal ? { signal, url } : { url });
+    const cleaned = cleanupMarkdownArtifacts(content, signal
+        ? { preserveEmptyHeadings: true, signal, url }
+        : { preserveEmptyHeadings: true, url });
     return url ? resolveRelativeUrls(cleaned, url, signal) : cleaned;
 }
 function appendMetadataFooter(content, metadata, url) {
@@ -815,6 +817,22 @@ const PRIMARY_HEADING_ROOT_SELECTORS = [
     '.entry-content',
     '[itemprop="text"]',
 ];
+function normalizeSyntheticTitleToken(value) {
+    return (value ?? '').replace(/\s+/g, ' ').trim().toLowerCase();
+}
+function shouldPreferPrimaryHeadingTitle(primaryHeading, title) {
+    const primary = normalizeSyntheticTitleToken(primaryHeading);
+    if (!primary)
+        return false;
+    const normalizedTitle = normalizeSyntheticTitleToken(title);
+    if (!normalizedTitle)
+        return true;
+    if (normalizedTitle === primary)
+        return true;
+    return normalizedTitle
+        .split(/\s*(?:[-|:•·]|–|—)\s*/u)
+        .some((part) => part === primary);
+}
 function findContentRoot(document) {
     for (const selector of CONTENT_ROOT_SELECTORS) {
         const element = document.querySelector(selector);
@@ -829,17 +847,31 @@ function findContentRoot(document) {
     return undefined;
 }
 function findPrimaryHeading(document) {
-    for (const selector of PRIMARY_HEADING_ROOT_SELECTORS) {
-        const root = document.querySelector(selector);
-        if (!root)
-            continue;
-        const heading = root.querySelector('h1, h2');
+    for (const headingSelector of ['[data-title="true"]', 'h1']) {
+        const heading = document.querySelector(headingSelector);
         if (!heading)
             continue;
         const text = heading.textContent.trim();
         if (text)
             return text;
     }
+    for (const selector of PRIMARY_HEADING_ROOT_SELECTORS) {
+        const root = document.querySelector(selector);
+        if (!root)
+            continue;
+        for (const headingSelector of [
+            '[data-title="true"]',
+            'h1',
+            'h2',
+        ]) {
+            const heading = root.querySelector(headingSelector);
+            if (!heading)
+                continue;
+            const text = heading.textContent.trim();
+            if (text)
+                return text;
+        }
+    }
     return undefined;
 }
 function countMatchingElements(root, selector) {
@@ -971,39 +1003,56 @@ function shouldUseArticleContent(article, document) {
 function buildContentSource(params) {
     const { html, url, article, extractedMeta, includeMetadata, useArticleContent, document, truncated, signal, } = params;
     const metadata = createContentMetadataBlock(url, article, extractedMeta, useArticleContent, includeMetadata);
+    const preparedDocument = document;
+    let primaryHeading = document
+        ? TransformHeuristics.findPrimaryHeading(document)
+        : undefined;
+    if (preparedDocument) {
+        prepareDocumentForMarkdown(preparedDocument, url, signal);
+        primaryHeading =
+            TransformHeuristics.findPrimaryHeading(preparedDocument) ??
+                primaryHeading;
+    }
     const base = {
         favicon: extractedMeta.favicon,
         metadata,
         extractedMetadata: extractedMeta,
         truncated,
-        primaryHeading: document
-            ? TransformHeuristics.findPrimaryHeading(document)
-            : undefined,
+        primaryHeading,
         originalHtml: html,
     };
     if (useArticleContent && article) {
         const { document: articleDoc } = parseHTML(`<!DOCTYPE html><html><body>${article.content}</body></html>`);
         prepareDocumentForMarkdown(articleDoc, url, signal);
-        const preferPrimaryHeading = TransformHeuristics.isGithubRepositoryRootUrl(url);
+        const articleTitle = article.title !== undefined
+            ? normalizeDocumentTitle(article.title, url)
+            : extractedMeta.title;
+        const preferPrimaryHeading = TransformHeuristics.isGithubRepositoryRootUrl(url) ||
+            shouldPreferPrimaryHeadingTitle(base.primaryHeading, articleTitle);
+        const resolvedTitle = (preferPrimaryHeading ? base.primaryHeading : undefined) ?? articleTitle;
         return {
             ...base,
             sourceHtml: articleDoc.body.innerHTML,
-            title: (preferPrimaryHeading ? base.primaryHeading : undefined) ??
-                (article.title !== undefined
-                    ? normalizeDocumentTitle(article.title, url)
-                    : undefined),
+            title: resolvedTitle,
+            suppressSyntheticFavicon: normalizeSyntheticTitleToken(resolvedTitle) ===
+                normalizeSyntheticTitleToken(base.primaryHeading),
             skipNoiseRemoval: true,
         };
     }
     if (document) {
-        prepareDocumentForMarkdown(document, url, signal);
-        const contentRoot = TransformHeuristics.findContentRoot(document);
+        const resolvedDocument = preparedDocument ?? document;
+        const contentRoot = TransformHeuristics.findContentRoot(resolvedDocument);
+        const preferPrimaryHeading = shouldPreferPrimaryHeadingTitle(base.primaryHeading, extractedMeta.title);
+        const resolvedTitle = (preferPrimaryHeading ? base.primaryHeading : undefined) ??
+            extractedMeta.title;
         return {
             ...base,
-            sourceHtml: contentRoot ?? serializeDocumentForMarkdown(document, html),
-            title: extractedMeta.title,
+            sourceHtml: contentRoot ?? serializeDocumentForMarkdown(resolvedDocument, html),
+            title: resolvedTitle,
+            suppressSyntheticFavicon: normalizeSyntheticTitleToken(resolvedTitle) ===
+                normalizeSyntheticTitleToken(base.primaryHeading),
             skipNoiseRemoval: true,
-            document,
+            document: resolvedDocument,
         };
     }
     return {
@@ -1326,8 +1375,8 @@ function maybeStripGithubPrimaryHeading(markdown, context, url) {
         return markdown;
     return stripLeadingHeading(markdown, context.primaryHeading ?? '');
 }
-function buildSyntheticTitlePrefix(url, favicon) {
-    if (!favicon)
+function buildSyntheticTitlePrefix(url, favicon, suppressFavicon) {
+    if (!favicon || suppressFavicon)
         return ' ';
     let alt = '';
     try {
@@ -1342,7 +1391,7 @@ function maybePrependSyntheticTitle(markdown, context, url) {
     if (!context.title || /^(#{1,6})\s/.test(markdown.trimStart())) {
         return markdown;
     }
-    return `#${buildSyntheticTitlePrefix(url, context.favicon)}${context.title}\n\n${markdown}`;
+    return `#${buildSyntheticTitlePrefix(url, context.favicon, context.suppressSyntheticFavicon)}${context.title}\n\n${markdown}`;
 }
 function buildMarkdownFromContext(context, url, signal) {
     let content = stageTracker.run(url, 'transform:markdown', () => htmlToMarkdown(context.sourceHtml, context.metadata, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/fetch-url-mcp",
-  "version": "1.9.4",
+  "version": "1.9.5",
   "mcpName": "io.github.j0hanz/fetch-url-mcp",
   "description": "A web content fetcher MCP server that converts HTML to clean, AI and human readable markdown.",
   "type": "module",
@@ -75,7 +75,7 @@
     "linkedom": "^0.18.12",
     "node-html-markdown": "^2.0.0",
     "ts-morph": "^27.0.2",
-    "undici": "^7.24.1",
+    "undici": "^7.24.3",
     "zod": "^4.3.6"
   },
   "devDependencies": {