@j0hanz/fetch-url-mcp 1.9.0 → 1.9.2

package/dist/lib/content.js

@@ -2,11 +2,38 @@ import { parseHTML } from 'linkedom';
 import {} from '../transform/types.js';
 import { config, logDebug } from './core.js';
 import { throwIfAborted } from './utils.js';
+// ASCII char codes used in hot-path charCodeAt comparisons
+const ASCII_SPACE = 32;
+const ASCII_TAB = 9;
+const ASCII_EXCLAMATION = 33;
+const ASCII_HASH = 35;
+const ASCII_ASTERISK = 42;
+const ASCII_PLUS = 43;
+const ASCII_DASH = 45;
+const ASCII_PERIOD = 46;
+const ASCII_DIGIT_0 = 48;
+const ASCII_DIGIT_9 = 57;
+const ASCII_LT = 60;
+const ASCII_QUESTION = 63;
+const ASCII_UPPER_A = 65;
+const ASCII_UPPER_Z = 90;
+const ASCII_BRACKET_OPEN = 91;
+const ASCII_LOWER_A = 97;
+const ASCII_LOWER_Z = 122;
+const ASCII_UNDERSCORE = 95;
+const HTML_TAG_DENSITY_LIMIT = 5;
+const TITLE_MIN_WORDS = 2;
+const TITLE_MAX_WORDS = 6;
+const TITLE_MIN_CAPITALIZED = 2;
+const PROPERTY_FIX_MAX_PASSES = 3;
+const BODY_SCAN_LIMIT = 5000;
+const HAS_FOLLOWING_LOOKAHEAD = 50;
 const NOISE_SCAN_LIMIT = 50_000;
 const MIN_BODY_CONTENT_LENGTH = 100;
 const DIALOG_MIN_CHARS_FOR_PRESERVATION = 500;
 const NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION = 500;
 const ABORT_CHECK_INTERVAL = 500;
+const NODE_FILTER_SHOW_TEXT = 4;
 const HTML_DOCUMENT_MARKERS = /<\s*(?:!doctype|html|head|body)\b/i;
 const HTML_FRAGMENT_MARKERS = /<\s*(?:article|main|section|div|nav|footer|header|aside|table|ul|ol)\b/i;
 const NOISE_PATTERNS = [
@@ -89,6 +116,7 @@ const PROMO_TOKENS_BY_CATEGORY = {
     newsletters: ['newsletter', 'subscribe'],
     'social-share': ['share', 'social'],
 };
+// Noise selector configurations
 const BASE_NOISE_SELECTORS = {
     navFooter: 'nav,footer,header[class*="site"],header[class*="nav"],header[class*="menu"],[role="banner"],[role="navigation"]',
     cookieBanners: '[role="dialog"]',
@@ -96,7 +124,7 @@ const BASE_NOISE_SELECTORS = {
 };
 const NO_MATCH_REGEX = /a^/i;
 let cachedContext;
-let lastConfigRef;
+let lastContextKey;
 function escapeRegexLiteral(value) {
     return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }
@@ -136,7 +164,16 @@ function getPromoMatchers(currentConfig, flags) {
 }
 function getContext() {
     const currentConfig = config.noiseRemoval;
-    if (cachedContext !== undefined && lastConfigRef === currentConfig)
+    const contextKey = JSON.stringify({
+        locale: config.i18n.locale,
+        enabledCategories: currentConfig.enabledCategories,
+        extraTokens: currentConfig.extraTokens,
+        extraSelectors: currentConfig.extraSelectors,
+        aggressiveMode: currentConfig.aggressiveMode,
+        preserveSvgCanvas: currentConfig.preserveSvgCanvas,
+        weights: currentConfig.weights,
+    });
+    if (cachedContext !== undefined && lastContextKey === contextKey)
         return cachedContext;
     const enabled = new Set(currentConfig.enabledCategories
         .map((c) => {
@@ -188,14 +225,15 @@ function getContext() {
         baseSelector,
         candidateSelector,
     };
-    lastConfigRef = currentConfig;
+    lastContextKey = contextKey;
     return cachedContext;
 }
 function isInteractive(element, role) {
     if (role && INTERACTIVE_CONTENT_ROLES.has(role))
         return true;
+    const tag = element.tagName.toLowerCase();
     const ds = element.getAttribute('data-state');
-    if (ds === 'inactive' || ds === 'closed')
+    if ((ds === 'inactive' || ds === 'closed') && !BASE_STRUCTURAL_TAGS.has(tag))
         return true;
     const dataOrientation = element.getAttribute('data-orientation');
     if (dataOrientation === 'horizontal' || dataOrientation === 'vertical')
@@ -215,6 +253,19 @@ function isWithinPrimaryContent(element) {
     }
     return false;
 }
+const ASIDE_NAV_LINK_DENSITY_THRESHOLD = 0.5;
+const ASIDE_NAV_MIN_LINKS = 10;
+function isNavigationAside(element) {
+    if (element.querySelector('nav'))
+        return true;
+    const links = element.querySelectorAll('a[href]');
+    if (links.length < ASIDE_NAV_MIN_LINKS)
+        return false;
+    const textLen = (element.textContent || '').trim().length;
+    if (textLen === 0)
+        return true;
+    return links.length / (textLen / 100) >= ASIDE_NAV_LINK_DENSITY_THRESHOLD;
+}
 function shouldPreserve(element, tagName) {
     // Check Dialog
     const role = element.getAttribute('role');
@@ -233,6 +284,12 @@ function shouldPreserve(element, tagName) {
         return ((element.textContent || '').trim().length >=
             NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION);
     }
+    // Check Aside — preserve only if it looks like article content, not navigation
+    if (tagName === 'aside') {
+        if (!isWithinPrimaryContent(element))
+            return false;
+        return !isNavigationAside(element);
+    }
     return false;
 }
 function removeNodes(nodes) {
@@ -243,142 +300,120 @@ function removeNodes(nodes) {
         }
     }
 }
-function scoreNavFooter(meta, weights) {
+const HIDDEN_STYLE_REGEX = /\b(?:display\s*:\s*none|visibility\s*:\s*hidden)\b/i;
+function calculateNavFooterScore(tagName, className, id, role, weights) {
     let score = 0;
-    if (ALWAYS_NOISE_TAGS.has(meta.tagName))
+    if (ALWAYS_NOISE_TAGS.has(tagName))
         score += weights.structural;
-    // Header Boilerplate
-    if (meta.tagName === 'header') {
-        if ((meta.role && NAVIGATION_ROLES.has(meta.role)) ||
-            HEADER_NOISE_PATTERN.test(`${meta.className} ${meta.id}`)) {
+    if (tagName === 'header') {
+        if ((role && NAVIGATION_ROLES.has(role)) ||
+            HEADER_NOISE_PATTERN.test(`${className} ${id}`)) {
             score += weights.structural;
         }
     }
-    // Role Noise
-    if (meta.role && NAVIGATION_ROLES.has(meta.role)) {
-        if (meta.tagName !== 'aside' || meta.role !== 'complementary') {
+    if (tagName === 'aside') {
+        score += weights.structural;
+    }
+    if (role && NAVIGATION_ROLES.has(role)) {
+        if (tagName !== 'aside' || role !== 'complementary') {
             score += weights.structural;
         }
     }
     return score;
 }
-function extractElementMetadata(element) {
+function calculatePromoScore(element, className, id, context) {
+    if (!context.promoEnabled)
+        return 0;
+    const aggTest = context.promoMatchers.aggressive.test(className) ||
+        context.promoMatchers.aggressive.test(id);
+    const isAggressiveMatch = aggTest && !isWithinPrimaryContent(element);
+    const isBaseMatch = !aggTest &&
+        (context.promoMatchers.base.test(className) ||
+            context.promoMatchers.base.test(id));
+    return isAggressiveMatch || isBaseMatch ? context.weights.promo : 0;
+}
+function isNoiseElement(element, context) {
     const tagName = element.tagName.toLowerCase();
     const className = element.getAttribute('class') ?? '';
     const id = element.getAttribute('id') ?? '';
     const role = element.getAttribute('role');
     const style = element.getAttribute('style');
-    const _isInteractive = isInteractive(element, role);
-    const isHidden = element.hasAttribute('hidden') ||
+    const elIsInteractive = isInteractive(element, role);
+    const elIsHidden = element.hasAttribute('hidden') ||
         element.getAttribute('aria-hidden') === 'true' ||
-        (style !== null &&
-            /\b(?:display\s*:\s*none|visibility\s*:\s*hidden)\b/i.test(style));
-    return {
-        tagName,
-        className,
-        id,
-        role,
-        style,
-        isInteractive: _isInteractive,
-        isHidden,
-    };
-}
-function isNoiseElement(element, context) {
-    const meta = extractElementMetadata(element);
+        (style !== null && HIDDEN_STYLE_REGEX.test(style));
     let score = 0;
     const { weights } = context;
     // Structural
-    if (context.structuralTags.has(meta.tagName) && !meta.isInteractive) {
+    if (context.structuralTags.has(tagName) && !elIsInteractive) {
         score += weights.structural;
     }
     // Nav/Footer Scoring
     if (context.flags.navFooter) {
-        score += scoreNavFooter(meta, weights);
+        score += calculateNavFooterScore(tagName, className, id, role, weights);
     }
     // Hidden
-    if (meta.isHidden && !meta.isInteractive) {
+    if (elIsHidden && !elIsInteractive) {
         score += weights.hidden;
     }
     // Sticky/Fixed
-    if (FIXED_OR_HIGH_Z_PATTERN.test(meta.className)) {
+    if (FIXED_OR_HIGH_Z_PATTERN.test(className)) {
         score += weights.stickyFixed;
     }
     // Promo
-    if (context.promoEnabled) {
-        const aggTest = context.promoMatchers.aggressive.test(meta.className) ||
-            context.promoMatchers.aggressive.test(meta.id);
-        const isAggressiveMatch = aggTest && !isWithinPrimaryContent(element);
-        const isBaseMatch = !aggTest &&
-            (context.promoMatchers.base.test(meta.className) ||
-                context.promoMatchers.base.test(meta.id));
-        if (isAggressiveMatch || isBaseMatch) {
-            score += weights.promo;
-        }
-    }
+    score += calculatePromoScore(element, className, id, context);
     return score >= weights.threshold;
 }
-function cleanHeadingWrapperDivs(h) {
-    const divs = h.querySelectorAll('div');
-    for (let j = divs.length - 1; j >= 0; j--) {
-        const d = divs[j];
-        if (!d?.parentNode)
-            continue;
-        const cls = d.getAttribute('class') ?? '';
-        const stl = d.getAttribute('style') ?? '';
-        if (cls.includes('absolute') ||
-            stl.includes('position') ||
-            d.getAttribute('tabindex') === '-1') {
-            d.remove();
-        }
-    }
-}
-function cleanHeadingAnchors(h) {
-    const anchors = h.querySelectorAll('a');
-    for (let j = anchors.length - 1; j >= 0; j--) {
-        const a = anchors[j];
-        if (!a?.parentNode)
-            continue;
-        const href = a.getAttribute('href') ?? '';
-        const txt = (a.textContent || '').replace(/[\u200B\s]/g, '');
-        if (href.startsWith('#') && txt.length === 0) {
-            a.remove();
-        }
-    }
-}
-function cleanHeadingZeroWidth(h, document) {
-    const walker = document.createTreeWalker(h, 4); // SHOW_TEXT
-    let node;
-    while ((node = walker.nextNode())) {
-        if (node.textContent?.includes('\u200B')) {
-            node.textContent = node.textContent.replace(/\u200B/g, '');
-        }
-    }
-}
 function cleanHeadings(document) {
-    // Clean Heading Anchors
     const headings = document.querySelectorAll('h1,h2,h3,h4,h5,h6');
     for (const h of headings) {
         if (!h.parentNode)
             continue;
-        cleanHeadingWrapperDivs(h);
-        cleanHeadingAnchors(h);
-        cleanHeadingZeroWidth(h, document);
+        // Remove absolute/positioned wrapper divs
+        const divs = h.querySelectorAll('div');
+        for (let j = divs.length - 1; j >= 0; j--) {
+            const d = divs[j];
+            if (!d?.parentNode)
+                continue;
+            const cls = d.getAttribute('class') ?? '';
+            const stl = d.getAttribute('style') ?? '';
+            if (cls.includes('absolute') ||
+                stl.includes('position') ||
+                d.getAttribute('tabindex') === '-1') {
+                d.remove();
+            }
+        }
+        // Remove empty hash-link anchors
+        const anchors = h.querySelectorAll('a');
+        for (let j = anchors.length - 1; j >= 0; j--) {
+            const a = anchors[j];
+            if (!a?.parentNode)
+                continue;
+            const href = a.getAttribute('href') ?? '';
+            const txt = (a.textContent || '').replace(/[\u200B\s]/g, '');
+            if (href.startsWith('#') && txt.length === 0) {
+                a.remove();
+            }
+        }
+        // Strip zero-width spaces from text nodes
+        const walker = document.createTreeWalker(h, NODE_FILTER_SHOW_TEXT);
+        let node;
+        while ((node = walker.nextNode())) {
+            if (node.textContent?.includes('\u200B')) {
+                node.textContent = node.textContent.replace(/\u200B/g, '');
+            }
+        }
     }
 }
 function stripNoise(document, context, signal) {
     cleanHeadings(document);
-    // Remove Base & Extra
+    // Structural Removal
     const { baseSelector, extraSelectors } = context;
-    // Base
-    const baseNodes = document.querySelectorAll(baseSelector);
-    removeNodes(baseNodes);
-    // Extra
+    removeNodes(document.querySelectorAll(baseSelector));
     if (extraSelectors.length > 0) {
-        const combinedExtra = extraSelectors.join(',');
-        const extraNodes = document.querySelectorAll(combinedExtra);
-        removeNodes(extraNodes);
+        removeNodes(document.querySelectorAll(extraSelectors.join(',')));
     }
-    // Candidates
+    // Candidates (conditional removal)
     const candidates = document.querySelectorAll(context.candidateSelector);
     for (let i = candidates.length - 1; i >= 0; i--) {
         if (i % ABORT_CHECK_INTERVAL === 0 && signal?.aborted) {
@@ -469,6 +504,29 @@ function mayContainNoise(html) {
         : `${html.substring(0, NOISE_SCAN_LIMIT)}\n${html.substring(html.length - NOISE_SCAN_LIMIT)}`;
     return NOISE_PATTERNS.some((re) => re.test(sample));
 }
+function stripTabTriggers(document) {
+    const tabs = document.querySelectorAll('button[role="tab"]');
+    for (let i = tabs.length - 1; i >= 0; i--) {
+        tabs[i]?.remove();
+    }
+}
+function escapeTableCellPipes(document) {
+    const codes = document.querySelectorAll('td code, th code');
+    for (const code of codes) {
+        if (code.textContent.includes('|')) {
+            code.textContent = code.textContent.replace(/\|/g, '\\|');
+        }
+    }
+}
+function separateAdjacentInlineElements(document) {
+    const badges = document.querySelectorAll('span.chakra-badge, [data-scope="badge"], [class*="badge"]');
+    for (const badge of badges) {
+        const next = badge.nextSibling;
+        if (next?.nodeType === 1) {
+            badge.after(document.createTextNode(' '));
+        }
+    }
+}
 export function prepareDocumentForMarkdown(document, baseUrl, signal) {
     const context = getContext();
     if (config.noiseRemoval.debug) {
@@ -477,9 +535,37 @@ export function prepareDocumentForMarkdown(document, baseUrl, signal) {
         });
     }
     stripNoise(document, context, signal);
+    stripTabTriggers(document);
+    separateAdjacentInlineElements(document);
+    flattenTableCellBreaks(document);
+    escapeTableCellPipes(document);
+    normalizeTableStructure(document);
     if (baseUrl)
         resolveUrls(document, baseUrl);
 }
+// Some sites put tbody/thead/tfoot inside td/th, which breaks markdown tables.
+function normalizeTableStructure(document) {
+    for (const table of document.querySelectorAll('table')) {
+        for (const cell of table.querySelectorAll('th, td')) {
+            for (const tag of ['tbody', 'thead', 'tfoot']) {
+                let nested = cell.querySelector(tag);
+                while (nested) {
+                    table.appendChild(nested);
+                    nested = cell.querySelector(tag);
+                }
+            }
+        }
+    }
+}
+function flattenTableCellBreaks(document) {
+    const cells = document.querySelectorAll('td, th');
+    for (const cell of cells) {
+        const brs = cell.querySelectorAll('br');
+        for (const br of brs) {
+            br.replaceWith(' ');
+        }
+    }
+}
 export function removeNoiseFromHtml(html, document, baseUrl, signal) {
     const shouldParse = isFullDocumentHtml(html) ||
         mayContainNoise(html) ||
@@ -495,6 +581,8 @@ export function removeNoiseFromHtml(html, document, baseUrl, signal) {
         return html;
     }
 }
+// endregion
+// region Language Detection
 class DetectionContext {
     code;
     _lower;
@@ -574,10 +662,10 @@ const CSS_PROPERTY_REGEX = /^\s*[a-z][\w-]*\s*:/;
 function containsJsxTag(code) {
     const len = code.length;
     for (let i = 0; i < len - 1; i++) {
-        if (code.charCodeAt(i) === 60 /* < */) {
+        if (code.charCodeAt(i) === ASCII_LT) {
             const next = code.charCodeAt(i + 1);
-            if (next >= 65 && next <= 90)
-                return true; // A-Z
+            if (next >= ASCII_UPPER_A && next <= ASCII_UPPER_Z)
+                return true;
         }
     }
     return false;
@@ -636,140 +724,100 @@ function detectYamlStructure(lines) {
         if (colonIdx <= 0)
             continue;
         const after = trimmed.charCodeAt(colonIdx + 1);
-        // space (32) or tab (9)
-        if (after === 32 || after === 9)
+        if (after === ASCII_SPACE || after === ASCII_TAB)
+            return true;
+    }
+    return false;
+}
+function matchRust(ctx) {
+    if (ctx.lower.includes('let mut'))
+        return true;
+    if (RUST_REGEX.test(ctx.lower))
+        return true;
+    return ctx.lower.includes('use ') && ctx.lower.includes('::');
+}
+function matchGo(ctx) {
+    if (ctx.lower.includes('import "'))
+        return true;
+    return /\b(?:package|func)\b/.test(ctx.lower);
+}
+function matchJsx(ctx) {
+    const l = ctx.lower;
+    if (l.includes('classname=') ||
+        l.includes('jsx:') ||
+        l.includes("from 'react'") ||
+        l.includes('from "react"')) {
+        return true;
+    }
+    return containsJsxTag(ctx.code);
+}
+function matchTypeScript(ctx) {
+    if (/\b(?:interface|type)\b/.test(ctx.lower))
+        return true;
+    const l = ctx.lower;
+    for (const hint of TYPESCRIPT_HINTS) {
+        if (l.includes(hint))
+            return true;
+    }
+    return false;
+}
+function matchSql(ctx) {
+    return /\b(?:select|insert|update|delete|create|alter|drop)\b/.test(ctx.lower);
+}
+function hasJsSignals(lowerCode) {
+    return (JS_SIGNAL_REGEX.test(lowerCode) ||
+        lowerCode.includes('{') ||
+        lowerCode.includes("from '"));
+}
+function matchPython(ctx) {
+    const l = ctx.lower;
+    if (l.includes('print(') || l.includes('__name__'))
+        return true;
+    if (l.includes('self.') || l.includes('elif '))
+        return true;
+    // Check for Python's None/True/False using original case (they are capitalized in Python)
+    if (ctx.code.includes('None') ||
+        ctx.code.includes('True') ||
+        ctx.code.includes('False')) {
+        return true;
+    }
+    if (PYTHON_UNIQUE_REGEX.test(l))
+        return true;
+    // Shared keywords (import, from, class) — only match if no JS signals present
+    if (/\b(?:import|from|class)\b/.test(l) && !hasJsSignals(l)) {
+        return true;
+    }
+    return false;
+}
+function matchHtml(ctx) {
+    const l = ctx.lower;
+    for (const tag of HTML_TAGS) {
+        if (l.includes(tag))
             return true;
     }
     return false;
 }
+// Pre-sorted by weight descending — first match wins in detectLanguageFromCode
 const LANGUAGES = [
-    {
-        lang: 'rust',
-        weight: 25,
-        match: (ctx) => {
-            if (ctx.lower.includes('let mut'))
-                return true;
-            if (RUST_REGEX.test(ctx.lower))
-                return true;
-            return ctx.lower.includes('use ') && ctx.lower.includes('::');
-        },
-    },
-    {
-        lang: 'go',
-        weight: 22,
-        match: (ctx) => {
-            if (ctx.lower.includes('import "'))
-                return true;
-            return /\b(?:package|func)\b/.test(ctx.lower);
-        },
-    },
-    {
-        lang: 'jsx',
-        weight: 22,
-        match: (ctx) => {
-            const l = ctx.lower;
-            if (l.includes('classname=') ||
-                l.includes('jsx:') ||
-                l.includes("from 'react'") ||
-                l.includes('from "react"')) {
-                return true;
-            }
-            return containsJsxTag(ctx.code);
-        },
-    },
-    {
-        lang: 'typescript',
-        weight: 20,
-        match: (ctx) => {
-            if (/\b(?:interface|type)\b/.test(ctx.lower))
-                return true;
-            const l = ctx.lower;
-            for (const hint of TYPESCRIPT_HINTS) {
-                if (l.includes(hint))
-                    return true;
-            }
-            return false;
-        },
-    },
-    {
-        lang: 'sql',
-        weight: 20,
-        match: (ctx) => {
-            const l = ctx.lower;
-            return /\b(?:select|insert|update|delete|create|alter|drop)\b/.test(l);
-        },
-    },
-    {
-        lang: 'python',
-        weight: 18,
-        match: (ctx) => {
-            const l = ctx.lower;
-            if (l.includes('print(') || l.includes('__name__'))
-                return true;
-            if (l.includes('self.') || l.includes('elif '))
-                return true;
-            // Check for Python's None/True/False using original case (they are capitalized in Python)
-            if (ctx.code.includes('None') ||
-                ctx.code.includes('True') ||
-                ctx.code.includes('False')) {
-                return true;
-            }
-            // Python-unique keywords that JS doesn't have
-            if (PYTHON_UNIQUE_REGEX.test(l))
-                return true;
-            // Shared keywords (import, from, class) — only match if no JS signals present
-            if (/\b(?:import|from|class)\b/.test(l) &&
-                !JS_SIGNAL_REGEX.test(l) &&
-                !l.includes('{') &&
-                !l.includes("from '")) {
-                return true;
-            }
-            return false;
-        },
-    },
+    { lang: 'rust', weight: 25, match: matchRust },
+    { lang: 'go', weight: 22, match: matchGo },
+    { lang: 'jsx', weight: 22, match: matchJsx },
+    { lang: 'typescript', weight: 20, match: matchTypeScript },
+    { lang: 'sql', weight: 20, match: matchSql },
+    { lang: 'python', weight: 18, match: matchPython },
     {
         lang: 'css',
         weight: 18,
-        match: (ctx) => {
-            if (CSS_REGEX.test(ctx.lower))
-                return true;
-            return detectCssStructure(ctx.lines);
-        },
-    },
-    {
-        lang: 'bash',
-        weight: 15,
-        match: (ctx) => detectBashIndicators(ctx.lines),
-    },
-    {
-        lang: 'yaml',
-        weight: 15,
-        match: (ctx) => detectYamlStructure(ctx.lines),
-    },
-    {
-        lang: 'javascript',
-        weight: 15,
-        match: (ctx) => JS_REGEX.test(ctx.lower),
-    },
-    {
-        lang: 'html',
-        weight: 12,
-        match: (ctx) => {
-            const l = ctx.lower;
-            for (const tag of HTML_TAGS) {
-                if (l.includes(tag))
-                    return true;
-            }
-            return false;
-        },
+        match: (ctx) => CSS_REGEX.test(ctx.lower) || detectCssStructure(ctx.lines),
     },
+    { lang: 'bash', weight: 15, match: (ctx) => detectBashIndicators(ctx.lines) },
+    { lang: 'yaml', weight: 15, match: (ctx) => detectYamlStructure(ctx.lines) },
+    { lang: 'javascript', weight: 15, match: (ctx) => JS_REGEX.test(ctx.lower) },
+    { lang: 'html', weight: 12, match: matchHtml },
     {
         lang: 'json',
         weight: 10,
-        match: (ctx) => {
-            const s = ctx.trimmedStart;
-            return s.startsWith('{') || s.startsWith('[');
-        },
+        match: (ctx) => ctx.trimmedStart.startsWith('{') || ctx.trimmedStart.startsWith('['),
     },
 ];
 function extractLanguageFromClassName(className) {
@@ -805,11 +853,10 @@ function resolveLanguageFromDataAttribute(dataLang) {
     // Check if \w+
     for (let i = 0; i < trimmed.length; i++) {
         const c = trimmed.charCodeAt(i);
-        // valid: A-Z, a-z, 0-9, _
-        const isUpper = c >= 65 && c <= 90;
-        const isLower = c >= 97 && c <= 122;
-        const isDigit = c >= 48 && c <= 57;
-        const isUnder = c === 95;
+        const isUpper = c >= ASCII_UPPER_A && c <= ASCII_UPPER_Z;
+        const isLower = c >= ASCII_LOWER_A && c <= ASCII_LOWER_Z;
+        const isDigit = c >= ASCII_DIGIT_0 && c <= ASCII_DIGIT_9;
+        const isUnder = c === ASCII_UNDERSCORE;
         if (!isUpper && !isLower && !isDigit && !isUnder) {
             return undefined;
         }
@@ -826,7 +873,7 @@ export function detectLanguageFromCode(code) {
     // Fast path for empty/whitespace only
     let empty = true;
     for (let i = 0; i < code.length; i++) {
-        if (code.charCodeAt(i) > 32) {
+        if (code.charCodeAt(i) > ASCII_SPACE) {
             empty = false;
             break;
         }
@@ -834,20 +881,15 @@ export function detectLanguageFromCode(code) {
     if (empty)
         return undefined;
     const ctx = new DetectionContext(code);
-    let bestLang;
-    let bestScore = -1;
+    // LANGUAGES is pre-sorted by weight descending — first match is highest confidence
     for (const def of LANGUAGES) {
-        if (def.match(ctx)) {
-            if (def.weight > bestScore) {
-                bestScore = def.weight;
-                bestLang = def.lang;
-                if (bestScore >= 25)
-                    break;
-            }
-        }
+        if (def.match(ctx))
+            return def.lang;
     }
-    return bestLang;
+    return undefined;
 }
+// endregion
+// region Markdown Cleanup
 const MAX_LINE_LENGTH = 80;
 const REGEX = {
     HEADING_MARKER: /^#{1,6}\s/m,
@@ -900,7 +942,7 @@ function isBlank(line) {
 }
 function hasFollowingContent(lines, startIndex) {
     // Optimization: Bound lookahead to avoid checking too many lines in huge files
-    for (let i = startIndex + 1; i < Math.min(lines.length, startIndex + 50); i++) {
+    for (let i = startIndex + 1; i < Math.min(lines.length, startIndex + HAS_FOLLOWING_LOOKAHEAD); i++) {
         if (!isBlank(lines[i]))
             return true;
     }
@@ -919,7 +961,7 @@ function isTitleCaseOrKeyword(trimmed) {
     // Split limited number of words
     const words = trimmed.split(/\s+/);
     const len = words.length;
-    if (len < 2 || len > 6)
+    if (len < TITLE_MIN_WORDS || len > TITLE_MAX_WORDS)
         return false;
     let capitalizedCount = 0;
     for (let i = 0; i < len; i++) {
@@ -932,20 +974,19 @@ function isTitleCaseOrKeyword(trimmed) {
         else if (!/^(?:and|or|the|of|in|for|to|a)$/i.test(w))
             return false;
     }
-    return capitalizedCount >= 2;
+    return capitalizedCount >= TITLE_MIN_CAPITALIZED;
 }
 function getHeadingPrefix(trimmed) {
     if (trimmed.length > MAX_LINE_LENGTH)
         return null;
     // Fast path: Check common markdown markers first
     const firstChar = trimmed.charCodeAt(0);
-    // # (35), - (45), * (42), + (43), digit (48-57), [ (91)
-    if (firstChar === 35 ||
-        firstChar === 45 ||
-        firstChar === 42 ||
-        firstChar === 43 ||
-        firstChar === 91 ||
-        (firstChar >= 48 && firstChar <= 57)) {
+    if (firstChar === ASCII_HASH ||
+        firstChar === ASCII_DASH ||
+        firstChar === ASCII_ASTERISK ||
+        firstChar === ASCII_PLUS ||
+        firstChar === ASCII_BRACKET_OPEN ||
+        (firstChar >= ASCII_DIGIT_0 && firstChar <= ASCII_DIGIT_9)) {
         if (REGEX.HEADING_MARKER.test(trimmed) ||
             REGEX.LIST_MARKER.test(trimmed) ||
             /^\d+\.\s/.test(trimmed) ||
@@ -957,8 +998,9 @@ function getHeadingPrefix(trimmed) {
         return /^example:\s/i.test(trimmed) ? '### ' : '## ';
     }
     const lastChar = trimmed.charCodeAt(trimmed.length - 1);
-    // . (46), ! (33), ? (63)
-    if (lastChar === 46 || lastChar === 33 || lastChar === 63)
+    if (lastChar === ASCII_PERIOD ||
+        lastChar === ASCII_EXCLAMATION ||
+        lastChar === ASCII_QUESTION)
         return null;
     return isTitleCaseOrKeyword(trimmed) ? '## ' : null;
 }
@@ -1073,48 +1115,63 @@ function processTextBuffer(lines, options) {
     const text = preprocessLines(lines, options);
     return applyGlobalRegexes(text, options);
 }
-function applyGlobalRegexes(text, options) {
+function removeTypeDocArtifacts(text) {
+    const filtered = text
+        .split('\n')
+        .filter((line) => !isTypeDocArtifactLine(line))
+        .join('\n');
+    return filtered.replace(REGEX.TYPEDOC_COMMENT, (match) => match.startsWith('`') ? match : '');
+}
+function removeSkipLinks(text) {
+    return text
+        .replace(REGEX.ZERO_WIDTH_ANCHOR, '')
+        .replace(REGEX.COMBINED_LINE_REMOVALS, '');
+}
+function normalizeMarkdownSpacing(text) {
+    let result = text
+        .replace(REGEX.SPACING_LINK_FIX, ']($1)\n\n[')
+        .replace(REGEX.SPACING_ADJ_COMBINED, '$& ')
+        .replace(REGEX.SPACING_CODE_DASH, '$1 - ')
+        .replace(REGEX.SPACING_ESCAPES, '$1')
+        .replace(REGEX.SPACING_LIST_NUM_COMBINED, '$1\n\n$2')
+        .replace(REGEX.PUNCT_ONLY_LIST_ARTIFACT, '')
+        .replace(REGEX.DOUBLE_NEWLINE_REDUCER, '\n\n');
+    // Trim leading whitespace inside inline code spans
+    result = result.replace(/(?<=\s|^)`\s+([^`]+)`/gm, '`$1`');
+    // Unescape backticks inside markdown link text
+    result = result.replace(/\[([^\]]*\\`[^\]]*)\]\(([^)]+)\)/g, (_match, linkText, url) => `[${linkText.replace(/\\`/g, '`')}](${url})`);
+    result = result.replace(/\[([^\]]*<[^\]]*)\]\(([^)]+)\)/g, (_match, linkText, url) => `[${linkText.replace(/</g, '\\<').replace(/>/g, '\\>')}](${url})`);
+    return normalizeNestedListIndentation(result);
+}
+function fixConcatenatedProperties(text) {
     let result = text;
+    for (let k = 0; k < PROPERTY_FIX_MAX_PASSES; k++) {
+        const next = result.replace(REGEX.CONCATENATED_PROPS, '$1$2\n\n$3');
+        if (next === result)
+            break;
+        result = next;
+    }
+    return result;
+}
+function applyGlobalRegexes(text, options) {
     const checkAbort = createAbortChecker(options);
+    let result = text.replace(/\u00A0/g, ' ');
     checkAbort('markdown:cleanup:headings');
-    // fixAndSpaceHeadings
     result = result
         .replace(REGEX.HEADING_SPACING, '$1\n\n$2')
         .replace(REGEX.HEADING_CODE_BLOCK, '$1\n\n```');
     if (config.markdownCleanup.removeTypeDocComments) {
         checkAbort('markdown:cleanup:typedoc');
-        result = result
-            .split('\n')
-            .filter((line) => !isTypeDocArtifactLine(line))
-            .join('\n');
-        result = result.replace(REGEX.TYPEDOC_COMMENT, (match) => match.startsWith('`') ? match : '');
+        result = removeTypeDocArtifacts(result);
     }
     if (config.markdownCleanup.removeSkipLinks) {
         checkAbort('markdown:cleanup:skip-links');
-        result = result
-            .replace(REGEX.ZERO_WIDTH_ANCHOR, '')
-            .replace(REGEX.COMBINED_LINE_REMOVALS, '');
+        result = removeSkipLinks(result);
     }
     checkAbort('markdown:cleanup:spacing');
-    // normalizeSpacing
-    result = result
-        .replace(REGEX.SPACING_LINK_FIX, ']($1)\n\n[')
-        .replace(REGEX.SPACING_ADJ_COMBINED, '$& ')
-        .replace(REGEX.SPACING_CODE_DASH, '$1 - ')
-        .replace(REGEX.SPACING_ESCAPES, '$1')
-        .replace(REGEX.SPACING_LIST_NUM_COMBINED, '$1\n\n$2')
-        .replace(REGEX.PUNCT_ONLY_LIST_ARTIFACT, '')
-        .replace(REGEX.DOUBLE_NEWLINE_REDUCER, '\n\n');
-    result = normalizeNestedListIndentation(result);
+    result = normalizeMarkdownSpacing(result);
     checkAbort('markdown:cleanup:properties');
-    // fixProperties
-    for (let k = 0; k < 3; k++) {
-        const next = result.replace(REGEX.CONCATENATED_PROPS, '$1$2\n\n$3');
-        if (next === result)
-            break;
-        result = next;
-    }
-    return result;
+    return fixConcatenatedProperties(result);
 }
 function normalizeNestedListIndentation(text) {
     return text.replace(REGEX.NESTED_LIST_INDENT, (match, spaces, marker) => {
@@ -1130,27 +1187,17 @@ export function cleanupMarkdownArtifacts(content, options) {
         return '';
     const checkAbort = createAbortChecker(options);
     checkAbort('markdown:cleanup:begin');
-    const len = content.length;
-    let lastIndex = 0;
+    const lines = content.split(/\r?\n/);
     let fenceMarker = null;
     const segments = [];
     let buffer = [];
-    while (lastIndex < len) {
-        let nextIndex = content.indexOf('\n', lastIndex);
-        let line;
-        if (nextIndex === -1) {
-            line = content.slice(lastIndex);
-            nextIndex = len;
-        }
-        else {
-            if (nextIndex > lastIndex && content.charCodeAt(nextIndex - 1) === 13) {
-                line = content.slice(lastIndex, nextIndex - 1);
-            }
-            else {
-                line = content.slice(lastIndex, nextIndex);
-            }
-            nextIndex++; // Skip \n
+    const flushBuffer = () => {
+        if (buffer.length > 0) {
+            segments.push(processTextBuffer(buffer, options));
+            buffer = [];
         }
+    };
+    for (const line of lines) {
         const trimmed = line.trimStart();
         if (fenceMarker) {
             segments.push(line);
@@ -1166,22 +1213,16 @@ export function cleanupMarkdownArtifacts(content, options) {
                 buffer.push(line);
             }
             else {
-                if (buffer.length > 0) {
-                    segments.push(processTextBuffer(buffer, options));
-                    buffer = [];
-                }
+                flushBuffer();
                 segments.push(line);
                 fenceMarker = newMarker;
             }
         }
-        lastIndex = nextIndex;
-    }
-    if (buffer.length > 0) {
-        segments.push(processTextBuffer(buffer, options));
     }
+    flushBuffer();
     return segments.join('\n').trim();
 }
-function detectFrontmatter(content) {
+function parseFrontmatter(content) {
     const len = content.length;
     if (len < 4)
         return null;
@@ -1201,57 +1242,43 @@ function detectFrontmatter(content) {
     const closeIndex = content.indexOf(fence, fenceLen);
     if (closeIndex === -1)
         return null;
-    return {
+    const range = {
         start: 0,
         end: closeIndex + fenceLen,
         linesStart: fenceLen,
         linesEnd: closeIndex,
         lineEnding,
     };
-}
-function parseFrontmatterEntry(line) {
-    const trimmed = line.trim();
-    const idx = trimmed.indexOf(':');
-    if (!trimmed || idx <= 0)
-        return null;
-    return {
-        key: trimmed.slice(0, idx).trim().toLowerCase(),
-        value: trimmed.slice(idx + 1).trim(),
-    };
-}
-function stripFrontmatterQuotes(val) {
-    const first = val.charAt(0);
-    const last = val.charAt(val.length - 1);
-    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
-        return val.slice(1, -1).trim();
-    }
-    return val;
-}
-function scanFrontmatterForTitle(content, fm) {
-    const fmBody = content.slice(fm.linesStart, fm.linesEnd);
+    // Parse key-value entries in one pass
+    const entries = new Map();
+    const fmBody = content.slice(range.linesStart, range.linesEnd);
     let lastIdx = 0;
     while (lastIdx < fmBody.length) {
-        let nextIdx = fmBody.indexOf(fm.lineEnding, lastIdx);
+        let nextIdx = fmBody.indexOf(lineEnding, lastIdx);
         if (nextIdx === -1)
             nextIdx = fmBody.length;
-        const line = fmBody.slice(lastIdx, nextIdx);
-        const entry = parseFrontmatterEntry(line);
-        if (entry) {
-            if (entry.key === 'title' || entry.key === 'name') {
-                const cleaned = stripFrontmatterQuotes(entry.value);
-                if (cleaned)
-                    return cleaned;
+        const line = fmBody.slice(lastIdx, nextIdx).trim();
+        const colonIdx = line.indexOf(':');
+        if (line && colonIdx > 0) {
+            const key = line.slice(0, colonIdx).trim().toLowerCase();
+            let value = line.slice(colonIdx + 1).trim();
+            // Strip surrounding quotes
+            const first = value.charAt(0);
+            const last = value.charAt(value.length - 1);
+            if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+                value = value.slice(1, -1).trim();
             }
+            if (value)
+                entries.set(key, value);
         }
-        lastIdx = nextIdx + fm.lineEnding.length;
+        lastIdx = nextIdx + lineEnding.length;
     }
-    return undefined;
+    return { range, entries };
 }
 function scanBodyForTitle(content) {
     const len = content.length;
     let scanIndex = 0;
-    const LIMIT = 5000;
-    const maxScan = Math.min(len, LIMIT);
+    const maxScan = Math.min(len, BODY_SCAN_LIMIT);
     while (scanIndex < maxScan) {
         let nextIndex = content.indexOf('\n', scanIndex);
         if (nextIndex === -1)
@@ -1271,16 +1298,16 @@ function scanBodyForTitle(content) {
     return undefined;
 }
 export function extractTitleFromRawMarkdown(content) {
-    const fm = detectFrontmatter(content);
+    const fm = parseFrontmatter(content);
     if (fm) {
-        const title = scanFrontmatterForTitle(content, fm);
+        const title = fm.entries.get('title') ?? fm.entries.get('name');
         if (title)
             return title;
     }
     return scanBodyForTitle(content);
 }
 export function addSourceToMarkdown(content, url) {
-    const fm = detectFrontmatter(content);
+    const fm = parseFrontmatter(content);
     const useMarkdownFormat = config.transform.metadataFormat === 'markdown';
     if (useMarkdownFormat && !fm) {
         if (REGEX.SOURCE_KEY.test(content))
@@ -1301,13 +1328,17 @@ export function addSourceToMarkdown(content, url) {
         const escapedUrl = url.replace(/"/g, '\\"');
         return `---${lineEnding}source: "${escapedUrl}"${lineEnding}---${lineEnding}${lineEnding}${content}`;
     }
-    const fmBody = content.slice(fm.linesStart, fm.linesEnd);
+    const fmBody = content.slice(fm.range.linesStart, fm.range.linesEnd);
     if (REGEX.SOURCE_KEY.test(fmBody))
         return content;
     const escapedUrl = url.replace(/"/g, '\\"');
-    const injection = `source: "${escapedUrl}"${fm.lineEnding}`;
-    return content.slice(0, fm.linesEnd) + injection + content.slice(fm.linesEnd);
+    const injection = `source: "${escapedUrl}"${fm.range.lineEnding}`;
+    return (content.slice(0, fm.range.linesEnd) +
+        injection +
+        content.slice(fm.range.linesEnd));
 }
+// endregion
+// region Content Detection & Metadata Footer
 function countCommonTags(content, limit) {
     if (limit <= 0)
         return 0;
@@ -1324,10 +1355,10 @@ export function isRawTextContent(content) {
     const trimmed = content.trim();
     if (REGEX.HTML_DOC_START.test(trimmed))
         return false;
-    if (detectFrontmatter(trimmed) !== null)
+    if (parseFrontmatter(trimmed) !== null)
         return true;
-    const tagCount = countCommonTags(content, 5);
-    if (tagCount > 5)
+    const tagCount = countCommonTags(content, HTML_TAG_DENSITY_LIMIT);
+    if (tagCount > HTML_TAG_DENSITY_LIMIT)
         return false;
     return (REGEX.HEADING_MARKER.test(content) ||
         REGEX.LIST_MARKER.test(content) ||
@@ -1365,3 +1396,4 @@ export function buildMetadataFooter(metadata, fallbackUrl) {
         lines.push(` <sub>${metadata.description}</sub>`);
     return lines.join('\n');
 }
+// endregion