@j0hanz/fetch-url-mcp 1.9.2 → 1.9.3

package/dist/lib/content.d.ts

@@ -2,6 +2,7 @@ import { type MetadataBlock } from '../transform/types.js';
 export declare function serializeDocumentForMarkdown(document: Document, fallback: string): string;
 export declare function prepareDocumentForMarkdown(document: Document, baseUrl?: string, signal?: AbortSignal): void;
 export declare function removeNoiseFromHtml(html: string, document?: Document, baseUrl?: string, signal?: AbortSignal): string;
+export declare function extractLanguageFromClassName(className: string): string | undefined;
 export declare function resolveLanguageFromAttributes(className: string, dataLang: string): string | undefined;
 export declare function detectLanguageFromCode(code: string): string | undefined;
 interface CleanupOptions {

package/dist/lib/content.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../src/lib/content.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAujB3D,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CAQR;AAuCD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,IAAI,CAiBN;AA0BD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,MAAM,CAcR;AAoTD,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAqBvE;AAoDD,UAAU,cAAc;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAqSD,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,cAAc,GACvB,MAAM,CA6CR;AAgGD,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAOpB;AACD,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAuCxE;AAmBD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAczD;AAaD,wBAAgB,mBAAmB,CACjC,QAAQ,CAAC,EAAE,aAAa,EACxB,WAAW,CAAC,EAAE,MAAM,GACnB,MAAM,CAmBR"}
+{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../src/lib/content.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAwjB3D,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CAQR;AA2DD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,IAAI,CAiBN;AA0BD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,GACnB,MAAM,CAcR;AAwQD,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,GAChB,MAAM,GAAG,SAAS,CAuBpB;AAqBD,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,MAAM,GAAG,SAAS,CAKpB;AACD,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAqBvE;AAsDD,UAAU,cAAc;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAqTD,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,cAAc,GACvB,MAAM,CA6CR;AAgGD,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CAOpB;AACD,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAuCxE;AAmBD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAczD;AAaD,wBAAgB,mBAAmB,CACjC,QAAQ,CAAC,EAAE,aAAa,EACxB,WAAW,CAAC,EAAE,MAAM,GACnB,MAAM,CAmBR"}

package/dist/lib/content.js

@@ -41,7 +41,7 @@ const NOISE_PATTERNS = [
     /[\s"']role\s*=\s*['"]?(?:navigation|banner|complementary|contentinfo|tree|menubar|menu)['"]?/i,
     /[\s"'](?:aria-hidden\s*=\s*['"]?true['"]?|hidden)/i,
     /[\s"'](?:banner|promo|announcement|cta|advert|newsletter|subscribe|cookie|consent|popup|modal|overlay|toast)\b/i,
-    /[\s"'](?:fixed|sticky|z-50|z-4|isolate|breadcrumb|pagination)\b/i,
+    /[\s"'](?:fixed|sticky|z-50|z-4|isolate|breadcrumbs?|pagination)\b/i,
 ];
 const HEADER_NOISE_PATTERN = /\b(site-header|masthead|topbar|navbar|nav(?:bar)?|menu|header-nav)\b/i;
 const FIXED_OR_HIGH_Z_PATTERN = /\b(?:fixed|sticky|z-(?:4\d|50)|isolate)\b/;
@@ -102,6 +102,7 @@ const PROMO_TOKENS_ALWAYS = [
     'sponsor',
     'recommend',
     'breadcrumb',
+    'breadcrumbs',
     'pagination',
     'pager',
     'taglist',
@@ -118,7 +119,7 @@ const PROMO_TOKENS_BY_CATEGORY = {
 };
 // Noise selector configurations
 const BASE_NOISE_SELECTORS = {
-    navFooter: 'nav,footer,header[class*="site"],header[class*="nav"],header[class*="menu"],[role="banner"],[role="navigation"]',
+    navFooter: 'nav,footer,header[class*="site"],header[class*="nav"],header[class*="menu"],[role="banner"],[role="navigation"],[class*="breadcrumb"]',
     cookieBanners: '[role="dialog"]',
     hidden: '[style*="display: none"],[style*="display:none"],[style*="visibility: hidden"],[style*="visibility:hidden"],[hidden],[aria-hidden="true"]',
 };
@@ -504,17 +505,31 @@ function mayContainNoise(html) {
         : `${html.substring(0, NOISE_SCAN_LIMIT)}\n${html.substring(html.length - NOISE_SCAN_LIMIT)}`;
     return NOISE_PATTERNS.some((re) => re.test(sample));
 }
+function surfaceHiddenTabPanels(document) {
+    const panels = document.querySelectorAll('[data-slot="tabContent"], [role="tabpanel"]');
+    for (const panel of panels) {
+        const style = panel.getAttribute('style') ?? '';
+        if (/display\s*:\s*none/i.test(style)) {
+            panel.setAttribute('style', style.replace(/display\s*:\s*none\s*;?/gi, '').trim());
+        }
+        panel.removeAttribute('hidden');
+    }
+}
 function stripTabTriggers(document) {
+    surfaceHiddenTabPanels(document);
     const tabs = document.querySelectorAll('button[role="tab"]');
     for (let i = tabs.length - 1; i >= 0; i--) {
         tabs[i]?.remove();
     }
 }
 function escapeTableCellPipes(document) {
-    const codes = document.querySelectorAll('td code, th code');
-    for (const code of codes) {
-        if (code.textContent.includes('|')) {
-            code.textContent = code.textContent.replace(/\|/g, '\\|');
+    const cells = document.querySelectorAll('td, th');
+    for (const cell of cells) {
+        for (const node of cell.childNodes) {
+            const text = node.textContent;
+            if (node.nodeType === 3 && text?.includes('|')) {
+                node.textContent = text.replace(/\|/g, '\\|');
+            }
         }
     }
 }
@@ -820,7 +835,7 @@ const LANGUAGES = [
         match: (ctx) => ctx.trimmedStart.startsWith('{') || ctx.trimmedStart.startsWith('['),
     },
 ];
-function extractLanguageFromClassName(className) {
+export function extractLanguageFromClassName(className) {
     if (!className)
         return undefined;
     // Split by whitespace and check for language indicators
@@ -895,12 +910,13 @@ const REGEX = {
     HEADING_MARKER: /^#{1,6}\s/m,
     HEADING_STRICT: /^#{1,6}\s+/m,
     EMPTY_HEADING_LINE: /^#{1,6}[ \t\u00A0]*$/,
+    ANCHOR_ONLY_HEADING: /^#{1,6}\s+\[[^\]]+\]\(#[^)]+\)\s*$/,
     FENCE_START: /^\s*(`{3,}|~{3,})/,
     LIST_MARKER: /^(?:[-*+])\s/m,
     TOC_LINK: /^- \[[^\]]+\]\(#[^)]+\)\s*$/,
-    TOC_HEADING: /^(?:#{1,6}\s+)?(?:table of contents|contents)\s*$/i,
+    TOC_HEADING: /^(?:#{1,6}\s+)?(?:table of contents|contents|on this page)\s*$/i,
     HTML_DOC_START: /^(<!doctype|<html)/i,
-    COMBINED_LINE_REMOVALS: /^(?:\[Skip to (?:main )?(?:content|navigation)\]\(#[^)]*\)|\[Skip link\]\(#[^)]*\)|Was this page helpful\??)\s*$/gim,
+    COMBINED_LINE_REMOVALS: /^(?:\[Skip to (?:main )?(?:content|navigation)\]\(#[^)]*\)|\[Skip link\]\(#[^)]*\)|Was this page helpful\??|\[Back to top\]\(#[^)]*\)|\[\s*\]\(https?:\/\/[^)]*\))\s*$/gim,
     ZERO_WIDTH_ANCHOR: /\[(?:\s|\u200B)*\]\(#[^)]*\)[ \t]*/g,
     CONCATENATED_PROPS: /([a-z_][a-z0-9_]{0,30}\??:\s+)([\u0022\u201C][^\u0022\u201C\u201D]*[\u0022\u201D])([a-z_][a-z0-9_]{0,30}\??:)/g,
     DOUBLE_NEWLINE_REDUCER: /\n{3,}/g,
@@ -948,6 +964,9 @@ function hasFollowingContent(lines, startIndex) {
     }
     return false;
 }
+function stripAnchorOnlyHeading(line) {
+    return line.replace(/^(#{1,6})\s+\[([^\]]+)\]\(#[^)]+\)\s*$/, '$1 $2');
+}
 function isTitleCaseOrKeyword(trimmed) {
     // Quick check for length to avoid regex on long strings
     if (trimmed.length > MAX_LINE_LENGTH)
@@ -1094,6 +1113,11 @@ function preprocessLines(lines, options) {
         const trimmed = line.trim();
         if (REGEX.EMPTY_HEADING_LINE.test(trimmed))
             continue;
+        if (REGEX.ANCHOR_ONLY_HEADING.test(trimmed)) {
+            if (!hasFollowingContent(lines, i))
+                continue;
+            line = stripAnchorOnlyHeading(trimmed);
+        }
         const tocSkip = shouldSkipAsToc(lines, i, trimmed, removeToc, options);
         if (tocSkip !== null) {
             skipUntil = tocSkip;
@@ -1127,6 +1151,16 @@ function removeSkipLinks(text) {
         .replace(REGEX.ZERO_WIDTH_ANCHOR, '')
         .replace(REGEX.COMBINED_LINE_REMOVALS, '');
 }
+function normalizeInlineCodeTokens(text) {
+    return text.replace(/`([^`\n]+)`/g, (match, inner) => {
+        const trimmed = inner.trim();
+        if (trimmed === inner || /\s/.test(trimmed))
+            return match;
+        if (!/^[*A-Za-z0-9_./:-]+$/.test(trimmed))
+            return match;
+        return `\`${trimmed}\``;
+    });
+}
 function normalizeMarkdownSpacing(text) {
     let result = text
         .replace(REGEX.SPACING_LINK_FIX, ']($1)\n\n[')
@@ -1136,8 +1170,8 @@ function normalizeMarkdownSpacing(text) {
         .replace(REGEX.SPACING_LIST_NUM_COMBINED, '$1\n\n$2')
         .replace(REGEX.PUNCT_ONLY_LIST_ARTIFACT, '')
         .replace(REGEX.DOUBLE_NEWLINE_REDUCER, '\n\n');
-    // Trim leading whitespace inside inline code spans
-    result = result.replace(/(?<=\s|^)`\s+([^`]+)`/gm, '`$1`');
+    // Trim whitespace around token-like inline code spans.
+    result = normalizeInlineCodeTokens(result);
     // Unescape backticks inside markdown link text
     result = result.replace(/\[([^\]]*\\`[^\]]*)\]\(([^)]+)\)/g, (_match, linkText, url) => `[${linkText.replace(/\\`/g, '`')}](${url})`);
     result = result.replace(/\[([^\]]*<[^\]]*)\]\(([^)]+)\)/g, (_match, linkText, url) => `[${linkText.replace(/</g, '\\<').replace(/>/g, '\\>')}](${url})`);

package/dist/transform/html-translators.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html-translators.d.ts","sourceRoot":"","sources":["../../src/transform/html-translators.ts"],"names":[],"mappings":"AA4fA,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE"}
+{"version":3,"file":"html-translators.d.ts","sourceRoot":"","sources":["../../src/transform/html-translators.ts"],"names":[],"mappings":"AAigBA,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE"}

package/dist/transform/html-translators.js

@@ -255,11 +255,13 @@ const GFM_ALERT_MAP = new Map([
     ['danger', 'CAUTION'],
     ['important', 'IMPORTANT'],
 ]);
+const ADMONITION_TOKEN_RE = /^(?:note|tip|hint|info|warning|warn|danger|caution|important)$/i;
 function resolveGfmAlertType(className) {
-    const lower = className.toLowerCase();
-    for (const [key, type] of GFM_ALERT_MAP) {
-        if (lower.includes(key))
-            return type;
+    const tokens = className.toLowerCase().split(/\s+/);
+    for (const token of tokens) {
+        const mapped = GFM_ALERT_MAP.get(token);
+        if (mapped)
+            return mapped;
     }
     return undefined;
 }
@@ -278,11 +280,12 @@ function buildDivTranslator(ctx) {
             postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
         };
     }
+    const classTokens = className.split(/\s+/);
     const isAdmonition = className.includes('admonition') ||
         className.includes('callout') ||
         className.includes('custom-block') ||
         getAttribute('role') === 'alert' ||
-        /\b(note|tip|info|warning|danger|caution|important)\b/i.test(className);
+        classTokens.some((t) => ADMONITION_TOKEN_RE.test(t));
     if (isAdmonition) {
         return {
             postprocess: ({ content }) => {

package/dist/transform/transform.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAuCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAuJD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAuVD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AA+OD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAwJD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAiED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AAsCD,iBAAS,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAc/D;AAED,iBAAS,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAYlE;AAED,iBAAS,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAcvD;AAED,eAAO,MAAM,mBAAmB;;;;CAItB,CAAC;AAiQX,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAmBzB;AA+CD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAsH1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}
+{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAyCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAuJD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAwWD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AA+OD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAuJD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAiED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AAuCD,iBAAS,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAc/D;AAED,iBAAS,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAYlE;AA6CD,iBAAS,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAcvD;AAED,eAAO,MAAM,mBAAmB;;;;CAItB,CAAC;AAurBX,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAmBzB;AA+CD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAsH1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}

package/dist/transform/transform.js

@@ -3,7 +3,7 @@ import diagnosticsChannel from 'node:diagnostics_channel';
 import { performance } from 'node:perf_hooks';
 import { isProbablyReaderable, Readability } from '@mozilla/readability';
 import { parseHTML } from 'linkedom';
-import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, } from '../lib/content.js';
+import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, detectLanguageFromCode, extractLanguageFromClassName, extractTitleFromRawMarkdown, isRawTextContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, } from '../lib/content.js';
 import { config } from '../lib/core.js';
 import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
 import { isRawTextContentUrl } from '../lib/http.js';
@@ -284,13 +284,22 @@ function resolveCollapsedTextLengthUpTo(text, max) {
     return length;
 }
 function preserveAlertElements(doc) {
-    const alerts = doc.querySelectorAll('[role="alert"], .admonition, .callout');
+    const alerts = doc.querySelectorAll('[role="alert"], .admonition, [class*="callout"]');
     for (const el of alerts) {
         const bq = doc.createElement('blockquote');
         bq.innerHTML = el.innerHTML;
         el.replaceWith(bq);
     }
 }
+function preserveCodeLanguageAttributes(doc) {
+    for (const el of doc.querySelectorAll('pre, code')) {
+        if (el.getAttribute('data-language'))
+            continue;
+        const lang = extractLanguageFromClassName(el.getAttribute('class') ?? '');
+        if (lang)
+            el.setAttribute('data-language', lang);
+    }
+}
 function extractArticle(document, url, signal) {
     if (!isReadabilityCompatible(document)) {
         logWarn('Document not compatible with Readability');
@@ -321,6 +330,10 @@ function extractArticle(document, url, signal) {
             ? doc.cloneNode(true)
             : doc;
         preserveAlertElements(readabilityDoc);
+        preserveCodeLanguageAttributes(readabilityDoc);
+        for (const el of readabilityDoc.querySelectorAll('[class*="breadcrumb"],[class*="pagination"]')) {
+            el.remove();
+        }
         checkAbort('extract:article:parse');
         const reader = new Readability(readabilityDoc, {
             maxElemsToParse: MAX_READABILITY_ELEMENTS,
@@ -654,6 +667,13 @@ const MIN_CONTENT_RATIO = 0.15;
 const MIN_HTML_LENGTH_FOR_GATE = 100;
 const MIN_HEADING_RETENTION_RATIO = 0.3;
 const MIN_CODE_BLOCK_RETENTION_RATIO = 0.15;
+const MIN_TABLE_RETENTION_RATIO = 0.5;
+const MIN_IMAGE_RETENTION_RATIO = 0.2;
+const MIN_INTERACTIVE_RETENTION_RATIO = 0.1;
+const MIN_INTERACTIVE_ELEMENTS_FOR_GATE = 6;
+const MIN_IMAGE_ELEMENTS_FOR_GATE = 4;
+const MIN_HEADINGS_FOR_EMPTY_SECTION_GATE = 5;
+const MAX_EMPTY_SECTION_RATIO = 0.05;
 const MIN_LINE_LENGTH_FOR_TRUNCATION_CHECK = 20;
 const MAX_TRUNCATED_LINE_RATIO = 0.95;
 function needsDocumentWrapper(html) {
@@ -679,13 +699,6 @@ function resolveHtmlDocument(htmlOrDocument) {
         return parseHTML('<!DOCTYPE html><html><body></body></html>').document;
     }
 }
-function countTagsInString(html, regex) {
-    let count = 0;
-    while (regex.exec(html) !== null) {
-        count++;
-    }
-    return count;
-}
 function stripNonVisibleNodes(root) {
     for (const el of root.querySelectorAll('script,style,noscript')) {
         el.remove();
@@ -868,6 +881,43 @@ function findPrimaryHeading(document) {
     }
     return undefined;
 }
+function countMatchingElements(root, selector) {
+    return root.querySelectorAll(selector).length;
+}
+function getHeadingLevel(heading) {
+    const match = /^H([1-6])$/.exec(heading.tagName);
+    if (!match)
+        return null;
+    return Number.parseInt(match[1] ?? '', 10);
+}
+function hasSectionContent(heading) {
+    const level = getHeadingLevel(heading);
+    if (level === null)
+        return false;
+    let current = heading.nextElementSibling;
+    while (current) {
+        const currentLevel = getHeadingLevel(current);
+        if (currentLevel !== null && currentLevel <= level)
+            return false;
+        const text = current.textContent.trim();
+        if (text.length > 0)
+            return true;
+        if (current.querySelector('img,table,pre,code,ul,ol,figure,blockquote')) {
+            return true;
+        }
+        current = current.nextElementSibling;
+    }
+    return false;
+}
+function countEmptyHeadingSections(root) {
+    let emptyCount = 0;
+    const headings = root.querySelectorAll('h1,h2,h3,h4,h5,h6');
+    for (const heading of headings) {
+        if (!hasSectionContent(heading))
+            emptyCount += 1;
+    }
+    return emptyCount;
+}
 function isGithubRepositoryRootUrl(url) {
     let parsed;
     try {
@@ -895,20 +945,49 @@ function shouldUseArticleContent(article, document) {
         if (ratio < MIN_CONTENT_RATIO)
             return false;
     }
-    const originalHeadings = document.querySelectorAll('h1,h2,h3,h4,h5,h6').length;
+    const { document: articleDoc } = parseHTML(`<!DOCTYPE html><html><body>${article.content}</body></html>`);
+    const originalHeadings = countMatchingElements(document, 'h1,h2,h3,h4,h5,h6');
+    const articleHeadings = countMatchingElements(articleDoc, 'h1,h2,h3,h4,h5,h6');
     if (originalHeadings > 0) {
-        const articleHeadings = countTagsInString(article.content, /<h[1-6]\b/gi);
         const retentionRatio = articleHeadings / originalHeadings;
         if (retentionRatio < MIN_HEADING_RETENTION_RATIO)
             return false;
     }
-    const originalCodeBlocks = document.querySelectorAll('pre').length;
+    const originalCodeBlocks = countMatchingElements(document, 'pre');
     if (originalCodeBlocks > 0) {
-        const articleCodeBlocks = countTagsInString(article.content, /<pre\b/gi);
+        const articleCodeBlocks = countMatchingElements(articleDoc, 'pre');
         const codeRetentionRatio = articleCodeBlocks / originalCodeBlocks;
         if (codeRetentionRatio < MIN_CODE_BLOCK_RETENTION_RATIO)
             return false;
     }
+    const originalTables = countMatchingElements(document, 'table');
+    if (originalTables > 0) {
+        const articleTables = countMatchingElements(articleDoc, 'table');
+        const tableRetentionRatio = articleTables / originalTables;
+        if (tableRetentionRatio < MIN_TABLE_RETENTION_RATIO)
+            return false;
+    }
+    const originalImages = countMatchingElements(document, 'img');
+    if (originalImages >= MIN_IMAGE_ELEMENTS_FOR_GATE) {
+        const articleImages = countMatchingElements(articleDoc, 'img');
+        const imageRetentionRatio = articleImages / originalImages;
+        if (imageRetentionRatio < MIN_IMAGE_RETENTION_RATIO)
+            return false;
+    }
+    const interactiveSelector = 'button,[role="tab"],[role="tabpanel"],[aria-controls]';
+    const originalInteractive = countMatchingElements(document, interactiveSelector);
+    if (originalInteractive >= MIN_INTERACTIVE_ELEMENTS_FOR_GATE) {
+        const articleInteractive = countMatchingElements(articleDoc, interactiveSelector);
+        const interactiveRetentionRatio = articleInteractive / originalInteractive;
+        if (interactiveRetentionRatio < MIN_INTERACTIVE_RETENTION_RATIO) {
+            return false;
+        }
+    }
+    if (articleHeadings >= MIN_HEADINGS_FOR_EMPTY_SECTION_GATE) {
+        const emptySectionRatio = countEmptyHeadingSections(articleDoc) / articleHeadings;
+        if (emptySectionRatio > MAX_EMPTY_SECTION_RATIO)
+            return false;
+    }
     return !hasTruncatedSentences(article.textContent);
 }
 function buildContentSource(params) {
@@ -922,6 +1001,7 @@ function buildContentSource(params) {
         primaryHeading: document
             ? TransformHeuristics.findPrimaryHeading(document)
             : undefined,
+        originalHtml: html,
     };
     if (useArticleContent && article) {
         const { document: articleDoc } = parseHTML(`<!DOCTYPE html><html><body>${article.content}</body></html>`);
@@ -954,6 +1034,290 @@ function buildContentSource(params) {
         title: extractedMeta.title,
     };
 }
+const NEXT_FLIGHT_PAYLOAD_RE = /self\.__next_f\.push\(\[1,"((?:\\.|[^"\\])*)"\]\)<\/script>/gs;
+const TEMPLATE_ASSIGNMENT_RE = /([A-Za-z_$][\w$]*)=`([\s\S]*?)`;/g;
+const OBJECT_ASSIGNMENT_RE = /([A-Za-z_$][\w$]*)=\{([^{}]+)\}/g;
+const FLIGHT_INSTALL_RE = /commands:\{cli:"([^"]+)",npm:"([^"]+)",yarn:"([^"]+)",pnpm:"([^"]+)",bun:"([^"]+)"\}/;
+const FLIGHT_IMPORT_RE = /commands:\{main:'([^']+)',individual:'([^']+)'\}/;
+const FLIGHT_DEMO_RE = /title:"([^"]+)",files:([A-Za-z_$][\w$]*)\.([A-Za-z_$][\w$]*)/g;
+const FLIGHT_API_RE = /children:"([^"]+)"\}\),`\\n`,\(0,e\.jsx\)\(o,\{data:\[([\s\S]*?)\]\}\)/g;
+const FLIGHT_API_ROW_RE = /attribute:"([^"]+)",type:"([^"]+)",description:"([^"]*)",default:"([^"]*)"/g;
+const FLIGHT_MERMAID_SECTION_RE = /_jsx\(Heading,\{\s*level:"[1-6]",\s*id:"[^"]+",\s*children:"((?:\\.|[^"\\])*)"\s*\}\)(?:(?!_jsx\(Heading,\{)[\s\S]){0,12000}?_jsx\(Mermaid,\{\s*chart:"((?:\\.|[^"\\])*)"\s*\}\)/g;
+function decodeHtmlEntities(value) {
+    return value
+        .replace(/&#39;|&#x27;/g, "'")
+        .replace(/&quot;/g, '"')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>');
+}
+function decodeFlightStringValue(value) {
+    try {
+        return JSON.parse(`"${value}"`);
+    }
+    catch {
+        return decodeHtmlEntities(value);
+    }
+}
+function decodeNextFlightPayloads(html) {
+    const payloads = [];
+    for (const match of html.matchAll(NEXT_FLIGHT_PAYLOAD_RE)) {
+        const rawPayload = match[1];
+        if (!rawPayload)
+            continue;
+        try {
+            payloads.push(JSON.parse(`"${rawPayload}"`));
+        }
+        catch {
+            // Ignore malformed payload fragments and continue with the rest.
+        }
+    }
+    return payloads;
+}
+function parseFlightObjectRefs(text) {
+    const templateMap = new Map();
+    const aliasMap = new Map();
+    const objectMaps = new Map();
+    for (const match of text.matchAll(TEMPLATE_ASSIGNMENT_RE)) {
+        const name = match[1];
+        const code = match[2];
+        if (name && code)
+            templateMap.set(name, decodeHtmlEntities(code));
+    }
+    for (const match of text.matchAll(OBJECT_ASSIGNMENT_RE)) {
+        const objectName = match[1];
+        const body = match[2]?.trim() ?? '';
+        if (!objectName || !body)
+            continue;
+        const spreadMatch = /^\.\.\.([A-Za-z_$][\w$]*)$/.exec(body);
+        if (spreadMatch?.[1]) {
+            aliasMap.set(objectName, spreadMatch[1]);
+            continue;
+        }
+        const entries = new Map();
+        for (const part of body.split(',')) {
+            const entryMatch = /(?:"([^"]+)"|([A-Za-z_$][\w$]*)):([A-Za-z_$][\w$]*)$/.exec(part.trim());
+            const key = entryMatch?.[1] ?? entryMatch?.[2];
+            const value = entryMatch?.[3];
+            if (key && value)
+                entries.set(key, value);
+        }
+        if (entries.size > 0)
+            objectMaps.set(objectName, entries);
+    }
+    return { templateMap, aliasMap, objectMaps };
+}
+function resolveFlightCodeRef(name, refs, seen = new Set()) {
+    if (!name || seen.has(name))
+        return undefined;
+    seen.add(name);
+    const direct = refs.templateMap.get(name);
+    if (direct)
+        return direct;
+    const alias = refs.aliasMap.get(name);
+    if (alias)
+        return resolveFlightCodeRef(alias, refs, seen);
+    const objectMap = refs.objectMaps.get(name);
+    if (!objectMap)
+        return undefined;
+    for (const ref of objectMap.values()) {
+        const resolved = resolveFlightCodeRef(ref, refs, seen);
+        if (resolved)
+            return resolved;
+    }
+    return undefined;
+}
+function escapeMarkdownTableCell(value) {
+    const normalized = decodeHtmlEntities(value).replace(/\s+/g, ' ').trim();
+    return (normalized || '-').replace(/\|/g, '\\|');
+}
+function buildMarkdownTable(rows) {
+    if (rows.length === 0)
+        return '';
+    const lines = [
+        '| Prop | Type | Description | Default |',
+        '| ---- | ---- | ----------- | ------- |',
+    ];
+    for (const row of rows) {
+        lines.push(`| ${escapeMarkdownTableCell(row.attribute)} | ${escapeMarkdownTableCell(row.type)} | ${escapeMarkdownTableCell(row.description)} | ${escapeMarkdownTableCell(row.defaultValue)} |`);
+    }
+    return lines.join('\n');
+}
+function buildCodeBlock(code) {
+    const trimmed = code.trim();
+    if (!trimmed)
+        return '';
+    const language = detectLanguageFromCode(trimmed) ?? 'tsx';
+    return `\`\`\`${language}\n${trimmed}\n\`\`\``;
+}
+function buildMermaidBlock(chart) {
+    const normalized = decodeFlightStringValue(chart).trim();
+    if (!normalized)
+        return '';
+    return `\`\`\`mermaid\n${normalized}\n\`\`\``;
+}
+function normalizeSupplementHeadingText(value) {
+    return value
+        .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .toLowerCase();
+}
+function getMarkdownHeadingInfo(line) {
+    const match = /^(#{1,6})\s+(.+?)\s*$/.exec(line.trim());
+    if (!match)
+        return null;
+    return {
+        level: match[1]?.length ?? 0,
+        title: normalizeSupplementHeadingText(match[2] ?? ''),
+    };
+}
+function findMarkdownSection(lines, title) {
+    const target = normalizeSupplementHeadingText(title);
+    for (let i = 0; i < lines.length; i += 1) {
+        const heading = getMarkdownHeadingInfo(lines[i] ?? '');
+        if (heading?.title !== target)
+            continue;
+        let end = lines.length;
+        for (let j = i + 1; j < lines.length; j += 1) {
+            const nextLine = lines[j];
+            const nextHeading = nextLine !== undefined ? getMarkdownHeadingInfo(nextLine) : null;
+            if (nextHeading && nextHeading.level <= heading.level) {
+                end = j;
+                break;
+            }
+        }
+        return { start: i, end };
+    }
+    return null;
+}
+function getSectionBody(lines, section) {
+    return lines
+        .slice(section.start + 1, section.end)
+        .join('\n')
+        .trim();
+}
+function replaceMarkdownSection(lines, title, body) {
+    const section = findMarkdownSection(lines, title);
+    if (!section)
+        return false;
+    const replacement = body.trim().length > 0 ? ['', ...body.trim().split('\n'), ''] : [''];
+    lines.splice(section.start + 1, section.end - section.start - 1, ...replacement);
+    return true;
+}
+function appendMarkdownSection(lines, title, body) {
+    const section = findMarkdownSection(lines, title);
+    if (!section)
+        return false;
+    const bodyText = getSectionBody(lines, section);
+    if (bodyText.includes('```'))
+        return false;
+    const nextBody = bodyText ? `${bodyText}\n\n${body.trim()}` : body.trim();
+    return replaceMarkdownSection(lines, title, nextBody);
+}
+function extractNextFlightSupplement(originalHtml) {
+    const payloads = decodeNextFlightPayloads(originalHtml);
+    if (payloads.length === 0)
+        return null;
+    const text = payloads.join('\n');
+    const refs = parseFlightObjectRefs(text);
+    const installMatch = FLIGHT_INSTALL_RE.exec(text);
+    const importMatch = FLIGHT_IMPORT_RE.exec(text);
+    const apiTables = new Map();
+    for (const match of text.matchAll(FLIGHT_API_RE)) {
+        const title = match[1];
+        const rawRows = match[2] ?? '';
+        if (!title)
+            continue;
+        const rows = [];
+        for (const rowMatch of rawRows.matchAll(FLIGHT_API_ROW_RE)) {
+            const attribute = rowMatch[1];
+            const type = rowMatch[2];
+            const description = rowMatch[3];
+            const defaultValue = rowMatch[4];
+            if (!attribute ||
+                !type ||
+                description === undefined ||
+                defaultValue === undefined) {
+                continue;
+            }
+            rows.push({ attribute, type, description, defaultValue });
+        }
+        const table = buildMarkdownTable(rows);
+        if (table)
+            apiTables.set(title, table);
+    }
+    const mermaidDiagrams = new Map();
+    for (const match of text.matchAll(FLIGHT_MERMAID_SECTION_RE)) {
+        const title = match[1] ? decodeFlightStringValue(match[1]).trim() : '';
+        const chart = match[2] ? buildMermaidBlock(match[2]) : '';
+        if (title && chart)
+            mermaidDiagrams.set(title, chart);
+    }
+    const demoCodeBlocks = new Map();
+    for (const match of text.matchAll(FLIGHT_DEMO_RE)) {
+        const title = match[1];
+        const objectName = match[2];
+        const key = match[3];
+        const ref = objectName
+            ? refs.objectMaps.get(objectName)?.get(key ?? '')
+            : undefined;
+        const code = resolveFlightCodeRef(ref, refs);
+        const codeBlock = code ? buildCodeBlock(code) : '';
+        if (title && codeBlock)
+            demoCodeBlocks.set(title, codeBlock);
+    }
+    return {
+        ...(installMatch ? { installationCommands: installMatch.slice(1) } : {}),
+        ...(importMatch ? { importCommands: importMatch.slice(1) } : {}),
+        apiTables,
+        demoCodeBlocks,
+        mermaidDiagrams,
+    };
+}
+function supplementMarkdownFromNextFlight(markdown, originalHtml) {
+    const supplement = extractNextFlightSupplement(originalHtml);
+    if (!supplement)
+        return markdown;
+    const lines = markdown.split('\n');
+    if (supplement.installationCommands?.length) {
+        const installationSection = findMarkdownSection(lines, 'Installation');
+        if (installationSection) {
+            const installBody = getSectionBody(lines, installationSection);
+            if (!/(npm|pnpm|yarn|bun|npx)\s+(install|add)/.test(installBody)) {
+                appendMarkdownSection(lines, 'Installation', buildCodeBlock(supplement.installationCommands.join('\n')));
+            }
+        }
+    }
+    if (supplement.importCommands?.length) {
+        const importSection = findMarkdownSection(lines, 'Import');
+        if (importSection) {
+            const importBody = getSectionBody(lines, importSection);
+            if (!/import\s+\{/.test(importBody)) {
+                appendMarkdownSection(lines, 'Import', buildCodeBlock(supplement.importCommands.join('\n\n')));
+            }
+        }
+    }
+    for (const [title, table] of supplement.apiTables) {
+        replaceMarkdownSection(lines, title, table);
+    }
+    for (const [title, mermaidBlock] of supplement.mermaidDiagrams) {
+        const section = findMarkdownSection(lines, title);
+        if (!section)
+            continue;
+        const sectionBody = getSectionBody(lines, section);
+        if (sectionBody.includes('```mermaid'))
+            continue;
+        const nextBody = sectionBody
+            ? `${sectionBody}\n\n${mermaidBlock}`
+            : mermaidBlock;
+        replaceMarkdownSection(lines, title, nextBody);
+    }
+    for (const [title, codeBlock] of supplement.demoCodeBlocks) {
+        appendMarkdownSection(lines, title, codeBlock);
+    }
+    return lines.join('\n');
+}
 function resolveContentSource(params) {
     const { article, metadata: extractedMeta, document, truncated, } = extractContentContext(params.html, params.url, {
         extractArticle: true,
@@ -1001,6 +1365,8 @@ function buildMarkdownFromContext(context, url, signal) {
         }
         content = `#${prefix}${context.title}\n\n${content}`;
     }
+    content = supplementMarkdownFromNextFlight(content, context.originalHtml);
+    content = cleanupMarkdownArtifacts(content, signal ? { signal, url } : { url });
     return {
         markdown: content,
         title: context.title,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/fetch-url-mcp",
-  "version": "1.9.2",
+  "version": "1.9.3",
   "mcpName": "io.github.j0hanz/fetch-url-mcp",
   "description": "A web content fetcher MCP server that converts HTML to clean, AI and human readable markdown.",
   "type": "module",