@j0hanz/fetch-url-mcp 1.10.2 → 1.10.4

package/dist/transform/transform.js

@@ -3,10 +3,10 @@ import diagnosticsChannel from 'node:diagnostics_channel';
 import { performance } from 'node:perf_hooks';
 import { isProbablyReaderable, Readability } from '@mozilla/readability';
 import { parseHTML } from 'linkedom';
-import { detectLanguageFromCode, extractLanguageFromClassName, } from '../lib/code-lang.js';
+import { extractLanguageFromClassName } from '../lib/code-lang.js';
 import { config } from '../lib/core.js';
 import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
-import { prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, } from '../lib/dom-prep.js';
+import { normalizeTabContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, } from '../lib/dom-prep.js';
 import { isRawTextContentUrl } from '../lib/http.js';
 import { cleanupMarkdownArtifacts, processFencedContent, } from '../lib/md-cleanup.js';
 import { addSourceToMarkdown, buildMetadataFooter, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/md-metadata.js';
@@ -15,6 +15,8 @@ import { FetchError, getErrorMessage, toError } from '../lib/utils.js';
 import { isObject } from '../lib/utils.js';
 import { translateHtmlFragmentToMarkdown } from './html-translators.js';
 import { extractMetadata, extractMetadataFromHead, mergeMetadata, normalizeDocumentTitle, } from './metadata.js';
+import { supplementMarkdownFromNextFlight } from './next-flight.js';
+import { isGithubRepositoryRootUrl, maybePrependSyntheticTitle, maybeStripGithubPrimaryHeading, normalizeSyntheticTitleToken, shouldPreferPrimaryHeadingTitle, } from './title-policy.js';
 import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
 function decodeInput(input, encoding) {
     if (typeof input === 'string')
@@ -40,13 +42,6 @@ function asError(value) {
 function isWhitespaceChar(code) {
     return code === 9 || code === 10 || code === 12 || code === 13 || code === 32;
 }
-function containsWhitespace(value) {
-    for (let i = 0; i < value.length; i += 1) {
-        if (isWhitespaceChar(value.charCodeAt(i)))
-            return true;
-    }
-    return false;
-}
 function buildTransformSignal(signal) {
     const { timeoutMs } = config.transform;
     if (timeoutMs <= 0)
@@ -302,6 +297,10 @@ function preserveCodeLanguageAttributes(doc) {
             el.setAttribute('data-language', lang);
     }
 }
+// Pre-Readability cleanup on a cloned document.
+// Must strip tabs/breadcrumbs before Readability mangles role attributes.
+// The original document is NOT yet prepared (prepareDocumentForMarkdown
+// runs later in buildContentSource), so this clone starts from raw HTML.
 function extractArticle(document, url, signal) {
     if (!isReadabilityCompatible(document)) {
         logWarn('Document not compatible with Readability');
@@ -333,6 +332,7 @@ function extractArticle(document, url, signal) {
             : doc;
         preserveAlertElements(readabilityDoc);
         preserveCodeLanguageAttributes(readabilityDoc);
+        normalizeTabContent(readabilityDoc);
         for (const el of readabilityDoc.querySelectorAll('[class*="breadcrumb"],[class*="pagination"]')) {
             el.remove();
         }
@@ -453,8 +453,12 @@ export function extractContent(html, url, options = {
 }
 function resolveRelativeHref(href, baseUrl, origin) {
     const trimmedHref = href.trim();
-    if (!trimmedHref || containsWhitespace(trimmedHref))
+    if (!trimmedHref)
         return href;
+    for (let i = 0; i < trimmedHref.length; i += 1) {
+        if (isWhitespaceChar(trimmedHref.charCodeAt(i)))
+            return href;
+    }
     if (isAbsoluteOrSpecialUrl(trimmedHref))
         return trimmedHref;
     try {
@@ -628,31 +632,30 @@ function tryTransformRawContent(params) {
 }
 const MIN_CONTENT_RATIO = 0.15;
 const MIN_HTML_LENGTH_FOR_GATE = 100;
-const MIN_HEADING_RETENTION_RATIO = 0.3;
-const MIN_CODE_BLOCK_RETENTION_RATIO = 0.15;
-const MIN_TABLE_RETENTION_RATIO = 0.5;
-const MIN_IMAGE_RETENTION_RATIO = 0.2;
-const MIN_INTERACTIVE_RETENTION_RATIO = 0.1;
-const MIN_INTERACTIVE_ELEMENTS_FOR_GATE = 6;
-const MIN_IMAGE_ELEMENTS_FOR_GATE = 4;
+const RETENTION_RULES = [
+    { selector: 'h1,h2,h3,h4,h5,h6', minOriginal: 1, ratio: 0.3 },
+    { selector: 'pre', minOriginal: 1, ratio: 0.15 },
+    { selector: 'table', minOriginal: 1, ratio: 0.5 },
+    { selector: 'img', minOriginal: 4, ratio: 0.2 },
+    {
+        selector: 'button,[role="tab"],[role="tabpanel"],[aria-controls]',
+        minOriginal: 6,
+        ratio: 0.1,
+    },
+];
 const MIN_HEADINGS_FOR_EMPTY_SECTION_GATE = 5;
 const MAX_EMPTY_SECTION_RATIO = 0.05;
 const MIN_LINE_LENGTH_FOR_TRUNCATION_CHECK = 20;
 const MAX_TRUNCATED_LINE_RATIO = 0.95;
-function needsDocumentWrapper(html) {
-    const trimmed = html.trim().toLowerCase();
-    return (!trimmed.startsWith('<!doctype') &&
-        !trimmed.startsWith('<html') &&
-        !trimmed.startsWith('<body'));
-}
-function wrapHtmlFragment(html) {
-    return `<!DOCTYPE html><html><body>${html}</body></html>`;
-}
 function resolveHtmlDocument(htmlOrDocument) {
     if (typeof htmlOrDocument !== 'string')
         return htmlOrDocument;
-    const htmlToParse = needsDocumentWrapper(htmlOrDocument)
-        ? wrapHtmlFragment(htmlOrDocument)
+    const trimmed = htmlOrDocument.trim().toLowerCase();
+    const needsWrapper = !trimmed.startsWith('<!doctype') &&
+        !trimmed.startsWith('<html') &&
+        !trimmed.startsWith('<body');
+    const htmlToParse = needsWrapper
+        ? `<!DOCTYPE html><html><body>${htmlOrDocument}</body></html>`
         : htmlOrDocument;
     try {
         return parseHTML(htmlToParse).document;
@@ -662,14 +665,6 @@ function resolveHtmlDocument(htmlOrDocument) {
         return parseHTML('<!DOCTYPE html><html><body></body></html>').document;
     }
 }
-function stripNonVisibleNodes(root) {
-    for (const el of root.querySelectorAll('script,style,noscript')) {
-        el.remove();
-    }
-}
-function resolveNodeText(node) {
-    return node.textContent ?? '';
-}
 function getTextContentSkippingHidden(node, parts) {
     const { nodeType } = node;
     if (nodeType === 3) {
@@ -700,8 +695,10 @@ function getTextContentSkippingHidden(node, parts) {
 function getVisibleTextLength(htmlOrDocument) {
     if (typeof htmlOrDocument === 'string') {
         const doc = resolveHtmlDocument(htmlOrDocument);
-        stripNonVisibleNodes(doc.body);
-        return resolveNodeText(doc.body).replace(/\s+/g, ' ').trim().length;
+        for (const el of doc.body.querySelectorAll('script,style,noscript')) {
+            el.remove();
+        }
+        return (doc.body.textContent || '').replace(/\s+/g, ' ').trim().length;
     }
     const parts = [];
     getTextContentSkippingHidden(htmlOrDocument.body, parts);
@@ -765,7 +762,6 @@ function hasTruncatedSentences(text) {
     return incompleteFound / linesFound > MAX_TRUNCATED_LINE_RATIO;
 }
 const MIN_CONTENT_ROOT_LENGTH = 100;
-const HEADING_SCAN_LIMIT = 12;
 const BINARY_SAMPLE_SIZE = 2000;
 export function determineContentExtractionSource(article) {
     return article !== null;
@@ -817,22 +813,6 @@ const PRIMARY_HEADING_ROOT_SELECTORS = [
     '.entry-content',
     '[itemprop="text"]',
 ];
-function normalizeSyntheticTitleToken(value) {
-    return (value ?? '').replace(/\s+/g, ' ').trim().toLowerCase();
-}
-function shouldPreferPrimaryHeadingTitle(primaryHeading, title) {
-    const primary = normalizeSyntheticTitleToken(primaryHeading);
-    if (!primary)
-        return false;
-    const normalizedTitle = normalizeSyntheticTitleToken(title);
-    if (!normalizedTitle)
-        return true;
-    if (normalizedTitle === primary)
-        return true;
-    return normalizedTitle
-        .split(/\s*(?:[-|:•·]|–|—)\s*/u)
-        .some((part) => part === primary);
-}
 function findContentRoot(document) {
     for (const selector of CONTENT_ROOT_SELECTORS) {
         const element = document.querySelector(selector);
@@ -911,93 +891,36 @@ function countEmptyHeadingSections(root) {
     }
     return emptyCount;
 }
-function isGithubRepositoryRootUrl(url) {
-    let parsed;
-    try {
-        parsed = new URL(url);
-    }
-    catch {
-        return false;
-    }
-    const hostname = parsed.hostname.toLowerCase();
-    if (hostname !== 'github.com' && hostname !== 'www.github.com') {
-        return false;
-    }
-    return parsed.pathname.split('/').filter(Boolean).length === 2;
-}
 const TransformHeuristics = {
     findContentRoot,
     findPrimaryHeading,
     isGithubRepositoryRootUrl,
 };
-const ARTICLE_INTERACTIVE_SELECTOR = 'button,[role="tab"],[role="tabpanel"],[aria-controls]';
-function buildArticleDocument(article) {
-    return parseHTML(`<!DOCTYPE html><html><body>${article.content}</body></html>`).document;
-}
-function hasSufficientArticleContentRatio(article, document) {
-    const originalLength = getVisibleTextLength(document);
-    if (originalLength < MIN_HTML_LENGTH_FOR_GATE)
-        return true;
-    return article.textContent.length / originalLength >= MIN_CONTENT_RATIO;
-}
-function retainsEnoughHeadings(articleDoc, document) {
-    const originalHeadings = countMatchingElements(document, 'h1,h2,h3,h4,h5,h6');
-    if (originalHeadings === 0)
-        return true;
-    const articleHeadings = countMatchingElements(articleDoc, 'h1,h2,h3,h4,h5,h6');
-    return articleHeadings / originalHeadings >= MIN_HEADING_RETENTION_RATIO;
-}
-function retainsEnoughCodeBlocks(articleDoc, document) {
-    const originalCodeBlocks = countMatchingElements(document, 'pre');
-    if (originalCodeBlocks === 0)
-        return true;
-    const articleCodeBlocks = countMatchingElements(articleDoc, 'pre');
-    return (articleCodeBlocks / originalCodeBlocks >= MIN_CODE_BLOCK_RETENTION_RATIO);
-}
-function retainsEnoughTables(articleDoc, document) {
-    const originalTables = countMatchingElements(document, 'table');
-    if (originalTables === 0)
-        return true;
-    const articleTables = countMatchingElements(articleDoc, 'table');
-    return articleTables / originalTables >= MIN_TABLE_RETENTION_RATIO;
-}
-function retainsEnoughImages(articleDoc, document) {
-    const originalImages = countMatchingElements(document, 'img');
-    if (originalImages < MIN_IMAGE_ELEMENTS_FOR_GATE)
-        return true;
-    const articleImages = countMatchingElements(articleDoc, 'img');
-    return articleImages / originalImages >= MIN_IMAGE_RETENTION_RATIO;
-}
-function retainsEnoughInteractiveElements(articleDoc, document) {
-    const originalInteractive = countMatchingElements(document, ARTICLE_INTERACTIVE_SELECTOR);
-    if (originalInteractive < MIN_INTERACTIVE_ELEMENTS_FOR_GATE)
-        return true;
-    const articleInteractive = countMatchingElements(articleDoc, ARTICLE_INTERACTIVE_SELECTOR);
-    return (articleInteractive / originalInteractive >= MIN_INTERACTIVE_RETENTION_RATIO);
-}
-function hasAcceptableEmptySectionRatio(articleDoc) {
-    const articleHeadings = countMatchingElements(articleDoc, 'h1,h2,h3,h4,h5,h6');
-    if (articleHeadings < MIN_HEADINGS_FOR_EMPTY_SECTION_GATE)
-        return true;
-    const emptySectionRatio = countEmptyHeadingSections(articleDoc) / articleHeadings;
-    return emptySectionRatio <= MAX_EMPTY_SECTION_RATIO;
-}
 function shouldUseArticleContent(article, document) {
-    if (!hasSufficientArticleContentRatio(article, document))
-        return false;
-    const articleDoc = buildArticleDocument(article);
-    if (!retainsEnoughHeadings(articleDoc, document))
-        return false;
-    if (!retainsEnoughCodeBlocks(articleDoc, document))
-        return false;
-    if (!retainsEnoughTables(articleDoc, document))
-        return false;
-    if (!retainsEnoughImages(articleDoc, document))
-        return false;
-    if (!retainsEnoughInteractiveElements(articleDoc, document))
-        return false;
-    if (!hasAcceptableEmptySectionRatio(articleDoc))
+    // Content ratio gate
+    const originalLength = getVisibleTextLength(document);
+    if (originalLength >= MIN_HTML_LENGTH_FOR_GATE) {
+        if (article.textContent.length / originalLength < MIN_CONTENT_RATIO)
+            return false;
+    }
+    const articleDoc = parseHTML(`<!DOCTYPE html><html><body>${article.content}</body></html>`).document;
+    // Retention checks
+    const passesRetention = RETENTION_RULES.every(({ selector, minOriginal, ratio }) => {
+        const original = countMatchingElements(document, selector);
+        if (original < minOriginal)
+            return true;
+        return countMatchingElements(articleDoc, selector) / original >= ratio;
+    });
+    if (!passesRetention)
         return false;
+    // Empty section ratio
+    const articleHeadings = countMatchingElements(articleDoc, 'h1,h2,h3,h4,h5,h6');
+    if (articleHeadings >= MIN_HEADINGS_FOR_EMPTY_SECTION_GATE) {
+        if (countEmptyHeadingSections(articleDoc) / articleHeadings >
+            MAX_EMPTY_SECTION_RATIO) {
+            return false;
+        }
+    }
     return !hasTruncatedSentences(article.textContent);
 }
 function buildContentSource(params) {
@@ -1008,6 +931,7 @@ function buildContentSource(params) {
         ? TransformHeuristics.findPrimaryHeading(document)
         : undefined;
     if (preparedDocument) {
+        // Post-extraction cleanup on the original document (non-article path)
         prepareDocumentForMarkdown(preparedDocument, url, signal);
         primaryHeading =
             TransformHeuristics.findPrimaryHeading(preparedDocument) ??
@@ -1023,6 +947,8 @@ function buildContentSource(params) {
     };
     if (useArticleContent && article) {
         const { document: articleDoc } = parseHTML(`<!DOCTYPE html><html><body>${article.content}</body></html>`);
+        // Runs on Readability output — tab/noise ops are no-ops here,
+        // but table normalization and URL resolution still apply.
         prepareDocumentForMarkdown(articleDoc, url, signal);
         const articleTitle = article.title !== undefined
             ? normalizeDocumentTitle(article.title, url)
@@ -1061,290 +987,6 @@ function buildContentSource(params) {
         title: extractedMeta.title,
     };
 }
-const NEXT_FLIGHT_PAYLOAD_RE = /self\.__next_f\.push\(\[1,"((?:\\.|[^"\\])*)"\]\)<\/script>/gs;
-const TEMPLATE_ASSIGNMENT_RE = /([A-Za-z_$][\w$]*)=`([\s\S]*?)`;/g;
-const OBJECT_ASSIGNMENT_RE = /([A-Za-z_$][\w$]*)=\{([^{}]+)\}/g;
-const FLIGHT_INSTALL_RE = /commands:\{cli:"([^"]+)",npm:"([^"]+)",yarn:"([^"]+)",pnpm:"([^"]+)",bun:"([^"]+)"\}/;
-const FLIGHT_IMPORT_RE = /commands:\{main:'([^']+)',individual:'([^']+)'\}/;
-const FLIGHT_DEMO_RE = /title:"([^"]+)",files:([A-Za-z_$][\w$]*)\.([A-Za-z_$][\w$]*)/g;
-const FLIGHT_API_RE = /children:"([^"]+)"\}\),`\\n`,\(0,e\.jsx\)\(o,\{data:\[([\s\S]*?)\]\}\)/g;
-const FLIGHT_API_ROW_RE = /attribute:"([^"]+)",type:"([^"]+)",description:"([^"]*)",default:"([^"]*)"/g;
-const FLIGHT_MERMAID_SECTION_RE = /_jsx\(Heading,\{\s*level:"[1-6]",\s*id:"[^"]+",\s*children:"((?:\\.|[^"\\])*)"\s*\}\)(?:(?!_jsx\(Heading,\{)[\s\S]){0,12000}?_jsx\(Mermaid,\{\s*chart:"((?:\\.|[^"\\])*)"\s*\}\)/g;
-function decodeHtmlEntities(value) {
-    return value
-        .replace(/&#39;|&#x27;/g, "'")
-        .replace(/&quot;/g, '"')
-        .replace(/&amp;/g, '&')
-        .replace(/&lt;/g, '<')
-        .replace(/&gt;/g, '>');
-}
-function decodeFlightStringValue(value) {
-    try {
-        return JSON.parse(`"${value}"`);
-    }
-    catch {
-        return decodeHtmlEntities(value);
-    }
-}
-function decodeNextFlightPayloads(html) {
-    const payloads = [];
-    for (const match of html.matchAll(NEXT_FLIGHT_PAYLOAD_RE)) {
-        const rawPayload = match[1];
-        if (!rawPayload)
-            continue;
-        try {
-            payloads.push(JSON.parse(`"${rawPayload}"`));
-        }
-        catch {
-            // Ignore malformed payload fragments and continue with the rest.
-        }
-    }
-    return payloads;
-}
-function parseFlightObjectRefs(text) {
-    const templateMap = new Map();
-    const aliasMap = new Map();
-    const objectMaps = new Map();
-    for (const match of text.matchAll(TEMPLATE_ASSIGNMENT_RE)) {
-        const name = match[1];
-        const code = match[2];
-        if (name && code)
-            templateMap.set(name, decodeHtmlEntities(code));
-    }
-    for (const match of text.matchAll(OBJECT_ASSIGNMENT_RE)) {
-        const objectName = match[1];
-        const body = match[2]?.trim() ?? '';
-        if (!objectName || !body)
-            continue;
-        const spreadMatch = /^\.\.\.([A-Za-z_$][\w$]*)$/.exec(body);
-        if (spreadMatch?.[1]) {
-            aliasMap.set(objectName, spreadMatch[1]);
-            continue;
-        }
-        const entries = new Map();
-        for (const part of body.split(',')) {
-            const entryMatch = /(?:"([^"]+)"|([A-Za-z_$][\w$]*)):([A-Za-z_$][\w$]*)$/.exec(part.trim());
-            const key = entryMatch?.[1] ?? entryMatch?.[2];
-            const value = entryMatch?.[3];
-            if (key && value)
-                entries.set(key, value);
-        }
-        if (entries.size > 0)
-            objectMaps.set(objectName, entries);
-    }
-    return { templateMap, aliasMap, objectMaps };
-}
-function resolveFlightCodeRef(name, refs, seen = new Set()) {
-    if (!name || seen.has(name))
-        return undefined;
-    seen.add(name);
-    const direct = refs.templateMap.get(name);
-    if (direct)
-        return direct;
-    const alias = refs.aliasMap.get(name);
-    if (alias)
-        return resolveFlightCodeRef(alias, refs, seen);
-    const objectMap = refs.objectMaps.get(name);
-    if (!objectMap)
-        return undefined;
-    for (const ref of objectMap.values()) {
-        const resolved = resolveFlightCodeRef(ref, refs, seen);
-        if (resolved)
-            return resolved;
-    }
-    return undefined;
-}
-function escapeMarkdownTableCell(value) {
-    const normalized = decodeHtmlEntities(value).replace(/\s+/g, ' ').trim();
-    return (normalized || '-').replace(/\|/g, '\\|');
-}
-function buildMarkdownTable(rows) {
-    if (rows.length === 0)
-        return '';
-    const lines = [
-        '| Prop | Type | Description | Default |',
-        '| ---- | ---- | ----------- | ------- |',
-    ];
-    for (const row of rows) {
-        lines.push(`| ${escapeMarkdownTableCell(row.attribute)} | ${escapeMarkdownTableCell(row.type)} | ${escapeMarkdownTableCell(row.description)} | ${escapeMarkdownTableCell(row.defaultValue)} |`);
-    }
-    return lines.join('\n');
-}
-function buildCodeBlock(code) {
-    const trimmed = code.trim();
-    if (!trimmed)
-        return '';
-    const language = detectLanguageFromCode(trimmed) ?? 'tsx';
-    return `\`\`\`${language}\n${trimmed}\n\`\`\``;
-}
-function buildMermaidBlock(chart) {
-    const normalized = decodeFlightStringValue(chart).trim();
-    if (!normalized)
-        return '';
-    return `\`\`\`mermaid\n${normalized}\n\`\`\``;
-}
-function normalizeSupplementHeadingText(value) {
-    return value
-        .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
-        .replace(/\s+/g, ' ')
-        .trim()
-        .toLowerCase();
-}
-function getMarkdownHeadingInfo(line) {
-    const match = /^(#{1,6})\s+(.+?)\s*$/.exec(line.trim());
-    if (!match)
-        return null;
-    return {
-        level: match[1]?.length ?? 0,
-        title: normalizeSupplementHeadingText(match[2] ?? ''),
-    };
-}
-function findMarkdownSection(lines, title) {
-    const target = normalizeSupplementHeadingText(title);
-    for (let i = 0; i < lines.length; i += 1) {
-        const heading = getMarkdownHeadingInfo(lines[i] ?? '');
-        if (heading?.title !== target)
-            continue;
-        let end = lines.length;
-        for (let j = i + 1; j < lines.length; j += 1) {
-            const nextLine = lines[j];
-            const nextHeading = nextLine !== undefined ? getMarkdownHeadingInfo(nextLine) : null;
-            if (nextHeading && nextHeading.level <= heading.level) {
-                end = j;
-                break;
-            }
-        }
-        return { start: i, end };
-    }
-    return null;
-}
-function getSectionBody(lines, section) {
-    return lines
-        .slice(section.start + 1, section.end)
-        .join('\n')
-        .trim();
-}
-function replaceMarkdownSection(lines, title, body) {
-    const section = findMarkdownSection(lines, title);
-    if (!section)
-        return false;
-    const replacement = body.trim().length > 0 ? ['', ...body.trim().split('\n'), ''] : [''];
-    lines.splice(section.start + 1, section.end - section.start - 1, ...replacement);
-    return true;
-}
-function appendMarkdownSection(lines, title, body) {
-    const section = findMarkdownSection(lines, title);
-    if (!section)
-        return false;
-    const bodyText = getSectionBody(lines, section);
-    if (bodyText.includes('```'))
-        return false;
-    const nextBody = bodyText ? `${bodyText}\n\n${body.trim()}` : body.trim();
-    return replaceMarkdownSection(lines, title, nextBody);
-}
-function extractNextFlightSupplement(originalHtml) {
-    const payloads = decodeNextFlightPayloads(originalHtml);
-    if (payloads.length === 0)
-        return null;
-    const text = payloads.join('\n');
-    const refs = parseFlightObjectRefs(text);
-    const installMatch = FLIGHT_INSTALL_RE.exec(text);
-    const importMatch = FLIGHT_IMPORT_RE.exec(text);
-    const apiTables = new Map();
-    for (const match of text.matchAll(FLIGHT_API_RE)) {
-        const title = match[1];
-        const rawRows = match[2] ?? '';
-        if (!title)
-            continue;
-        const rows = [];
-        for (const rowMatch of rawRows.matchAll(FLIGHT_API_ROW_RE)) {
-            const attribute = rowMatch[1];
-            const type = rowMatch[2];
-            const description = rowMatch[3];
-            const defaultValue = rowMatch[4];
-            if (!attribute ||
-                !type ||
-                description === undefined ||
-                defaultValue === undefined) {
-                continue;
-            }
-            rows.push({ attribute, type, description, defaultValue });
-        }
-        const table = buildMarkdownTable(rows);
-        if (table)
-            apiTables.set(title, table);
-    }
-    const mermaidDiagrams = new Map();
-    for (const match of text.matchAll(FLIGHT_MERMAID_SECTION_RE)) {
-        const title = match[1] ? decodeFlightStringValue(match[1]).trim() : '';
-        const chart = match[2] ? buildMermaidBlock(match[2]) : '';
-        if (title && chart)
-            mermaidDiagrams.set(title, chart);
-    }
-    const demoCodeBlocks = new Map();
-    for (const match of text.matchAll(FLIGHT_DEMO_RE)) {
-        const title = match[1];
-        const objectName = match[2];
-        const key = match[3];
-        const ref = objectName
-            ? refs.objectMaps.get(objectName)?.get(key ?? '')
-            : undefined;
-        const code = resolveFlightCodeRef(ref, refs);
-        const codeBlock = code ? buildCodeBlock(code) : '';
-        if (title && codeBlock)
-            demoCodeBlocks.set(title, codeBlock);
-    }
-    return {
-        ...(installMatch ? { installationCommands: installMatch.slice(1) } : {}),
-        ...(importMatch ? { importCommands: importMatch.slice(1) } : {}),
-        apiTables,
-        demoCodeBlocks,
-        mermaidDiagrams,
-    };
-}
-function supplementMarkdownFromNextFlight(markdown, originalHtml) {
-    const supplement = extractNextFlightSupplement(originalHtml);
-    if (!supplement)
-        return markdown;
-    const lines = markdown.split('\n');
-    if (supplement.installationCommands?.length) {
-        const installationSection = findMarkdownSection(lines, 'Installation');
-        if (installationSection) {
-            const installBody = getSectionBody(lines, installationSection);
-            if (!/(npm|pnpm|yarn|bun|npx)\s+(install|add)/.test(installBody)) {
-                appendMarkdownSection(lines, 'Installation', buildCodeBlock(supplement.installationCommands.join('\n')));
-            }
-        }
-    }
-    if (supplement.importCommands?.length) {
-        const importSection = findMarkdownSection(lines, 'Import');
-        if (importSection) {
-            const importBody = getSectionBody(lines, importSection);
-            if (!/import\s+\{/.test(importBody)) {
-                appendMarkdownSection(lines, 'Import', buildCodeBlock(supplement.importCommands.join('\n\n')));
-            }
-        }
-    }
-    for (const [title, table] of supplement.apiTables) {
-        replaceMarkdownSection(lines, title, table);
-    }
-    for (const [title, mermaidBlock] of supplement.mermaidDiagrams) {
-        const section = findMarkdownSection(lines, title);
-        if (!section)
-            continue;
-        const sectionBody = getSectionBody(lines, section);
-        if (sectionBody.includes('```mermaid'))
-            continue;
-        const nextBody = sectionBody
-            ? `${sectionBody}\n\n${mermaidBlock}`
-            : mermaidBlock;
-        replaceMarkdownSection(lines, title, nextBody);
-    }
-    for (const [title, codeBlock] of supplement.demoCodeBlocks) {
-        appendMarkdownSection(lines, title, codeBlock);
-    }
-    return lines.join('\n');
-}
 function resolveContentSource(params) {
     const { article, metadata: extractedMeta, document, truncated, } = extractContentContext(params.html, params.url, {
         extractArticle: true,
@@ -1366,33 +1008,6 @@ function resolveContentSource(params) {
         ...(params.signal ? { signal: params.signal } : {}),
     });
 }
-function shouldStripGithubPrimaryHeading(context, url) {
-    return (context.primaryHeading !== undefined &&
-        TransformHeuristics.isGithubRepositoryRootUrl(url));
-}
-function maybeStripGithubPrimaryHeading(markdown, context, url) {
-    if (!shouldStripGithubPrimaryHeading(context, url))
-        return markdown;
-    return stripLeadingHeading(markdown, context.primaryHeading ?? '');
-}
-function buildSyntheticTitlePrefix(url, favicon, suppressFavicon) {
-    if (!favicon || suppressFavicon)
-        return ' ';
-    let alt = '';
-    try {
-        alt = new URL(url).hostname;
-    }
-    catch {
-        /* skip */
-    }
-    return ` ![${alt}](${favicon}) `;
-}
-function maybePrependSyntheticTitle(markdown, context, url) {
-    if (!context.title || /^(#{1,6})\s/.test(markdown.trimStart())) {
-        return markdown;
-    }
-    return `#${buildSyntheticTitlePrefix(url, context.favicon, context.suppressSyntheticFavicon)}${context.title}\n\n${markdown}`;
-}
 function buildMarkdownFromContext(context, url, signal) {
     let content = stageTracker.run(url, 'transform:markdown', () => htmlToMarkdown(context.sourceHtml, context.metadata, {
         url,
@@ -1400,7 +1015,7 @@ function buildMarkdownFromContext(context, url, signal) {
         ...(context.document ? { document: context.document } : {}),
         ...(context.skipNoiseRemoval ? { skipNoiseRemoval: true } : {}),
     }));
-    content = maybeStripGithubPrimaryHeading(content, context, url);
+    content = maybeStripGithubPrimaryHeading(content, context.primaryHeading, url);
     content = maybePrependSyntheticTitle(content, context, url);
     content = supplementMarkdownFromNextFlight(content, context.originalHtml);
     content = cleanupMarkdownArtifacts(content, signal ? { signal, url } : { url });
@@ -1411,34 +1026,6 @@ function buildMarkdownFromContext(context, url, signal) {
         metadata: context.extractedMetadata,
     };
 }
-function normalizeHeadingText(value) {
-    return value.replace(/\s+/g, ' ').trim().toLowerCase();
-}
-function stripLeadingHeading(markdown, headingText) {
-    if (!markdown)
-        return markdown;
-    const lines = markdown.split('\n');
-    const target = normalizeHeadingText(headingText);
-    let nonEmptySeen = 0;
-    for (let i = 0; i < lines.length && nonEmptySeen < HEADING_SCAN_LIMIT; i += 1) {
-        const trimmed = lines[i]?.trim() ?? '';
-        if (!trimmed)
-            continue;
-        nonEmptySeen += 1;
-        const match = /^(#{1,6})\s+(.+?)\s*$/.exec(trimmed);
-        if (!match)
-            continue;
-        const current = normalizeHeadingText(match[2] ?? '');
-        if (current !== target)
-            return markdown;
-        lines.splice(i, 1);
-        if ((lines[i] ?? '').trim() === '') {
-            lines.splice(i, 1);
-        }
-        return lines.join('\n');
-    }
-    return markdown;
-}
 const REPLACEMENT_CHAR = '\ufffd';
 const BINARY_INDICATOR_THRESHOLD = 0.1;
 function hasBinaryIndicators(content) {