@j0hanz/fetch-url-mcp 1.12.7 → 1.12.8

package/dist/transform/transform.js

@@ -1,1042 +0,0 @@
-import diagnosticsChannel from 'node:diagnostics_channel';
-import { isProbablyReaderable, Readability } from '@mozilla/readability';
-import { parseHTML } from 'linkedom';
-import { config, getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
-import { SystemErrors } from '../lib/error-codes.js';
-import { isRawTextContentUrl } from '../lib/http.js';
-import { Loggers } from '../lib/logger-names.js';
-import { composeAbortSignal, FetchError, getErrorMessage, getUtf8ByteLength, isAsciiOnly, isObject, throwIfAborted, toError, trimDanglingTagFragment, truncateToUtf8Boundary, } from '../lib/utils.js';
-import { evaluateArticleContent, extractNoscriptImages, getVisibleTextLength, normalizeTabContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, stripDocsControls, stripScreenReaderText, surfaceCodeEditorContent, } from './dom-prep.js';
-import { extractLanguageFromClassName } from './html-translators.js';
-import { translateHtmlFragmentToMarkdown } from './html-translators.js';
-import { cleanupMarkdownArtifacts, finalizeMarkdownSections, processFencedContent, } from './markdown-cleanup.js';
-import { addSourceToMarkdown, buildMetadataFooter, extractTitleFromRawMarkdown, isRawTextContent, } from './metadata.js';
-import { extractMetadata, extractMetadataFromHead, isGithubRepositoryRootUrl, maybePrependSyntheticTitle, maybeStripGithubPrimaryHeading, mergeMetadata, normalizeDocumentTitle, shouldPreferPrimaryHeadingTitle, } from './metadata.js';
-import { supplementMarkdownFromNextFlight } from './next-flight.js';
-import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
-function decodeInput(input, encoding) {
-    if (typeof input === 'string')
-        return input;
-    const normalizedEncoding = encoding?.trim().toLowerCase();
-    if (!normalizedEncoding ||
-        normalizedEncoding === 'utf-8' ||
-        normalizedEncoding === 'utf8') {
-        const decoded = new TextDecoder('utf-8').decode(input);
-        return decoded;
-    }
-    try {
-        const decoded = new TextDecoder(normalizedEncoding, { fatal: true }).decode(input);
-        return decoded;
-    }
-    catch {
-        const decoded = new TextDecoder('utf-8').decode(input);
-        return decoded;
-    }
-}
-function buildTransformSignal(signal) {
-    return composeAbortSignal(signal, config.transform.timeoutMs);
-}
-class StageTracker {
-    channel = diagnosticsChannel.channel('fetch-url-mcp.transform');
-    start(url, stage, budget) {
-        if (this.shouldSkipTracking(budget))
-            return null;
-        const remainingBudgetMs = budget
-            ? budget.totalBudgetMs - budget.elapsedMs
-            : undefined;
-        const base = {
-            stage,
-            startTime: performance.now(),
-            url: redactUrl(url),
-        };
-        if (remainingBudgetMs !== undefined && budget) {
-            return {
-                ...base,
-                budgetMs: remainingBudgetMs,
-                totalBudgetMs: budget.totalBudgetMs,
-            };
-        }
-        return base;
-    }
-    end(context, options) {
-        if (!context)
-            return 0;
-        const durationMs = performance.now() - context.startTime;
-        const requestId = getRequestId();
-        const operationId = getOperationId();
-        if (context.totalBudgetMs !== undefined) {
-            const warnThresholdMs = context.totalBudgetMs * config.transform.stageWarnRatio;
-            if (durationMs > warnThresholdMs) {
-                logWarn('Transform stage exceeded warning threshold', {
-                    stage: context.stage,
-                    durationMs: Math.round(durationMs),
-                    thresholdMs: Math.round(warnThresholdMs),
-                    url: context.url,
-                }, Loggers.LOG_TRANSFORM);
-            }
-        }
-        const event = {
-            v: 1,
-            type: 'stage',
-            stage: context.stage,
-            durationMs,
-            url: context.url,
-            ...(requestId ? { requestId } : {}),
-            ...(operationId ? { operationId } : {}),
-            ...(options?.truncated !== undefined
-                ? { truncated: options.truncated }
-                : {}),
-        };
-        this.publish(event);
-        return durationMs;
-    }
-    checkBudget(url, stage, budget) {
-        if (budget && budget.elapsedMs >= budget.totalBudgetMs) {
-            const error = new FetchError('Transform budget exhausted', url, 504, {
-                reason: 'timeout',
-                stage: `${stage}:budget_exhausted`,
-                elapsedMs: budget.elapsedMs,
-                totalBudgetMs: budget.totalBudgetMs,
-            });
-            throw error;
-        }
-    }
-    run(url, stage, fn, budget) {
-        if (this.shouldSkipTracking(budget)) {
-            return fn();
-        }
-        this.checkBudget(url, stage, budget);
-        const ctx = this.start(url, stage, budget);
-        try {
-            return fn();
-        }
-        finally {
-            this.end(ctx);
-        }
-    }
-    async runAsync(url, stage, fn, budget) {
-        if (this.shouldSkipTracking(budget)) {
-            return fn();
-        }
-        this.checkBudget(url, stage, budget);
-        const ctx = this.start(url, stage, budget);
-        try {
-            return await fn();
-        }
-        finally {
-            this.end(ctx);
-        }
-    }
-    shouldSkipTracking(budget) {
-        return !this.channel.hasSubscribers && !budget;
-    }
-    publish(event) {
-        if (!this.channel.hasSubscribers)
-            return;
-        try {
-            this.channel.publish(event);
-        }
-        catch (error) {
-            logDebug('Diagnostic channel publish failed', {
-                stage: event.stage,
-                error: getErrorMessage(error),
-            }, Loggers.LOG_TRANSFORM);
-        }
-    }
-    runTrackedSync(url, signal, fn) {
-        const totalStage = this.start(url, 'transform:total');
-        try {
-            throwIfAborted(signal, url, 'transform:begin');
-            const result = fn();
-            this.end(totalStage, result.truncated !== undefined
-                ? { truncated: result.truncated }
-                : undefined);
-            return result;
-        }
-        catch (error) {
-            this.end(totalStage);
-            throw error;
-        }
-    }
-    async runTrackedAsync(url, signal, fn) {
-        const totalStage = this.start(url, 'transform:total');
-        try {
-            throwIfAborted(signal, url, 'transform:begin');
-            const result = await fn();
-            this.end(totalStage, result.truncated !== undefined
-                ? { truncated: result.truncated }
-                : undefined);
-            return result;
-        }
-        catch (error) {
-            this.end(totalStage);
-            throw error;
-        }
-    }
-}
-const stageTracker = new StageTracker();
-export function startTransformStage(url, stage, budget) {
-    return stageTracker.start(url, stage, budget);
-}
-export function endTransformStage(context, options) {
-    return stageTracker.end(context, options);
-}
-function truncateHtml(html, inputTruncated = false) {
-    const maxSize = config.constants.maxHtmlBytes;
-    if (maxSize <= 0)
-        return { html, truncated: false };
-    if (html.length <= maxSize) {
-        if (isAsciiOnly(html) && !inputTruncated)
-            return { html, truncated: false };
-        const byteLength = getUtf8ByteLength(html);
-        if (byteLength <= maxSize && !inputTruncated)
-            return { html, truncated: false };
-    }
-    const sliced = html.slice(0, maxSize);
-    if (getUtf8ByteLength(sliced) <= maxSize) {
-        return { html: trimDanglingTagFragment(sliced), truncated: true };
-    }
-    const content = truncateToUtf8Boundary(sliced, maxSize);
-    logWarn('HTML content exceeds maximum size, truncating', {
-        size: getUtf8ByteLength(html),
-        maxSize,
-        truncatedSize: getUtf8ByteLength(content),
-    }, Loggers.LOG_TRANSFORM);
-    return { html: content, truncated: true };
-}
-const MIN_SPA_CONTENT_LENGTH = 100;
-const MIN_READERABLE_TEXT_LENGTH = 400;
-const MAX_READABILITY_ELEMENTS = 20_000;
-function isReadabilityCompatible(doc) {
-    if (!isObject(doc))
-        return false;
-    const { querySelectorAll, querySelector } = doc;
-    return ('documentElement' in doc &&
-        typeof querySelectorAll === 'function' &&
-        typeof querySelector === 'function');
-}
-function getNormalizedTextLengthUpTo(text, max) {
-    if (max <= 0)
-        return 0;
-    let length = 0;
-    let seenNonWhitespace = false;
-    let pendingSpace = false;
-    for (let i = 0; i < text.length; i += 1) {
-        const code = text.charCodeAt(i);
-        const isWhitespace = code <= 0x20;
-        if (isWhitespace) {
-            if (seenNonWhitespace)
-                pendingSpace = true;
-            continue;
-        }
-        if (!seenNonWhitespace) {
-            seenNonWhitespace = true;
-        }
-        else if (pendingSpace) {
-            length += 1;
-            pendingSpace = false;
-            if (length >= max)
-                return length;
-        }
-        length += 1;
-        if (length >= max)
-            return length;
-    }
-    return length;
-}
-function preserveGalleryImages(doc) {
-    const galleries = doc.querySelectorAll('[class*="gallery"],[class*="slideshow"],[class*="carousel"]');
-    for (const gallery of galleries) {
-        const images = gallery.querySelectorAll('img');
-        if (images.length === 0)
-            continue;
-        const fragment = doc.createDocumentFragment();
-        for (const img of images) {
-            const figure = doc.createElement('figure');
-            figure.appendChild(img.cloneNode(true));
-            fragment.appendChild(figure);
-        }
-        gallery.replaceWith(fragment);
-    }
-}
-function preserveAlertElements(doc) {
-    const alerts = doc.querySelectorAll('[role="alert"], .admonition, [class*="callout"]');
-    for (const el of alerts) {
-        const bq = doc.createElement('blockquote');
-        bq.innerHTML = el.innerHTML;
-        el.replaceWith(bq);
-    }
-}
-function preserveHeadingLayouts(doc) {
-    // Readability aggressively drops elements matching /header/i in their class/id.
-    // Many technical docs use `<div class="layout__header">` to wrap their title and intro text,
-    // causing the ENTIRE intro and H1 to be dropped.
-    for (const heading of doc.querySelectorAll('h1, h2')) {
-        let p = heading.parentNode;
-        while (p && p.tagName !== 'BODY' && p.tagName !== 'HTML') {
-            const cls = p.getAttribute('class');
-            if (cls && /header/i.test(cls)) {
-                p.setAttribute('class', cls.replace(/header/gi, 'hdr-preserved'));
-            }
-            const id = p.getAttribute('id');
-            if (id && /header/i.test(id)) {
-                p.setAttribute('id', id.replace(/header/gi, 'hdr-preserved'));
-            }
-            p = p.parentNode;
-        }
-    }
-    // To prevent Readability from penalizing sibling document sections
-    // (e.g. intro vs reference tables) and picking only one, we unwrap structural wrappers inside main boundaries.
-    for (const main of doc.querySelectorAll('main, [role="main"], article')) {
-        for (const child of Array.from(main.children)) {
-            // Don't unwrap nav, aside, or blockquotes (alerts are already converted to blockquotes here)
-            if (child.tagName === 'DIV' ||
-                child.tagName === 'HEADER' ||
-                child.tagName === 'SECTION') {
-                // preserve specific structural features Readability might want to keep
-                const cls = child.getAttribute('class') ?? '';
-                if (cls.includes('mermaid'))
-                    continue;
-                const frag = doc.createDocumentFragment();
-                while (child.firstChild) {
-                    frag.appendChild(child.firstChild);
-                }
-                child.replaceWith(frag);
-            }
-        }
-    }
-}
-function preserveCodeLanguageAttributes(doc) {
-    for (const el of doc.querySelectorAll('pre, code')) {
-        if (el.getAttribute('data-language'))
-            continue;
-        const lang = extractLanguageFromClassName(el.getAttribute('class') ?? '');
-        if (lang)
-            el.setAttribute('data-language', lang);
-    }
-}
-function prepareReadabilityDocument(readabilityDoc) {
-    extractNoscriptImages(readabilityDoc);
-    preserveGalleryImages(readabilityDoc);
-    preserveAlertElements(readabilityDoc);
-    preserveHeadingLayouts(readabilityDoc);
-    preserveCodeLanguageAttributes(readabilityDoc);
-    normalizeTabContent(readabilityDoc);
-    surfaceCodeEditorContent(readabilityDoc);
-    stripDocsControls(readabilityDoc);
-    stripScreenReaderText(readabilityDoc);
-    for (const el of readabilityDoc.querySelectorAll('[class*="breadcrumb"],[class*="pagination"]')) {
-        if (el.tagName === 'HTML' || el.tagName === 'BODY')
-            continue;
-        el.remove();
-    }
-}
-function validateReaderability(doc, url, signal) {
-    throwIfAborted(signal, url, 'extract:article:textCheck');
-    const rawText = doc.querySelector('body')?.textContent ??
-        doc.documentElement.textContent ??
-        '';
-    const textLength = getNormalizedTextLengthUpTo(rawText, MIN_READERABLE_TEXT_LENGTH + 1);
-    if (textLength < MIN_SPA_CONTENT_LENGTH) {
-        logWarn('Very minimal server-rendered content detected (< 100 chars). ' +
-            'This might be a client-side rendered (SPA) application. ' +
-            'Content extraction may be incomplete.', { textLength }, Loggers.LOG_TRANSFORM);
-    }
-    throwIfAborted(signal, url, 'extract:article:readabilityCheck');
-    if (textLength >= MIN_READERABLE_TEXT_LENGTH && !isProbablyReaderable(doc)) {
-        return false;
-    }
-    return true;
-}
-function invokeReadability(doc, url, signal) {
-    throwIfAborted(signal, url, 'extract:article:clone');
-    const readabilityDoc = typeof doc.cloneNode === 'function'
-        ? doc.cloneNode(true)
-        : doc;
-    prepareReadabilityDocument(readabilityDoc);
-    throwIfAborted(signal, url, 'extract:article:parse');
-    const reader = new Readability(readabilityDoc, {
-        charThreshold: 140,
-        maxElemsToParse: MAX_READABILITY_ELEMENTS,
-        classesToPreserve: [
-            'admonition',
-            'callout',
-            'custom-block',
-            'alert',
-            'note',
-            'tip',
-            'info',
-            'warning',
-            'danger',
-            'caution',
-            'important',
-            'mermaid',
-        ],
-    });
-    return reader.parse();
-}
-function mapReadabilityResult(parsed) {
-    return {
-        content: parsed.content ?? '',
-        textContent: parsed.textContent ?? '',
-        ...(parsed.title != null && { title: parsed.title }),
-        ...(parsed.byline != null && { byline: parsed.byline }),
-        ...(parsed.excerpt != null && { excerpt: parsed.excerpt }),
-        ...(parsed.siteName != null && { siteName: parsed.siteName }),
-    };
-}
-// Pre-Readability cleanup on a cloned document.
-// Must strip tabs/breadcrumbs before Readability mangles role attributes.
-// The original document is NOT yet prepared (prepareDocumentForMarkdown
-// runs later in buildContentSource), so this clone starts from raw HTML.
-function extractArticle(document, url, signal) {
-    if (!isReadabilityCompatible(document)) {
-        logWarn('Document not compatible with Readability', undefined, Loggers.LOG_TRANSFORM);
-        return null;
-    }
-    try {
-        if (!validateReaderability(document, url, signal)) {
-            return null;
-        }
-        const parsed = invokeReadability(document, url, signal);
-        if (!parsed)
-            return null;
-        return mapReadabilityResult(parsed);
-    }
-    catch (error) {
-        logError('Failed to extract article with Readability', error instanceof Error ? error : undefined, Loggers.LOG_TRANSFORM);
-        return null;
-    }
-}
-function isValidInput(html, url) {
-    if (typeof html !== 'string' || html.length === 0) {
-        logWarn('extractContent called with invalid HTML input', undefined, Loggers.LOG_TRANSFORM);
-        return false;
-    }
-    if (typeof url !== 'string' || url.length === 0) {
-        logWarn('extractContent called with invalid URL', undefined, Loggers.LOG_TRANSFORM);
-        return false;
-    }
-    return true;
-}
-function applyBaseUri(document, url) {
-    try {
-        Object.defineProperty(document, 'baseURI', { value: url, writable: true });
-    }
-    catch (error) {
-        logInfo('Failed to set baseURI (non-critical)', {
-            url: url.substring(0, 100),
-            error: getErrorMessage(error),
-        }, Loggers.LOG_TRANSFORM);
-    }
-}
-function createEmptyExtractionContext() {
-    const { document } = parseHTML('<html></html>');
-    return { article: null, metadata: {}, document };
-}
-function extractEarlyMetadataIfNeeded(html, url) {
-    const maxSize = config.constants.maxHtmlBytes;
-    if (maxSize <= 0)
-        return null;
-    if (html.length <= maxSize &&
-        (isAsciiOnly(html) || getUtf8ByteLength(html) <= maxSize)) {
-        return null;
-    }
-    return stageTracker.run(url, 'extract:early-metadata', () => extractMetadataFromHead(html, url));
-}
-function parseExtractionDocument(html, url, inputTruncated) {
-    const { html: limitedHtml, truncated } = truncateHtml(html, inputTruncated);
-    const { document } = stageTracker.run(url, 'extract:parse', () => parseHTML(limitedHtml));
-    return { document, truncated };
-}
-function extractMergedMetadata(html, url, document) {
-    const earlyMetadata = extractEarlyMetadataIfNeeded(html, url);
-    const lateMetadata = stageTracker.run(url, 'extract:metadata', () => extractMetadata(document, url));
-    return mergeMetadata(earlyMetadata, lateMetadata);
-}
-function extractArticleIfRequested(document, url, options) {
-    if (!options.extractArticle)
-        return null;
-    return stageTracker.run(url, 'extract:article', () => extractArticle(document, url, options.signal));
-}
-function extractContentContext(html, url, options) {
-    if (!isValidInput(html, url)) {
-        return createEmptyExtractionContext();
-    }
-    try {
-        throwIfAborted(options.signal, url, 'extract:begin');
-        const { document, truncated } = parseExtractionDocument(html, url, options.inputTruncated);
-        throwIfAborted(options.signal, url, 'extract:parsed');
-        applyBaseUri(document, url);
-        const metadata = extractMergedMetadata(html, url, document);
-        throwIfAborted(options.signal, url, 'extract:metadata');
-        const article = extractArticleIfRequested(document, url, options);
-        throwIfAborted(options.signal, url, 'extract:article');
-        return {
-            article,
-            metadata,
-            document,
-            ...(truncated ? { truncated: true } : {}),
-        };
-    }
-    catch (error) {
-        if (error instanceof FetchError)
-            throw error;
-        throwIfAborted(options.signal, url, 'extract:error');
-        logError('Failed to extract content', error instanceof Error ? error : undefined, Loggers.LOG_TRANSFORM);
-        return createEmptyExtractionContext();
-    }
-}
-export function extractContent(html, url, options = {
-    extractArticle: true,
-}) {
-    const result = extractContentContext(html, url, options);
-    return { article: result.article, metadata: result.metadata };
-}
-function resolveRelativeHref(href, baseUrl, origin) {
-    const trimmedHref = href.trim();
-    if (!trimmedHref || /[\t\n\f\r ]/.test(trimmedHref))
-        return href;
-    if (isAbsoluteOrSpecialUrl(trimmedHref))
-        return trimmedHref;
-    const resolved = URL.parse(trimmedHref, baseUrl);
-    if (resolved)
-        return resolved.href;
-    if (trimmedHref.startsWith('/'))
-        return `${origin}${trimmedHref}`;
-    return trimmedHref;
-}
-function findBalancedCloseParen(text, start) {
-    let depth = 1;
-    for (let i = start; i < text.length; i++) {
-        const ch = text[i];
-        if (ch === '(') {
-            depth++;
-        }
-        else if (ch === ')') {
-            depth--;
-            if (depth === 0)
-                return i;
-        }
-    }
-    return -1;
-}
-function findInlineLink(markdown, start) {
-    let openBracket = markdown.indexOf('[', start);
-    while (openBracket !== -1) {
-        const closeBracket = markdown.indexOf(']', openBracket + 1);
-        if (closeBracket === -1)
-            return null;
-        if (markdown[closeBracket + 1] !== '(') {
-            openBracket = markdown.indexOf('[', closeBracket + 1);
-            continue;
-        }
-        const closeParen = findBalancedCloseParen(markdown, closeBracket + 2);
-        if (closeParen === -1)
-            return null;
-        const isImage = openBracket > 0 && markdown[openBracket - 1] === '!';
-        const prefixStart = isImage ? openBracket - 1 : openBracket;
-        return {
-            prefixStart,
-            closeParen,
-            prefix: markdown.slice(prefixStart, closeBracket + 1),
-            href: markdown.slice(closeBracket + 2, closeParen),
-        };
-    }
-    return null;
-}
-function isAbsoluteOrSpecialUrl(href) {
-    const trimmedHref = href.trim();
-    if (!trimmedHref)
-        return false;
-    if (trimmedHref.startsWith('#'))
-        return true;
-    return URL.canParse(trimmedHref);
-}
-function resolveRelativeUrlsInSegment(markdown, baseUrl, origin) {
-    let cursor = 0;
-    const parts = [];
-    while (cursor < markdown.length) {
-        const link = findInlineLink(markdown, cursor);
-        if (!link) {
-            parts.push(markdown.slice(cursor));
-            break;
-        }
-        parts.push(markdown.slice(cursor, link.prefixStart));
-        parts.push(`${link.prefix}(${resolveRelativeHref(link.href, baseUrl, origin)})`);
-        cursor = link.closeParen + 1;
-    }
-    return parts.join('');
-}
-function resolveRelativeUrls(markdown, baseUrl, signal) {
-    const parsedBase = URL.parse(baseUrl);
-    if (!parsedBase)
-        return markdown;
-    const { origin } = parsedBase;
-    if (!markdown)
-        return markdown;
-    return processFencedContent(markdown, (text) => {
-        throwIfAborted(signal, baseUrl, 'markdown:resolve-urls');
-        return resolveRelativeUrlsInSegment(text, baseUrl, origin);
-    });
-}
-function translateHtmlToMarkdown(params) {
-    const { html, url, signal, document, skipNoiseRemoval } = params;
-    throwIfAborted(signal, url, 'markdown:begin');
-    const cleanedHtml = skipNoiseRemoval
-        ? html
-        : stageTracker.run(url, 'markdown:noise', () => removeNoiseFromHtml(html, document, url, signal));
-    throwIfAborted(signal, url, 'markdown:cleaned');
-    const content = stageTracker.run(url, 'markdown:translate', () => translateHtmlFragmentToMarkdown(cleanedHtml));
-    throwIfAborted(signal, url, 'markdown:translated');
-    const cleaned = cleanupMarkdownArtifacts(content, signal
-        ? { preserveEmptyHeadings: true, signal, url }
-        : { preserveEmptyHeadings: true, url });
-    return url ? resolveRelativeUrls(cleaned, url, signal) : cleaned;
-}
-function appendMetadataFooter(content, metadata, url) {
-    const footer = buildMetadataFooter(metadata, url);
-    if (!content.trim() && footer) {
-        const note = '> **Note:** This page contains no readable content. It may require JavaScript to render.\n\n';
-        return `${note}${footer}`;
-    }
-    return footer ? `${content}\n\n${footer}` : content;
-}
-export function htmlToMarkdown(html, metadata, options) {
-    const url = options?.url ?? metadata?.url ?? '';
-    if (!html)
-        return buildMetadataFooter(metadata, url);
-    try {
-        const content = translateHtmlToMarkdown({
-            html,
-            url,
-            signal: options?.signal,
-            document: options?.document,
-            skipNoiseRemoval: options?.skipNoiseRemoval,
-        });
-        return appendMetadataFooter(content, metadata, url);
-    }
-    catch (error) {
-        if (error instanceof FetchError)
-            throw error;
-        logError('Failed to convert HTML to markdown', error instanceof Error ? error : undefined, Loggers.LOG_TRANSFORM);
-        const fetchError = new FetchError('Failed to convert HTML to markdown', url, 500, {
-            reason: 'markdown_convert_failed',
-        });
-        throw fetchError;
-    }
-}
-const HTML_DOCUMENT_START = /^\s*<(?:!doctype|html|head|body)\b/i;
-const STRUCTURAL_HTML_TAGS = /<(?:html|head|body|div|p|span|section|article|main|nav|footer|header)\b/i;
-function shouldPreserveRawContent(url, content) {
-    if (isRawTextContentUrl(url)) {
-        return !HTML_DOCUMENT_START.test(content.trim());
-    }
-    if (!isRawTextContent(content))
-        return false;
-    return !STRUCTURAL_HTML_TAGS.test(content);
-}
-function buildRawMarkdownPayload(params) {
-    const title = extractTitleFromRawMarkdown(params.rawContent);
-    let content = params.includeMetadataFooter
-        ? addSourceToMarkdown(params.rawContent, params.url)
-        : params.rawContent;
-    if (params.url) {
-        content = resolveRelativeUrls(content, params.url);
-    }
-    return { content, title };
-}
-function tryTransformRawContent(params) {
-    if (!shouldPreserveRawContent(params.url, params.html))
-        return null;
-    logDebug('Preserving raw markdown content', {
-        url: params.url.substring(0, 80),
-    }, Loggers.LOG_TRANSFORM);
-    const { content, title } = buildRawMarkdownPayload({
-        rawContent: params.html,
-        url: params.url,
-        includeMetadataFooter: params.includeMetadataFooter,
-    });
-    return {
-        markdown: content,
-        title,
-        truncated: params.inputTruncated ?? false,
-    };
-}
-const MIN_CONTENT_RATIO = 0.15;
-const MIN_HTML_LENGTH_FOR_GATE = 100;
-export function isExtractionSufficient(article, originalHtmlOrDocument) {
-    if (!article)
-        return false;
-    const articleLength = article.textContent.length;
-    const originalLength = getVisibleTextLength(originalHtmlOrDocument);
-    if (originalLength < MIN_HTML_LENGTH_FOR_GATE)
-        return true;
-    return articleLength / originalLength >= MIN_CONTENT_RATIO;
-}
-const MIN_CONTENT_ROOT_LENGTH = 100;
-const BINARY_SAMPLE_SIZE = 2000;
-export function determineContentExtractionSource(article) {
-    return article !== null;
-}
-export function createContentMetadataBlock(url, article, extractedMeta, shouldExtractFromArticle, includeMetadataFooter) {
-    if (!includeMetadataFooter)
-        return undefined;
-    const metadata = {
-        type: 'metadata',
-        url,
-        fetchedAt: new Date().toISOString(),
-    };
-    if (shouldExtractFromArticle && article) {
-        if (article.title !== undefined) {
-            metadata.title = normalizeDocumentTitle(article.title, url);
-        }
-        if (article.byline !== undefined)
-            metadata.author = article.byline;
-    }
-    else {
-        if (extractedMeta.title !== undefined)
-            metadata.title = extractedMeta.title;
-        if (extractedMeta.description !== undefined)
-            metadata.description = extractedMeta.description;
-        if (extractedMeta.author !== undefined)
-            metadata.author = extractedMeta.author;
-    }
-    return metadata;
-}
-function prepareContentSourceDocument(document, url, signal) {
-    const initialPrimaryHeading = TransformHeuristics.findPrimaryHeading(document);
-    prepareDocumentForMarkdown(document, url, signal);
-    return {
-        document,
-        primaryHeading: TransformHeuristics.findPrimaryHeading(document) ?? initialPrimaryHeading,
-    };
-}
-function resolveContentTitle(params) {
-    const resolvedTitle = (params.preferPrimaryHeading ? params.primaryHeading : undefined) ??
-        params.title;
-    return {
-        title: resolvedTitle,
-    };
-}
-function resolveSourceTitle(base, candidateTitle, url) {
-    return resolveContentTitle({
-        primaryHeading: base.primaryHeading,
-        title: candidateTitle,
-        preferPrimaryHeading: TransformHeuristics.isGithubRepositoryRootUrl(url) ||
-            shouldPreferPrimaryHeadingTitle(base.primaryHeading, candidateTitle),
-    });
-}
-const CONTENT_REGION_SELECTORS = [
-    'article',
-    'main',
-    '[role="main"]',
-    '#content',
-    '#main-content',
-    '.content',
-    '.main-content',
-    '.post-content',
-    '.article-content',
-    '.entry-content',
-    '[itemprop="articleBody"]',
-    '[data-content]',
-    '.post-body',
-    '.article-body',
-];
-const HEADING_REGION_EXTRA_SELECTORS = [
-    '.markdown-body',
-    '[itemprop="text"]',
-];
-function findContentRoot(document) {
-    for (const selector of CONTENT_REGION_SELECTORS) {
-        const element = document.querySelector(selector);
-        if (!element)
-            continue;
-        const innerHTML = typeof element.innerHTML === 'string'
-            ? element.innerHTML
-            : undefined;
-        if (innerHTML && innerHTML.trim().length > MIN_CONTENT_ROOT_LENGTH)
-            return innerHTML;
-    }
-    return undefined;
-}
-const PRIMARY_HEADING_SELECTORS_GLOBAL = ['[data-title="true"]', 'h1'];
-const PRIMARY_HEADING_SELECTORS_LOCAL = [
-    '[data-title="true"]',
-    'h1',
-    'h2',
-];
-function extractHeadingText(root, selectors) {
-    for (const selector of selectors) {
-        const heading = root.querySelector(selector);
-        if (!heading)
-            continue;
-        const text = heading.textContent.trim();
-        if (text)
-            return text;
-    }
-    return undefined;
-}
-function findPrimaryHeading(document) {
-    const globalHeading = extractHeadingText(document, PRIMARY_HEADING_SELECTORS_GLOBAL);
-    if (globalHeading)
-        return globalHeading;
-    for (const selector of [
-        ...CONTENT_REGION_SELECTORS,
-        ...HEADING_REGION_EXTRA_SELECTORS,
-    ]) {
-        const root = document.querySelector(selector);
-        if (!root)
-            continue;
-        const localHeading = extractHeadingText(root, PRIMARY_HEADING_SELECTORS_LOCAL);
-        if (localHeading)
-            return localHeading;
-    }
-    return undefined;
-}
-const TransformHeuristics = {
-    findContentRoot,
-    findPrimaryHeading,
-    isGithubRepositoryRootUrl,
-};
-function buildArticleSource(base, params) {
-    const { evaluatedArticleDoc, article, extractedMeta, url, signal } = params;
-    prepareDocumentForMarkdown(evaluatedArticleDoc, url, signal);
-    const articleTitle = article.title !== undefined
-        ? normalizeDocumentTitle(article.title, url)
-        : extractedMeta.title;
-    const title = resolveSourceTitle(base, articleTitle, url);
-    return {
-        ...base,
-        sourceHtml: evaluatedArticleDoc.body.innerHTML,
-        ...title,
-        skipNoiseRemoval: true,
-    };
-}
-function buildDocumentSource(base, params) {
-    const { resolvedDocument, html, extractedMeta, url } = params;
-    const contentRoot = TransformHeuristics.findContentRoot(resolvedDocument);
-    const title = resolveSourceTitle(base, extractedMeta.title, url);
-    return {
-        ...base,
-        sourceHtml: contentRoot ?? serializeDocumentForMarkdown(resolvedDocument, html),
-        ...title,
-        skipNoiseRemoval: true,
-        document: resolvedDocument,
-    };
-}
-function buildRawSource(base, params) {
-    return {
-        ...base,
-        sourceHtml: params.html,
-        title: params.extractedMeta.title,
-    };
-}
-function resolveBaseContentSource(input) {
-    const { html, url, article, extractedMeta, includeMetadataFooter, evaluatedArticleDoc, document, truncated, signal, } = input;
-    const metadata = createContentMetadataBlock(url, article, extractedMeta, evaluatedArticleDoc !== null, includeMetadataFooter);
-    const preparedDocument = document
-        ? prepareContentSourceDocument(document, url, signal)
-        : undefined;
-    const base = {
-        favicon: extractedMeta.favicon,
-        metadata,
-        extractedMetadata: extractedMeta,
-        truncated,
-        primaryHeading: preparedDocument?.primaryHeading,
-        originalHtml: html,
-    };
-    return { base, preparedDocument };
-}
-function buildContentSource(input) {
-    const { base, preparedDocument } = resolveBaseContentSource(input);
-    if (input.evaluatedArticleDoc && input.article) {
-        return buildArticleSource(base, {
-            evaluatedArticleDoc: input.evaluatedArticleDoc,
-            article: input.article,
-            extractedMeta: input.extractedMeta,
-            url: input.url,
-            signal: input.signal,
-        });
-    }
-    if (preparedDocument) {
-        return buildDocumentSource(base, {
-            resolvedDocument: preparedDocument.document,
-            html: input.html,
-            extractedMeta: input.extractedMeta,
-            url: input.url,
-        });
-    }
-    return buildRawSource(base, {
-        html: input.html,
-        extractedMeta: input.extractedMeta,
-    });
-}
-function resolveContentSource(params) {
-    const { article, metadata: extractedMeta, document, truncated, } = extractContentContext(params.html, params.url, {
-        extractArticle: true,
-        signal: params.signal,
-        inputTruncated: params.inputTruncated,
-    });
-    const evaluatedArticleDoc = article
-        ? evaluateArticleContent(article, document)
-        : null;
-    return buildContentSource({
-        html: params.html,
-        url: params.url,
-        article,
-        extractedMeta,
-        includeMetadataFooter: params.includeMetadataFooter,
-        evaluatedArticleDoc,
-        document,
-        truncated: truncated ?? false,
-        signal: params.signal,
-    });
-}
-function renderMarkdownStage({ context, url, signal, }) {
-    return stageTracker.run(url, 'transform:markdown', () => htmlToMarkdown(context.sourceHtml, context.metadata, {
-        url,
-        signal,
-        document: context.document,
-        skipNoiseRemoval: context.skipNoiseRemoval,
-    }));
-}
-function postprocessMarkdownStage({ context, url, signal }, markdown) {
-    let content = maybeStripGithubPrimaryHeading(markdown, context.primaryHeading, url);
-    content = maybePrependSyntheticTitle(content, context);
-    content = supplementMarkdownFromNextFlight(content, context.originalHtml);
-    content = finalizeMarkdownSections(content, signal ? { signal, url } : { url });
-    return {
-        markdown: content,
-        title: context.title,
-        truncated: context.truncated,
-        metadata: context.extractedMetadata,
-    };
-}
-function buildMarkdownFromContext(context, url, signal) {
-    const renderContext = { context, url, signal };
-    const markdown = renderMarkdownStage(renderContext);
-    return postprocessMarkdownStage(renderContext, markdown);
-}
-function resolveTransformContentResult(html, url, options, signal) {
-    const rawResult = stageTracker.run(url, 'transform:raw', () => tryTransformRawContent({
-        html,
-        url,
-        includeMetadataFooter: options.includeMetadataFooter,
-        inputTruncated: options.inputTruncated,
-    }));
-    if (rawResult)
-        return rawResult;
-    const context = stageTracker.run(url, 'transform:extract', () => resolveContentSource({
-        html,
-        url,
-        includeMetadataFooter: options.includeMetadataFooter,
-        signal,
-        inputTruncated: options.inputTruncated,
-    }));
-    return buildMarkdownFromContext(context, url, signal);
-}
-const REPLACEMENT_CHAR = '\ufffd';
-const BINARY_INDICATOR_THRESHOLD = 0.1;
-function hasBinaryIndicators(content) {
-    if (!content)
-        return false;
-    if (content.includes('\x00'))
-        return true;
-    const sampleSize = Math.min(content.length, BINARY_SAMPLE_SIZE);
-    let replacementCount = 0;
-    let i = -1;
-    while ((i = content.indexOf(REPLACEMENT_CHAR, i + 1)) !== -1 &&
-        i < sampleSize) {
-        replacementCount++;
-    }
-    return replacementCount > sampleSize * BINARY_INDICATOR_THRESHOLD;
-}
-export function transformHtmlToMarkdownInProcess(html, url, options) {
-    const signal = buildTransformSignal(options.signal);
-    return stageTracker.runTrackedSync(url, signal, () => {
-        validateBinaryContent(html, url);
-        return resolveTransformContentResult(html, url, options, signal);
-    });
-}
-function validateBinaryContent(html, url) {
-    if (hasBinaryIndicators(html)) {
-        const error = new FetchError('Content appears to be binary data (high replacement character ratio or null bytes)', url, 415, { reason: 'binary_content_detected', stage: 'transform:validate' });
-        throw error;
-    }
-}
-export function getTransformPoolStats() {
-    return getWorkerPoolStats();
-}
-export async function shutdownTransformWorkerPool() {
-    await shutdownWorkerPool();
-}
-function transformInputInProcess(htmlOrBuffer, url, options) {
-    return transformHtmlToMarkdownInProcess(decodeInput(htmlOrBuffer, options.encoding), url, options);
-}
-function workerTransformOptions(options) {
-    return {
-        includeMetadataFooter: options.includeMetadataFooter,
-        ...(options.signal ? { signal: options.signal } : {}),
-        ...(options.inputTruncated
-            ? { inputTruncated: options.inputTruncated }
-            : {}),
-    };
-}
-async function transformWithWorkerPool(htmlOrBuffer, url, options) {
-    const pool = getOrCreateWorkerPool();
-    if (pool.getCapacity() === 0) {
-        return transformInputInProcess(htmlOrBuffer, url, options);
-    }
-    if (typeof htmlOrBuffer === 'string') {
-        return pool.transform(htmlOrBuffer, url, workerTransformOptions(options));
-    }
-    return pool.transform(htmlOrBuffer, url, {
-        ...workerTransformOptions(options),
-        ...(options.encoding ? { encoding: options.encoding } : {}),
-    });
-}
-function resolveWorkerFallback(error, htmlOrBuffer, url, options) {
-    const poolStats = getWorkerPoolStats();
-    const isQueueFull = error instanceof FetchError &&
-        error.details['reason'] === SystemErrors.QUEUE_FULL;
-    if (isQueueFull) {
-        logWarn('Transform worker queue full; falling back to in-process', {
-            url: redactUrl(url),
-            ...(poolStats ?? {}),
-        }, Loggers.LOG_TRANSFORM);
-        return transformInputInProcess(htmlOrBuffer, url, options);
-    }
-    throwIfAborted(options.signal, url, 'transform:worker-fallback');
-    if (error instanceof FetchError)
-        throw error;
-    if (!(error instanceof Error))
-        throw toError(error);
-    const message = getErrorMessage(error);
-    logWarn('Transform worker failed; falling back to in-process', {
-        url: redactUrl(url),
-        error: message,
-        ...(poolStats ?? {}),
-    }, Loggers.LOG_TRANSFORM);
-    return transformInputInProcess(htmlOrBuffer, url, options);
-}
-async function runWorkerTransformWithFallback(htmlOrBuffer, url, options) {
-    return stageTracker.runAsync(url, 'transform:worker', async () => {
-        try {
-            return await transformWithWorkerPool(htmlOrBuffer, url, options);
-        }
-        catch (error) {
-            return resolveWorkerFallback(error, htmlOrBuffer, url, options);
-        }
-    });
-}
-async function transformInputToMarkdown(htmlOrBuffer, url, options) {
-    return stageTracker.runTrackedAsync(url, options.signal, () => runWorkerTransformWithFallback(htmlOrBuffer, url, options));
-}
-export async function transformHtmlToMarkdown(html, url, options) {
-    return transformInputToMarkdown(html, url, options);
-}
-export async function transformBufferToMarkdown(htmlBuffer, url, options) {
-    return transformInputToMarkdown(htmlBuffer, url, options);
-}
-export { cleanupMarkdownArtifacts, finalizeMarkdownSections, processFencedContent, };