@j0hanz/fetch-url-mcp 1.3.1 → 1.4.0

package/dist/transform/transform.js

@@ -4,14 +4,14 @@ import { performance } from 'node:perf_hooks';
 import { isProbablyReaderable, Readability } from '@mozilla/readability';
 import { parseHTML } from 'linkedom';
 import { NodeHtmlMarkdown, } from 'node-html-markdown';
-import { config } from '../config.js';
-import { removeNoiseFromHtml } from '../dom-noise-removal.js';
-import { FetchError, getErrorMessage } from '../errors.js';
-import { isRawTextContentUrl } from '../fetch.js';
-import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../language-detection.js';
-import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../markdown-cleanup.js';
-import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../observability.js';
-import { isLikeNode, isObject } from '../type-guards.js';
+import { config } from '../lib/config.js';
+import { removeNoiseFromHtml } from '../lib/dom-noise-removal.js';
+import { FetchError, getErrorMessage } from '../lib/errors.js';
+import { isRawTextContentUrl } from '../lib/fetch.js';
+import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../lib/language-detection.js';
+import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/markdown-cleanup.js';
+import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/observability.js';
+import { isLikeNode, isObject } from '../lib/type-guards.js';
 import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
 const utf8Decoder = new TextDecoder('utf-8');
 function decodeInput(input, encoding) {
@@ -213,7 +213,11 @@ function trimUtf8Buffer(buffer, maxBytes) {
 function trimDanglingTagFragment(content) {
     const lastOpen = content.lastIndexOf('<');
     const lastClose = content.lastIndexOf('>');
-    return lastOpen > lastClose ? content.substring(0, lastOpen) : content;
+    if (lastOpen > lastClose &&
+        /^<([a-zA-Z/!?]|$)/.test(content.substring(lastOpen))) {
+        return content.substring(0, lastOpen);
+    }
+    return content;
 }
 function truncateHtml(html, inputTruncated = false) {
     const maxSize = config.constants.maxHtmlSize;
@@ -270,24 +274,19 @@ function mergeMetadata(early, late) {
     if (!early)
         return late;
     const merged = {};
-    const title = late.title ?? early.title;
-    const description = late.description ?? early.description;
-    const author = late.author ?? early.author;
-    const image = late.image ?? early.image;
-    const publishedAt = late.publishedAt ?? early.publishedAt;
-    const modifiedAt = late.modifiedAt ?? early.modifiedAt;
-    if (title !== undefined)
-        merged.title = title;
-    if (description !== undefined)
-        merged.description = description;
-    if (author !== undefined)
-        merged.author = author;
-    if (image !== undefined)
-        merged.image = image;
-    if (publishedAt !== undefined)
-        merged.publishedAt = publishedAt;
-    if (modifiedAt !== undefined)
-        merged.modifiedAt = modifiedAt;
+    const keys = [
+        'title',
+        'description',
+        'author',
+        'image',
+        'publishedAt',
+        'modifiedAt',
+    ];
+    for (const key of keys) {
+        const value = late[key] ?? early[key];
+        if (value !== undefined)
+            merged[key] = value;
+    }
     return merged;
 }
 const META_PROPERTY_HANDLERS = new Map([
@@ -653,9 +652,6 @@ function isCodeBlock(parent) {
     const tagName = getTagName(parent);
     return tagName === 'PRE' || tagName === 'WRAPPED-PRE';
 }
-function isAnchor(node) {
-    return getTagName(node) === 'A';
-}
 function resolveAttributeLanguage(node) {
     const getAttribute = hasGetAttribute(node)
         ? node.getAttribute.bind(node)
@@ -762,7 +758,7 @@ function resolveImageSrc(getAttribute) {
 function buildImageTranslator(ctx) {
     if (!isObject(ctx))
         return { content: '' };
-    const { node, parent } = ctx;
+    const { node } = ctx;
     const getAttribute = hasGetAttribute(node)
         ? node.getAttribute.bind(node)
         : undefined;
@@ -770,10 +766,7 @@ function buildImageTranslator(ctx) {
     const existingAlt = getAttribute?.('alt') ?? '';
     const alt = existingAlt.trim() || deriveAltFromImageUrl(src);
     const markdown = `![${alt}](${src})`;
-    if (isAnchor(parent)) {
-        return { content: markdown };
-    }
-    return { content: `\n\n${markdown}\n\n` };
+    return { content: markdown };
 }
 const GFM_ALERT_MAP = new Map([
     ['note', 'NOTE'],
@@ -831,6 +824,112 @@ function buildPreTranslator(ctx) {
         postprocess: createCodeBlockPostprocessor(attributeLanguage),
     };
 }
+function getNodeAttr(node) {
+    if (!isLikeNode(node))
+        return undefined;
+    return typeof node.getAttribute === 'function'
+        ? node.getAttribute.bind(node)
+        : undefined;
+}
+function buildDivTranslator(ctx) {
+    if (!isObject(ctx))
+        return {};
+    const { node } = ctx;
+    const getAttribute = getNodeAttr(node);
+    if (!getAttribute)
+        return {};
+    const className = getAttribute('class') ?? '';
+    if (className.includes('mermaid')) {
+        return {
+            noEscape: true,
+            preserveWhitespace: true,
+            postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
+        };
+    }
+    const isAdmonition = className.includes('admonition') ||
+        className.includes('callout') ||
+        className.includes('custom-block') ||
+        getAttribute('role') === 'alert' ||
+        /\b(note|tip|info|warning|danger|caution|important)\b/i.test(className);
+    if (isAdmonition) {
+        return {
+            postprocess: ({ content }) => {
+                const alertType = resolveGfmAlertType(className);
+                const lines = content.trim().split('\n');
+                const header = alertType ? `> [!${alertType}]\n` : '';
+                return `\n\n${header}> ${lines.join('\n> ')}\n\n`;
+            },
+        };
+    }
+    if (!className.includes('type'))
+        return {};
+    return {
+        postprocess: ({ content }) => {
+            const lines = content.split('\n');
+            const separated = [];
+            for (let i = 0; i < lines.length; i++) {
+                const line = lines[i] ?? '';
+                const nextLine = i < lines.length - 1 ? (lines[i + 1] ?? '') : '';
+                separated.push(line);
+                if (line.trim() &&
+                    nextLine.trim() &&
+                    line.includes(':') &&
+                    nextLine.includes(':') &&
+                    !line.startsWith(' ') &&
+                    !nextLine.startsWith(' ')) {
+                    separated.push('');
+                }
+            }
+            return separated.join('\n');
+        },
+    };
+}
+function buildSectionTranslator(ctx) {
+    if (isObject(ctx)) {
+        const { node } = ctx;
+        const getAttribute = getNodeAttr(node);
+        if (getAttribute?.('class')?.includes('tsd-member')) {
+            return {
+                postprocess: ({ content }) => `\n\n&nbsp;\n\n${content}\n\n`,
+            };
+        }
+    }
+    return {
+        postprocess: ({ content }) => `\n\n${content}\n\n`,
+    };
+}
+function buildSpanTranslator(ctx) {
+    if (!isObject(ctx))
+        return {};
+    const { node } = ctx;
+    const getAttribute = getNodeAttr(node);
+    if (!getAttribute)
+        return {};
+    const dataAs = getAttribute('data-as') ?? '';
+    if (dataAs === 'p') {
+        return {
+            postprocess: ({ content }) => `\n\n${content.trim()}\n\n`,
+        };
+    }
+    return {};
+}
+function buildMermaidPreTranslator(ctx) {
+    if (!isObject(ctx))
+        return buildPreTranslator(ctx);
+    const { node } = ctx;
+    const getAttribute = getNodeAttr(node);
+    if (!getAttribute)
+        return buildPreTranslator(ctx);
+    const className = getAttribute('class') ?? '';
+    if (className.includes('mermaid')) {
+        return {
+            noEscape: true,
+            preserveWhitespace: true,
+            postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
+        };
+    }
+    return buildPreTranslator(ctx);
+}
 function createCustomTranslators() {
     return {
         code: (ctx) => buildCodeTranslator(ctx),
@@ -866,61 +965,7 @@ function createCustomTranslators() {
             }
             return { content: items ? `\n${items}\n` : '' };
         },
-        div: (ctx) => {
-            if (!isObject(ctx))
-                return {};
-            const { node } = ctx;
-            if (!isLikeNode(node))
-                return {};
-            const getAttribute = typeof node.getAttribute === 'function'
-                ? node.getAttribute.bind(node)
-                : undefined;
-            const className = getAttribute?.('class') ?? '';
-            if (className.includes('mermaid')) {
-                return {
-                    noEscape: true,
-                    preserveWhitespace: true,
-                    postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
-                };
-            }
-            const isAdmonition = className.includes('admonition') ||
-                className.includes('callout') ||
-                className.includes('custom-block') ||
-                getAttribute?.('role') === 'alert' ||
-                /\b(note|tip|info|warning|danger|caution|important)\b/i.test(className);
-            if (isAdmonition) {
-                return {
-                    postprocess: ({ content }) => {
-                        const alertType = resolveGfmAlertType(className);
-                        const lines = content.trim().split('\n');
-                        const header = alertType ? `> [!${alertType}]\n` : '';
-                        return `\n\n${header}> ${lines.join('\n> ')}\n\n`;
-                    },
-                };
-            }
-            if (!className.includes('type'))
-                return {};
-            return {
-                postprocess: ({ content }) => {
-                    const lines = content.split('\n');
-                    const separated = [];
-                    for (let i = 0; i < lines.length; i++) {
-                        const line = lines[i] ?? '';
-                        const nextLine = i < lines.length - 1 ? (lines[i + 1] ?? '') : '';
-                        separated.push(line);
-                        if (line.trim() &&
-                            nextLine.trim() &&
-                            line.includes(':') &&
-                            nextLine.includes(':') &&
-                            !line.startsWith(' ') &&
-                            !nextLine.startsWith(' ')) {
-                            separated.push('');
-                        }
-                    }
-                    return separated.join('\n');
-                },
-            };
-        },
+        div: buildDivTranslator,
         kbd: () => ({
             postprocess: ({ content }) => `\`${content}\``,
         }),
@@ -933,24 +978,7 @@ function createCustomTranslators() {
         sup: () => ({
             postprocess: ({ content }) => `^${content}^`,
         }),
-        section: (ctx) => {
-            if (isObject(ctx)) {
-                const { node } = ctx;
-                if (isLikeNode(node)) {
-                    const getAttribute = typeof node.getAttribute === 'function'
-                        ? node.getAttribute.bind(node)
-                        : undefined;
-                    if (getAttribute?.('class')?.includes('tsd-member')) {
-                        return {
-                            postprocess: ({ content }) => `\n\n&nbsp;\n\n${content}\n\n`,
-                        };
-                    }
-                }
-            }
-            return {
-                postprocess: ({ content }) => `\n\n${content}\n\n`,
-            };
-        },
+        section: buildSectionTranslator,
         details: () => ({
             postprocess: ({ content }) => {
                 const trimmed = content.trim();
@@ -962,43 +990,8 @@ function createCustomTranslators() {
         summary: () => ({
             postprocess: ({ content }) => `${content.trim()}\n\n`,
         }),
-        span: (ctx) => {
-            if (!isObject(ctx))
-                return {};
-            const { node } = ctx;
-            if (!isLikeNode(node))
-                return {};
-            const getAttribute = typeof node.getAttribute === 'function'
-                ? node.getAttribute.bind(node)
-                : undefined;
-            const dataAs = getAttribute?.('data-as') ?? '';
-            if (dataAs === 'p') {
-                return {
-                    postprocess: ({ content }) => `\n\n${content.trim()}\n\n`,
-                };
-            }
-            return {};
-        },
-        pre: (ctx) => {
-            if (!isObject(ctx))
-                return buildPreTranslator(ctx);
-            const { node } = ctx;
-            if (!isLikeNode(node)) {
-                return buildPreTranslator(ctx);
-            }
-            const getAttribute = typeof node.getAttribute === 'function'
-                ? node.getAttribute.bind(node)
-                : undefined;
-            const className = getAttribute?.('class') ?? '';
-            if (className.includes('mermaid')) {
-                return {
-                    noEscape: true,
-                    preserveWhitespace: true,
-                    postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
-                };
-            }
-            return buildPreTranslator(ctx);
-        },
+        span: buildSpanTranslator,
+        pre: buildMermaidPreTranslator,
     };
 }
 let markdownConverter = null;
@@ -1127,8 +1120,8 @@ function resolveRelativeUrls(markdown, baseUrl, signal) {
     let lastIndex = 0;
     let lineCount = 0;
     while (lastIndex < len) {
-        if (++lineCount % 500 === 0 && signal?.aborted) {
-            throw new Error('Transform aborted during URL resolution');
+        if (++lineCount % 500 === 0) {
+            abortPolicy.throwIfAborted(signal, baseUrl, 'markdown:resolve-urls');
         }
         let nextIndex = markdown.indexOf('\n', lastIndex);
         let line;
@@ -1315,7 +1308,12 @@ function getTextContentSkippingHidden(node, parts) {
     }
     if (nodeType !== 1)
         return;
-    const { tagName } = node;
+    const element = node;
+    if (element.hasAttribute('hidden') ||
+        element.getAttribute('aria-hidden') === 'true') {
+        return;
+    }
+    const { tagName } = element;
     if (tagName === 'SCRIPT' || tagName === 'STYLE' || tagName === 'NOSCRIPT')
         return;
     const { childNodes } = node;
@@ -1479,19 +1477,21 @@ function shouldUseArticleContent(article, originalHtmlOrDocument) {
 function buildContentSource(params) {
     const { html, url, article, extractedMeta, includeMetadata, useArticleContent, document, truncated, skipNoiseRemoval, signal, } = params;
     const metadata = createContentMetadataBlock(url, article, extractedMeta, useArticleContent, includeMetadata);
+    const base = {
+        favicon: extractedMeta.favicon,
+        metadata,
+        extractedMetadata: extractedMeta,
+        truncated,
+    };
     if (useArticleContent && article) {
-        // Readability output can still be noisy (unless user requested skip).
         const cleanedArticleHtml = skipNoiseRemoval
             ? article.content
             : removeNoiseFromHtml(article.content, undefined, url, signal);
         return {
+            ...base,
             sourceHtml: cleanedArticleHtml,
             title: article.title,
-            favicon: extractedMeta.favicon,
-            metadata,
-            extractedMetadata: extractedMeta,
             skipNoiseRemoval: true,
-            truncated,
         };
     }
     if (document) {
@@ -1499,36 +1499,18 @@ function buildContentSource(params) {
             ? html
             : removeNoiseFromHtml(html, document, url, signal);
         const contentRoot = findContentRoot(document);
-        if (contentRoot) {
-            return {
-                sourceHtml: contentRoot,
-                title: extractedMeta.title,
-                favicon: extractedMeta.favicon,
-                metadata,
-                extractedMetadata: extractedMeta,
-                skipNoiseRemoval: true,
-                document,
-                truncated,
-            };
-        }
         return {
-            sourceHtml: cleanedHtml,
+            ...base,
+            sourceHtml: contentRoot ?? cleanedHtml,
             title: extractedMeta.title,
-            favicon: extractedMeta.favicon,
-            metadata,
-            extractedMetadata: extractedMeta,
             skipNoiseRemoval: true,
             document,
-            truncated,
         };
     }
     return {
+        ...base,
         sourceHtml: html,
         title: extractedMeta.title,
-        favicon: extractedMeta.favicon,
-        metadata,
-        extractedMetadata: extractedMeta,
-        truncated,
     };
 }
 function resolveContentSource(params) {