@j0hanz/fetch-url-mcp 1.4.0 → 1.5.0

package/dist/transform/html-translators.js

@@ -0,0 +1,454 @@
+import { NodeHtmlMarkdown, } from 'node-html-markdown';
+import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../lib/content.js';
+import { isLikeNode, isObject } from '../lib/utils.js';
+// ---------------------------------------------------------------------------
+// Shared constant
+// ---------------------------------------------------------------------------
+const CODE_BLOCK = {
+    fence: '```',
+    format: (code, language = '') => `\`\`\`${language}\n${code}\n\`\`\``,
+};
+// ---------------------------------------------------------------------------
+// DOM helpers (translator-only)
+// ---------------------------------------------------------------------------
+function getTagName(node) {
+    if (!isLikeNode(node))
+        return '';
+    const raw = node.tagName;
+    return typeof raw === 'string' ? raw.toUpperCase() : '';
+}
+function hasGetAttribute(value) {
+    return (isObject(value) &&
+        typeof value.getAttribute === 'function');
+}
+function getNodeAttr(node) {
+    if (!isLikeNode(node))
+        return undefined;
+    return typeof node.getAttribute === 'function'
+        ? node.getAttribute.bind(node)
+        : undefined;
+}
+// ---------------------------------------------------------------------------
+// Code translators
+// ---------------------------------------------------------------------------
+function buildInlineCode(content) {
+    const trimmed = content.trim();
+    if (!trimmed)
+        return '``';
+    let maxBackticks = 0;
+    let currentRun = 0;
+    for (const char of trimmed) {
+        if (char === '`')
+            currentRun += 1;
+        else {
+            if (currentRun > maxBackticks)
+                maxBackticks = currentRun;
+            currentRun = 0;
+        }
+    }
+    if (currentRun > maxBackticks)
+        maxBackticks = currentRun;
+    const delimiter = '`'.repeat(maxBackticks + 1);
+    const padding = trimmed.startsWith('`') || trimmed.endsWith('`') ? ' ' : '';
+    return `${delimiter}${padding}${trimmed}${padding}${delimiter}`;
+}
+function isCodeBlock(parent) {
+    const tagName = getTagName(parent);
+    return tagName === 'PRE' || tagName === 'WRAPPED-PRE';
+}
+function resolveAttributeLanguage(node) {
+    const getAttribute = hasGetAttribute(node)
+        ? node.getAttribute.bind(node)
+        : undefined;
+    const className = getAttribute?.('class') ?? '';
+    const dataLanguage = getAttribute?.('data-language') ?? '';
+    return resolveLanguageFromAttributes(className, dataLanguage);
+}
+function findLanguageFromCodeChild(node) {
+    if (!isLikeNode(node))
+        return undefined;
+    const childNodes = Array.from(node.childNodes ?? []);
+    for (const child of childNodes) {
+        if (!isLikeNode(child))
+            continue;
+        const raw = child.rawTagName;
+        const tagName = typeof raw === 'string' ? raw.toUpperCase() : '';
+        if (tagName === 'CODE')
+            return resolveAttributeLanguage(child);
+    }
+    return undefined;
+}
+function createCodeBlockPostprocessor(language) {
+    return ({ content }) => {
+        const trimmed = content.trim();
+        if (!trimmed)
+            return '';
+        const resolvedLanguage = language ?? detectLanguageFromCode(trimmed) ?? '';
+        return CODE_BLOCK.format(trimmed, resolvedLanguage);
+    };
+}
+function buildInlineCodeTranslator() {
+    return {
+        spaceIfRepeatingChar: true,
+        noEscape: true,
+        postprocess: ({ content }) => buildInlineCode(content),
+    };
+}
+function buildCodeTranslator(ctx) {
+    const inlineCodeTranslator = buildInlineCodeTranslator();
+    if (!isObject(ctx))
+        return inlineCodeTranslator;
+    const { parent } = ctx;
+    if (!isCodeBlock(parent))
+        return inlineCodeTranslator;
+    return { noEscape: true, preserveWhitespace: true };
+}
+// ---------------------------------------------------------------------------
+// Image translators
+// ---------------------------------------------------------------------------
+function extractFirstSrcsetUrl(srcset) {
+    const first = srcset.split(',')[0];
+    if (!first)
+        return '';
+    return first.trim().split(/\s+/)[0] ?? '';
+}
+const LAZY_SRC_ATTRIBUTES = [
+    'data-src',
+    'data-lazy-src',
+    'data-original',
+    'data-srcset',
+];
+function isDataUri(value) {
+    return value.startsWith('data:');
+}
+function extractNonDataSrcsetUrl(value) {
+    const url = extractFirstSrcsetUrl(value);
+    return url && !isDataUri(url) ? url : undefined;
+}
+function resolveLazySrc(getAttribute) {
+    for (const attr of LAZY_SRC_ATTRIBUTES) {
+        const lazy = getAttribute(attr);
+        if (!lazy || isDataUri(lazy))
+            continue;
+        if (attr === 'data-srcset') {
+            const url = extractNonDataSrcsetUrl(lazy);
+            if (url)
+                return url;
+            continue;
+        }
+        return lazy;
+    }
+    return undefined;
+}
+function resolveImageSrc(getAttribute) {
+    if (!getAttribute)
+        return '';
+    const srcRaw = getAttribute('src') ?? '';
+    if (srcRaw && !isDataUri(srcRaw))
+        return srcRaw;
+    // First check common lazy-loading attributes that may contain non-data URLs before falling back to the native srcset, as some sites use data URIs in lazy attributes while still providing valid URLs in srcset.
+    const lazySrc = resolveLazySrc(getAttribute);
+    if (lazySrc)
+        return lazySrc;
+    // If the src is a data URI or missing, check srcset for a valid URL. Some sites use srcset with data URIs in src and actual URLs in srcset for responsive images.
+    const srcset = getAttribute('srcset');
+    if (srcset) {
+        const url = extractNonDataSrcsetUrl(srcset);
+        if (url)
+            return url;
+    }
+    // If the only available src is a data URI, we choose to omit it rather than include the raw data in the alt text or URL, as data URIs can be very long and are not useful in Markdown output.
+    if (isDataUri(srcRaw))
+        return '[data URI removed]';
+    return '';
+}
+function deriveAltFromImageUrl(src) {
+    if (!src)
+        return '';
+    try {
+        const isAbsolute = URL.canParse(src);
+        let parsed = null;
+        if (isAbsolute) {
+            parsed = new URL(src);
+        }
+        else if (URL.canParse(src, 'http://localhost')) {
+            parsed = new URL(src, 'http://localhost');
+        }
+        if (!parsed)
+            return '';
+        if (isAbsolute &&
+            parsed.protocol !== 'http:' &&
+            parsed.protocol !== 'https:') {
+            return '';
+        }
+        const { pathname } = parsed;
+        const segments = pathname.split('/');
+        const filename = segments.pop() ?? '';
+        if (!filename)
+            return '';
+        const dotIndex = filename.lastIndexOf('.');
+        const name = dotIndex > 0 ? filename.slice(0, dotIndex) : filename;
+        return name.replace(/[_-]+/g, ' ').trim();
+    }
+    catch {
+        return '';
+    }
+}
+function buildImageTranslator(ctx) {
+    if (!isObject(ctx))
+        return { content: '' };
+    const { node } = ctx;
+    const getAttribute = hasGetAttribute(node)
+        ? node.getAttribute.bind(node)
+        : undefined;
+    const src = resolveImageSrc(getAttribute);
+    const existingAlt = getAttribute?.('alt') ?? '';
+    const alt = existingAlt.trim() || deriveAltFromImageUrl(src);
+    const markdown = `![${alt}](${src})`;
+    return { content: markdown };
+}
+// ---------------------------------------------------------------------------
+// Pre / Mermaid translators
+// ---------------------------------------------------------------------------
+function buildPreTranslator(ctx) {
+    if (!isObject(ctx))
+        return {};
+    const { node } = ctx;
+    const attributeLanguage = resolveAttributeLanguage(node) ?? findLanguageFromCodeChild(node);
+    return {
+        noEscape: true,
+        preserveWhitespace: true,
+        postprocess: createCodeBlockPostprocessor(attributeLanguage),
+    };
+}
+function buildMermaidPreTranslator(ctx) {
+    if (!isObject(ctx))
+        return buildPreTranslator(ctx);
+    const { node } = ctx;
+    const getAttribute = getNodeAttr(node);
+    if (!getAttribute)
+        return buildPreTranslator(ctx);
+    const className = getAttribute('class') ?? '';
+    if (className.includes('mermaid')) {
+        return {
+            noEscape: true,
+            preserveWhitespace: true,
+            postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
+        };
+    }
+    return buildPreTranslator(ctx);
+}
+// ---------------------------------------------------------------------------
+// Block-level translators (div, section, span, table, dl, etc.)
+// ---------------------------------------------------------------------------
+const GFM_ALERT_MAP = new Map([
+    ['note', 'NOTE'],
+    ['info', 'NOTE'],
+    ['tip', 'TIP'],
+    ['hint', 'TIP'],
+    ['warning', 'WARNING'],
+    ['warn', 'WARNING'],
+    ['caution', 'CAUTION'],
+    ['danger', 'CAUTION'],
+    ['important', 'IMPORTANT'],
+]);
+function resolveGfmAlertType(className) {
+    const lower = className.toLowerCase();
+    for (const [key, type] of GFM_ALERT_MAP) {
+        if (lower.includes(key))
+            return type;
+    }
+    return undefined;
+}
+function buildDivTranslator(ctx) {
+    if (!isObject(ctx))
+        return {};
+    const { node } = ctx;
+    const getAttribute = getNodeAttr(node);
+    if (!getAttribute)
+        return {};
+    const className = getAttribute('class') ?? '';
+    if (className.includes('mermaid')) {
+        return {
+            noEscape: true,
+            preserveWhitespace: true,
+            postprocess: ({ content }) => `\n\n\`\`\`mermaid\n${content.trim()}\n\`\`\`\n\n`,
+        };
+    }
+    const isAdmonition = className.includes('admonition') ||
+        className.includes('callout') ||
+        className.includes('custom-block') ||
+        getAttribute('role') === 'alert' ||
+        /\b(note|tip|info|warning|danger|caution|important)\b/i.test(className);
+    if (isAdmonition) {
+        return {
+            postprocess: ({ content }) => {
+                const alertType = resolveGfmAlertType(className);
+                const lines = content.trim().split('\n');
+                const header = alertType ? `> [!${alertType}]\n` : '';
+                return `\n\n${header}> ${lines.join('\n> ')}\n\n`;
+            },
+        };
+    }
+    if (!className.includes('type'))
+        return {};
+    return {
+        postprocess: ({ content }) => {
+            const lines = content.split('\n');
+            const separated = [];
+            for (let i = 0; i < lines.length; i++) {
+                const line = lines[i] ?? '';
+                const nextLine = i < lines.length - 1 ? (lines[i + 1] ?? '') : '';
+                separated.push(line);
+                if (line.trim() &&
+                    nextLine.trim() &&
+                    line.includes(':') &&
+                    nextLine.includes(':') &&
+                    !line.startsWith(' ') &&
+                    !nextLine.startsWith(' ')) {
+                    separated.push('');
+                }
+            }
+            return separated.join('\n');
+        },
+    };
+}
+function buildSectionTranslator(ctx) {
+    if (isObject(ctx)) {
+        const { node } = ctx;
+        const getAttribute = getNodeAttr(node);
+        if (getAttribute?.('class')?.includes('tsd-member')) {
+            return {
+                postprocess: ({ content }) => `\n\n&nbsp;\n\n${content}\n\n`,
+            };
+        }
+    }
+    return {
+        postprocess: ({ content }) => `\n\n${content}\n\n`,
+    };
+}
+function buildSpanTranslator(ctx) {
+    if (!isObject(ctx))
+        return {};
+    const { node } = ctx;
+    const getAttribute = getNodeAttr(node);
+    if (!getAttribute)
+        return {};
+    const dataAs = getAttribute('data-as') ?? '';
+    if (dataAs === 'p') {
+        return {
+            postprocess: ({ content }) => `\n\n${content.trim()}\n\n`,
+        };
+    }
+    return {};
+}
+// ---------------------------------------------------------------------------
+// Table / DL helpers
+// ---------------------------------------------------------------------------
+function hasComplexTableLayout(node) {
+    if (!isLikeNode(node))
+        return false;
+    const innerHTML = typeof node.innerHTML === 'string' ? node.innerHTML : '';
+    return /(?:colspan|rowspan)=["']?[2-9]/i.test(innerHTML);
+}
+function resolveDlNodeName(child) {
+    if (!isLikeNode(child))
+        return '';
+    const raw = child.nodeName;
+    return typeof raw === 'string' ? raw.toUpperCase() : '';
+}
+function resolveDlTextContent(child) {
+    if (!isLikeNode(child))
+        return '';
+    const raw = child.textContent;
+    return typeof raw === 'string' ? raw.trim() : '';
+}
+function buildDlChildFragment(child) {
+    const nodeName = resolveDlNodeName(child);
+    if (nodeName === 'DT')
+        return `**${resolveDlTextContent(child)}**\n`;
+    if (nodeName === 'DD')
+        return `: ${resolveDlTextContent(child)}\n`;
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Translator registry + converter singleton
+// ---------------------------------------------------------------------------
+function createCustomTranslators() {
+    return {
+        code: (ctx) => buildCodeTranslator(ctx),
+        img: (ctx) => buildImageTranslator(ctx),
+        table: (ctx) => {
+            if (!isObject(ctx))
+                return {};
+            const { node } = ctx;
+            if (hasComplexTableLayout(node)) {
+                return {
+                    postprocess: ({ content }) => {
+                        const trimmed = content.trim();
+                        if (!trimmed)
+                            return '';
+                        return `\n\n${trimmed}\n\n`;
+                    },
+                };
+            }
+            return {};
+        },
+        dl: (ctx) => {
+            if (!isObject(ctx))
+                return { content: '' };
+            const { node } = ctx;
+            if (!isLikeNode(node))
+                return { content: '' };
+            const childNodes = Array.from(node.childNodes ?? []);
+            let items = '';
+            for (const child of childNodes) {
+                const fragment = buildDlChildFragment(child);
+                if (fragment !== null)
+                    items += fragment;
+            }
+            return { content: items ? `\n${items}\n` : '' };
+        },
+        div: buildDivTranslator,
+        kbd: () => ({
+            postprocess: ({ content }) => `\`${content}\``,
+        }),
+        mark: () => ({
+            postprocess: ({ content }) => `==${content}==`,
+        }),
+        sub: () => ({
+            postprocess: ({ content }) => `~${content}~`,
+        }),
+        sup: () => ({
+            postprocess: ({ content }) => `^${content}^`,
+        }),
+        section: buildSectionTranslator,
+        details: () => ({
+            postprocess: ({ content }) => {
+                const trimmed = content.trim();
+                if (!trimmed)
+                    return '';
+                return `\n\n${trimmed}\n\n`;
+            },
+        }),
+        summary: () => ({
+            postprocess: ({ content }) => `${content.trim()}\n\n`,
+        }),
+        span: buildSpanTranslator,
+        pre: buildMermaidPreTranslator,
+    };
+}
+let markdownConverter = null;
+function getMarkdownConverter() {
+    markdownConverter ??= new NodeHtmlMarkdown({
+        codeFence: CODE_BLOCK.fence,
+        codeBlockStyle: 'fenced',
+        emDelimiter: '_',
+        bulletMarker: '-',
+        globalEscape: [/[\\`*_~]/gm, '\\$&'],
+    }, createCustomTranslators());
+    return markdownConverter;
+}
+export function translateHtmlFragmentToMarkdown(html) {
+    return getMarkdownConverter().translate(html).trim();
+}

package/dist/transform/metadata.d.ts

@@ -0,0 +1,4 @@
+import type { ExtractedMetadata } from './types.js';
+export declare function extractMetadata(document: Document, baseUrl?: string): ExtractedMetadata;
+export declare function extractMetadataFromHead(html: string, baseUrl?: string): ExtractedMetadata | null;
+export declare function mergeMetadata(early: ExtractedMetadata | null, late: ExtractedMetadata): ExtractedMetadata;

package/dist/transform/metadata.js

@@ -0,0 +1,183 @@
+import { parseHTML } from 'linkedom';
+// ---------------------------------------------------------------------------
+// Head-section parsing
+// ---------------------------------------------------------------------------
+const HEAD_END_PATTERN = /<\/head\s*>|<body\b/i;
+const MAX_HEAD_SCAN_LENGTH = 50_000;
+function extractHeadSection(html) {
+    if (html.length <= MAX_HEAD_SCAN_LENGTH) {
+        const match = HEAD_END_PATTERN.exec(html);
+        return match ? html.substring(0, match.index) : null;
+    }
+    const searchText = html.substring(0, MAX_HEAD_SCAN_LENGTH);
+    const match = HEAD_END_PATTERN.exec(searchText);
+    if (!match)
+        return null;
+    return html.substring(0, match.index);
+}
+const META_PROPERTY_HANDLERS = new Map([
+    [
+        'og:title',
+        (ctx, c) => {
+            ctx.title.og = c;
+        },
+    ],
+    [
+        'og:description',
+        (ctx, c) => {
+            ctx.description.og = c;
+        },
+    ],
+    [
+        'og:image',
+        (ctx, c) => {
+            ctx.image = c;
+        },
+    ],
+    [
+        'article:published_time',
+        (ctx, c) => {
+            ctx.publishedAt = c;
+        },
+    ],
+    [
+        'article:modified_time',
+        (ctx, c) => {
+            ctx.modifiedAt = c;
+        },
+    ],
+]);
+const META_NAME_HANDLERS = new Map([
+    [
+        'twitter:title',
+        (ctx, c) => {
+            ctx.title.twitter = c;
+        },
+    ],
+    [
+        'twitter:description',
+        (ctx, c) => {
+            ctx.description.twitter = c;
+        },
+    ],
+    [
+        'description',
+        (ctx, c) => {
+            ctx.description.standard = c;
+        },
+    ],
+    [
+        'author',
+        (ctx, c) => {
+            ctx.author = c;
+        },
+    ],
+]);
+function processMetaTag(ctx, tag) {
+    const content = tag.getAttribute('content')?.trim();
+    if (!content)
+        return;
+    const property = tag.getAttribute('property');
+    if (property)
+        META_PROPERTY_HANDLERS.get(property)?.(ctx, content);
+    const name = tag.getAttribute('name');
+    if (name)
+        META_NAME_HANDLERS.get(name)?.(ctx, content);
+}
+function buildMetaContext(document) {
+    const ctx = { title: {}, description: {} };
+    for (const tag of document.querySelectorAll('meta')) {
+        processMetaTag(ctx, tag);
+    }
+    const titleEl = document.querySelector('title');
+    if (!ctx.title.standard && titleEl?.textContent) {
+        ctx.title.standard = titleEl.textContent.trim();
+    }
+    return ctx;
+}
+function resolveMetadataFromContext(ctx) {
+    const metadata = {};
+    const resolvedTitle = ctx.title.og ?? ctx.title.twitter ?? ctx.title.standard;
+    const resolvedDesc = ctx.description.og ?? ctx.description.twitter ?? ctx.description.standard;
+    if (resolvedTitle)
+        metadata.title = resolvedTitle;
+    if (resolvedDesc)
+        metadata.description = resolvedDesc;
+    if (ctx.author)
+        metadata.author = ctx.author;
+    if (ctx.image)
+        metadata.image = ctx.image;
+    if (ctx.publishedAt)
+        metadata.publishedAt = ctx.publishedAt;
+    if (ctx.modifiedAt)
+        metadata.modifiedAt = ctx.modifiedAt;
+    return metadata;
+}
+// ---------------------------------------------------------------------------
+// Favicon resolution
+// ---------------------------------------------------------------------------
+function resolveFaviconUrl(href, baseUrl) {
+    const trimmed = href.trim();
+    if (!trimmed)
+        return undefined;
+    if (trimmed.toLowerCase().startsWith('data:'))
+        return undefined;
+    try {
+        const resolved = new URL(trimmed, baseUrl);
+        if (resolved.protocol !== 'http:' && resolved.protocol !== 'https:') {
+            return undefined;
+        }
+        return resolved.toString();
+    }
+    catch {
+        return undefined;
+    }
+}
+// ---------------------------------------------------------------------------
+// Public interface
+// ---------------------------------------------------------------------------
+export function extractMetadata(document, baseUrl) {
+    const ctx = buildMetaContext(document);
+    const metadata = resolveMetadataFromContext(ctx);
+    if (baseUrl) {
+        const icon32 = document.querySelector('link[rel="icon"][sizes="32x32"]');
+        const href = icon32?.getAttribute('href');
+        if (href) {
+            const resolved = resolveFaviconUrl(href, baseUrl);
+            if (resolved)
+                metadata.favicon = resolved;
+        }
+    }
+    return metadata;
+}
+export function extractMetadataFromHead(html, baseUrl) {
+    const headSection = extractHeadSection(html);
+    if (!headSection)
+        return null;
+    try {
+        const { document } = parseHTML(`<!DOCTYPE html><html>${headSection}</head><body></body></html>`);
+        return extractMetadata(document, baseUrl);
+    }
+    catch {
+        return null;
+    }
+}
+export function mergeMetadata(early, late) {
+    if (!early)
+        return late;
+    const merged = {};
+    const keys = [
+        'title',
+        'description',
+        'author',
+        'image',
+        'publishedAt',
+        'modifiedAt',
+    ];
+    for (const key of keys) {
+        const value = late[key] ?? early[key];
+        if (value !== undefined)
+            merged[key] = value;
+    }
+    return merged;
+}

package/dist/transform/transform.d.ts

@@ -34,4 +34,3 @@ type TransformExecutionOptions = TransformOptions & {
 export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
 export declare function transformBufferToMarkdown(htmlBuffer: Uint8Array, url: string, options: TransformExecutionOptions): Promise<MarkdownTransformResult>;
 export {};
-//# sourceMappingURL=transform.d.ts.map