@j0hanz/fetch-url-mcp 1.12.7 → 1.12.8

package/dist/transform/metadata.js

@@ -1,462 +0,0 @@
-import { parseHTML } from 'linkedom';
-import { config } from '../lib/core.js';
-import { parseUrlOrNull } from '../lib/utils.js';
-// ---------------------------------------------------------------------------
-// Head-section parsing
-// ---------------------------------------------------------------------------
-const HEAD_END_PATTERN = /<\/head\s*>|<body\b/i;
-const MAX_HEAD_SCAN_LENGTH = 50_000;
-function extractHeadSection(html) {
-    const searchText = html.length <= MAX_HEAD_SCAN_LENGTH
-        ? html
-        : html.slice(0, MAX_HEAD_SCAN_LENGTH);
-    const match = HEAD_END_PATTERN.exec(searchText);
-    return match ? html.slice(0, match.index) : null;
-}
-export function normalizeDocumentTitle(title, baseUrl) {
-    if (!baseUrl || !title.startsWith('GitHub - '))
-        return title;
-    const parsed = parseUrlOrNull(baseUrl);
-    if (!parsed)
-        return title;
-    const hostname = parsed.hostname.toLowerCase();
-    if (hostname !== 'github.com' && hostname !== 'www.github.com') {
-        return title;
-    }
-    const segments = parsed.pathname.split('/').filter(Boolean);
-    if (segments.length !== 2)
-        return title;
-    const [owner, repo] = segments;
-    if (!owner || !repo)
-        return title;
-    return `${owner}/${repo}`;
-}
-const META_PROPERTY_HANDLERS = new Map([
-    [
-        'og:title',
-        (ctx, c) => {
-            ctx.title.og = c;
-        },
-    ],
-    [
-        'og:description',
-        (ctx, c) => {
-            ctx.description.og = c;
-        },
-    ],
-    [
-        'og:image',
-        (ctx, c) => {
-            ctx.image = c;
-        },
-    ],
-    [
-        'article:published_time',
-        (ctx, c) => {
-            ctx.publishedAt = c;
-        },
-    ],
-    [
-        'article:modified_time',
-        (ctx, c) => {
-            ctx.modifiedAt = c;
-        },
-    ],
-]);
-const META_NAME_HANDLERS = new Map([
-    [
-        'twitter:title',
-        (ctx, c) => {
-            ctx.title.twitter = c;
-        },
-    ],
-    [
-        'twitter:description',
-        (ctx, c) => {
-            ctx.description.twitter = c;
-        },
-    ],
-    [
-        'description',
-        (ctx, c) => {
-            ctx.description.standard = c;
-        },
-    ],
-    [
-        'author',
-        (ctx, c) => {
-            ctx.author = c;
-        },
-    ],
-]);
-function processMetaTag(ctx, tag) {
-    const content = tag.getAttribute('content')?.trim();
-    if (!content)
-        return;
-    const property = tag.getAttribute('property');
-    if (property)
-        META_PROPERTY_HANDLERS.get(property)?.(ctx, content);
-    const name = tag.getAttribute('name');
-    if (name)
-        META_NAME_HANDLERS.get(name)?.(ctx, content);
-}
-function buildMetaContext(document) {
-    const ctx = { title: {}, description: {} };
-    for (const tag of document.querySelectorAll('meta')) {
-        processMetaTag(ctx, tag);
-    }
-    const titleEl = document.querySelector('title');
-    if (!ctx.title.standard && titleEl?.textContent) {
-        ctx.title.standard = titleEl.textContent.trim();
-    }
-    return ctx;
-}
-function resolveMetadataFromContext(ctx) {
-    const metadata = {};
-    const resolvedTitle = ctx.title.og ?? ctx.title.twitter ?? ctx.title.standard;
-    const resolvedDesc = ctx.description.og ?? ctx.description.twitter ?? ctx.description.standard;
-    if (resolvedTitle)
-        metadata.title = resolvedTitle;
-    if (resolvedDesc)
-        metadata.description = resolvedDesc;
-    if (ctx.author)
-        metadata.author = ctx.author;
-    if (ctx.image)
-        metadata.image = ctx.image;
-    if (ctx.publishedAt)
-        metadata.publishedAt = ctx.publishedAt;
-    if (ctx.modifiedAt)
-        metadata.modifiedAt = ctx.modifiedAt;
-    return metadata;
-}
-// ---------------------------------------------------------------------------
-// Favicon resolution
-// ---------------------------------------------------------------------------
-/** Ordered by preference: exact 32×32, SVG, any generic icon, legacy shortcut. */
-const FAVICON_SELECTORS = [
-    'link[rel="icon"][sizes="32x32"]',
-    'link[rel="icon"][type="image/svg+xml"]',
-    'link[rel="icon"]',
-    'link[rel="shortcut icon"]',
-];
-function resolveFaviconUrl(href, baseUrl) {
-    const trimmed = href.trim();
-    if (!trimmed || trimmed.toLowerCase().startsWith('data:'))
-        return undefined;
-    const resolved = parseUrlOrNull(trimmed, baseUrl);
-    if (resolved?.protocol !== 'http:' && resolved?.protocol !== 'https:') {
-        return undefined;
-    }
-    return resolved.toString();
-}
-function extractFavicon(document, baseUrl) {
-    for (const selector of FAVICON_SELECTORS) {
-        for (const el of document.querySelectorAll(selector)) {
-            const href = el.getAttribute('href');
-            if (href) {
-                const resolved = resolveFaviconUrl(href, baseUrl);
-                if (resolved)
-                    return resolved;
-            }
-        }
-    }
-    return undefined;
-}
-// ---------------------------------------------------------------------------
-// Public interface
-// ---------------------------------------------------------------------------
-export function extractMetadata(document, baseUrl) {
-    const ctx = buildMetaContext(document);
-    const metadata = resolveMetadataFromContext(ctx);
-    if (metadata.title) {
-        metadata.title = normalizeDocumentTitle(metadata.title, baseUrl);
-    }
-    if (baseUrl) {
-        const favicon = extractFavicon(document, baseUrl);
-        if (favicon)
-            metadata.favicon = favicon;
-    }
-    return metadata;
-}
-export function extractMetadataFromHead(html, baseUrl) {
-    const headSection = extractHeadSection(html);
-    if (!headSection)
-        return null;
-    try {
-        const { document } = parseHTML(`<!DOCTYPE html><html>${headSection}</head><body></body></html>`);
-        return extractMetadata(document, baseUrl);
-    }
-    catch {
-        return null;
-    }
-}
-export function mergeMetadata(early, late) {
-    if (!early)
-        return late;
-    const merged = {};
-    const keys = [
-        'title',
-        'description',
-        'author',
-        'image',
-        'favicon',
-        'publishedAt',
-        'modifiedAt',
-    ];
-    for (const key of keys) {
-        const value = late[key] ?? early[key];
-        if (value !== undefined)
-            merged[key] = value;
-    }
-    return merged;
-}
-const BODY_SCAN_LIMIT = 500;
-const HTML_TAG_DENSITY_LIMIT = 5;
-const HEADING_MARKER = /^#{1,6}\s/m;
-const HEADING_STRICT = /^#{1,6}\s+/m;
-const SOURCE_KEY = /^source:\s/im;
-const HTML_DOC_START = /^(<!doctype|<html)/i;
-const LIST_MARKER = /^(?:[-*+])\s/m;
-function getLineEnding(content) {
-    return content.includes('\r\n') ? '\r\n' : '\n';
-}
-function parseFrontmatter(content) {
-    const len = content.length;
-    if (len < 4)
-        return null;
-    let lineEnding = null;
-    let fenceLen = 0;
-    if (content.startsWith('---\n')) {
-        lineEnding = '\n';
-        fenceLen = 4;
-    }
-    else if (content.startsWith('---\r\n')) {
-        lineEnding = '\r\n';
-        fenceLen = 5;
-    }
-    if (!lineEnding)
-        return null;
-    const fence = `---${lineEnding}`;
-    const closeIndex = content.indexOf(fence, fenceLen);
-    if (closeIndex === -1)
-        return null;
-    const range = {
-        start: 0,
-        end: closeIndex + fenceLen,
-        linesStart: fenceLen,
-        linesEnd: closeIndex,
-        lineEnding,
-    };
-    // Parse key-value entries in one pass
-    const entries = new Map();
-    const fmBody = content.slice(range.linesStart, range.linesEnd);
-    let lastIdx = 0;
-    while (lastIdx < fmBody.length) {
-        let nextIdx = fmBody.indexOf(lineEnding, lastIdx);
-        if (nextIdx === -1)
-            nextIdx = fmBody.length;
-        const line = fmBody.slice(lastIdx, nextIdx).trim();
-        const colonIdx = line.indexOf(':');
-        if (line && colonIdx > 0) {
-            const key = line.slice(0, colonIdx).trim().toLowerCase();
-            let value = line.slice(colonIdx + 1).trim();
-            // Strip surrounding quotes
-            const first = value.charAt(0);
-            const last = value.charAt(value.length - 1);
-            if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
-                value = value.slice(1, -1).trim();
-            }
-            if (value)
-                entries.set(key, value);
-        }
-        lastIdx = nextIdx + lineEnding.length;
-    }
-    return { range, entries };
-}
-function scanBodyForTitle(content) {
-    const len = content.length;
-    let scanIndex = 0;
-    const maxScan = Math.min(len, BODY_SCAN_LIMIT);
-    while (scanIndex < maxScan) {
-        let nextIndex = content.indexOf('\n', scanIndex);
-        if (nextIndex === -1)
-            nextIndex = len;
-        let line = content.slice(scanIndex, nextIndex);
-        if (line.endsWith('\r'))
-            line = line.slice(0, -1);
-        const trimmed = line.trim();
-        if (trimmed) {
-            if (HEADING_STRICT.test(trimmed)) {
-                return trimmed.replace(HEADING_MARKER, '').trim() || undefined;
-            }
-        }
-        scanIndex = nextIndex + 1;
-    }
-    return undefined;
-}
-export function extractTitleFromRawMarkdown(content) {
-    const fm = parseFrontmatter(content);
-    if (fm) {
-        const title = fm.entries.get('title') ?? fm.entries.get('name');
-        if (title)
-            return title;
-    }
-    return scanBodyForTitle(fm ? content.slice(fm.range.end) : content);
-}
-export function addSourceToMarkdown(content, url) {
-    const fm = parseFrontmatter(content);
-    const useMarkdownFormat = config.transform.metadataFormat === 'markdown';
-    if (useMarkdownFormat && !fm) {
-        if (SOURCE_KEY.test(content))
-            return content;
-        const lineEnding = getLineEnding(content);
-        const firstH1Match = HEADING_MARKER.exec(content);
-        if (firstH1Match) {
-            const h1Index = firstH1Match.index;
-            const lineEndIndex = content.indexOf(lineEnding, h1Index);
-            const insertPos = lineEndIndex === -1 ? content.length : lineEndIndex + lineEnding.length;
-            const injection = `${lineEnding}Source: ${url}${lineEnding}`;
-            return content.slice(0, insertPos) + injection + content.slice(insertPos);
-        }
-        return `Source: ${url}${lineEnding}${lineEnding}${content}`;
-    }
-    if (!fm) {
-        const lineEnding = getLineEnding(content);
-        const escapedUrl = url.replace(/"/g, '\\"');
-        return `---${lineEnding}source: "${escapedUrl}"${lineEnding}---${lineEnding}${lineEnding}${content}`;
-    }
-    const fmBody = content.slice(fm.range.linesStart, fm.range.linesEnd);
-    if (SOURCE_KEY.test(fmBody))
-        return content;
-    const escapedUrl = url.replace(/"/g, '\\"');
-    const injection = `source: "${escapedUrl}"${fm.range.lineEnding}`;
-    return (content.slice(0, fm.range.linesEnd) +
-        injection +
-        content.slice(fm.range.linesEnd));
-}
-// endregion
-// region Content Detection & Metadata Footer
-function countCommonTags(content, limit) {
-    if (limit <= 0)
-        return 0;
-    const regex = /<(html|head|body|div|span|script|style|meta|link)\b/gi;
-    let count = 0;
-    while (regex.exec(content)) {
-        count += 1;
-        if (count > limit)
-            break;
-    }
-    return count;
-}
-export function isRawTextContent(content) {
-    const trimmed = content.trim();
-    if (HTML_DOC_START.test(trimmed))
-        return false;
-    if (parseFrontmatter(trimmed) !== null)
-        return true;
-    const tagCount = countCommonTags(content, HTML_TAG_DENSITY_LIMIT);
-    if (tagCount > HTML_TAG_DENSITY_LIMIT)
-        return false;
-    return (HEADING_MARKER.test(content) ||
-        LIST_MARKER.test(content) ||
-        content.includes('```'));
-}
-function formatFetchedAt(value) {
-    const date = new Date(value);
-    if (Number.isNaN(date.getTime()))
-        return value;
-    const formatter = new Intl.DateTimeFormat(config.i18n.locale, {
-        day: '2-digit',
-        month: '2-digit',
-        year: 'numeric',
-    });
-    return formatter.format(date);
-}
-export function buildMetadataFooter(metadata, fallbackUrl) {
-    if (!metadata)
-        return '';
-    const lines = ['---', ''];
-    const url = metadata.url || fallbackUrl;
-    const parts = [];
-    if (metadata.title)
-        parts.push(`_${metadata.title}_`);
-    if (metadata.author)
-        parts.push(`_${metadata.author}_`);
-    if (url)
-        parts.push(`[_Original Source_](${url})`);
-    if (metadata.fetchedAt) {
-        parts.push(`_${formatFetchedAt(metadata.fetchedAt)}_`);
-    }
-    if (parts.length > 0)
-        lines.push(` ${parts.join(' | ')}`);
-    if (metadata.description)
-        lines.push(` <sub>${metadata.description}</sub>`);
-    return lines.join('\n');
-}
-const TITLE_PART_SEPARATOR = /\s*(?:[-|:•·]|–|—)\s*/u;
-const LEADING_HEADING_PATTERN = /^(#{1,6})\s+(.+?)\s*$/;
-const HEADING_SCAN_LIMIT = 12;
-export function normalizeSyntheticTitleToken(value) {
-    return (value ?? '').replace(/\s+/g, ' ').trim().toLowerCase();
-}
-export function shouldPreferPrimaryHeadingTitle(primaryHeading, title) {
-    const primary = normalizeSyntheticTitleToken(primaryHeading);
-    if (!primary)
-        return false;
-    const normalizedTitle = normalizeSyntheticTitleToken(title);
-    if (!normalizedTitle)
-        return true;
-    if (normalizedTitle === primary)
-        return true;
-    return normalizedTitle
-        .split(TITLE_PART_SEPARATOR)
-        .some((part) => part === primary);
-}
-export function isGithubRepositoryRootUrl(url) {
-    const parsed = parseUrlOrNull(url);
-    if (!parsed)
-        return false;
-    const hostname = parsed.hostname.toLowerCase();
-    if (hostname !== 'github.com' && hostname !== 'www.github.com') {
-        return false;
-    }
-    return parsed.pathname.split('/').filter(Boolean).length === 2;
-}
-function stripLeadingHeading(markdown, headingText) {
-    if (!markdown)
-        return markdown;
-    const lines = markdown.split('\n');
-    const target = normalizeSyntheticTitleToken(headingText);
-    let nonEmptySeen = 0;
-    for (let index = 0; index < lines.length && nonEmptySeen < HEADING_SCAN_LIMIT; index += 1) {
-        const trimmed = lines[index]?.trim() ?? '';
-        if (!trimmed)
-            continue;
-        nonEmptySeen += 1;
-        const match = LEADING_HEADING_PATTERN.exec(trimmed);
-        if (!match)
-            continue;
-        const current = normalizeSyntheticTitleToken(match[2] ?? '');
-        if (current !== target)
-            return markdown;
-        lines.splice(index, 1);
-        if ((lines[index] ?? '').trim() === '') {
-            lines.splice(index, 1);
-        }
-        return lines.join('\n');
-    }
-    return markdown;
-}
-export function maybeStripGithubPrimaryHeading(markdown, primaryHeading, url) {
-    if (primaryHeading === undefined || !isGithubRepositoryRootUrl(url)) {
-        return markdown;
-    }
-    return stripLeadingHeading(markdown, primaryHeading);
-}
-export function maybePrependSyntheticTitle(markdown, context) {
-    if (!context.title || /^#\s/.test(markdown.trimStart())) {
-        return markdown;
-    }
-    return `# ${context.title}\n\n${markdown}`;
-}
-// endregion

package/dist/transform/next-flight.d.ts

@@ -1,2 +0,0 @@
-export declare function supplementMarkdownFromNextFlight(markdown: string, originalHtml: string): string;
-//# sourceMappingURL=next-flight.d.ts.map

package/dist/transform/next-flight.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"next-flight.d.ts","sourceRoot":"","sources":["../../src/transform/next-flight.ts"],"names":[],"mappings":"AA2bA,wBAAgB,gCAAgC,CAC9C,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,GACnB,MAAM,CAuCR"}