@j0hanz/fetch-url-mcp 0.0.1

package/dist/dom-noise-removal.js

@@ -0,0 +1,488 @@
+import { parseHTML } from 'linkedom';
+import { config } from './config.js';
+import { logDebug } from './observability.js';
+// --- Constants & Pre-compiled Regex ---
+const NOISE_SCAN_LIMIT = 50_000;
+const MIN_BODY_CONTENT_LENGTH = 100;
+const DIALOG_MIN_CHARS_FOR_PRESERVATION = 500;
+const NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION = 500;
+// Merged markers for fast rejection
+const HTML_DOCUMENT_MARKERS = /<\s*(?:!doctype|html|head|body)\b/i;
+const HTML_FRAGMENT_MARKERS = /<\s*(?:article|main|section|div|nav|footer|header|aside|table|ul|ol)\b/i;
+// Split into smaller regexes to stay within sonarjs/regex-complexity limit
+const NOISE_PATTERNS = [
+    /<\s*(?:script|style|noscript|iframe|nav|footer|header|form|button|input|select|textarea|svg|canvas)\b/i,
+    /[\s"']role\s*=\s*['"]?(?:navigation|banner|complementary|contentinfo|tree|menubar|menu)['"]?/i,
+    /[\s"'](?:aria-hidden\s*=\s*['"]?true['"]?|hidden)/i,
+    /[\s"'](?:banner|promo|announcement|cta|advert|newsletter|subscribe|cookie|consent|popup|modal|overlay|toast)\b/i,
+    /[\s"'](?:fixed|sticky|z-50|z-4|isolate|breadcrumb|pagination)\b/i,
+];
+const HEADER_NOISE_PATTERN = /\b(site-header|masthead|topbar|navbar|nav(?:bar)?|menu|header-nav)\b/i;
+const FIXED_OR_HIGH_Z_PATTERN = /\b(?:fixed|sticky|z-(?:4\d|50)|isolate)\b/;
+const SKIP_URL_PREFIXES = [
+    '#',
+    'java' + 'script:',
+    'mailto:',
+    'tel:',
+    'data:',
+    'blob:',
+];
+const BASE_STRUCTURAL_TAGS = new Set([
+    'script',
+    'style',
+    'noscript',
+    'iframe',
+    'form',
+    'button',
+    'input',
+    'select',
+    'textarea',
+]);
+const ALWAYS_NOISE_TAGS = new Set(['nav', 'footer']);
+const NAVIGATION_ROLES = new Set([
+    'navigation',
+    'banner',
+    'complementary',
+    'contentinfo',
+    'tree',
+    'menubar',
+    'menu',
+    'dialog',
+    'alertdialog',
+    'search',
+]);
+const INTERACTIVE_CONTENT_ROLES = new Set([
+    'tabpanel',
+    'tab',
+    'tablist',
+    'dialog',
+    'alertdialog',
+    'menu',
+    'menuitem',
+    'option',
+    'listbox',
+    'combobox',
+    'tooltip',
+    'alert',
+]);
+const PROMO_TOKENS_ALWAYS = [
+    'banner',
+    'promo',
+    'announcement',
+    'cta',
+    'advert',
+    'ads',
+    'sponsor',
+    'recommend',
+    'breadcrumb',
+    'pagination',
+    'pager',
+    'taglist',
+];
+const PROMO_TOKENS_AGGRESSIVE = ['ad', 'related', 'comment'];
+const PROMO_TOKENS_BY_CATEGORY = {
+    'cookie-banners': ['cookie', 'consent', 'popup', 'modal', 'overlay', 'toast'],
+    newsletters: ['newsletter', 'subscribe'],
+    'social-share': ['share', 'social'],
+};
+const BASE_NOISE_SELECTORS = {
+    navFooter: 'nav,footer,header[class*="site"],header[class*="nav"],header[class*="menu"],[role="banner"],[role="navigation"]',
+    cookieBanners: '[role="dialog"]',
+    hidden: '[style*="display: none"],[style*="display:none"],[hidden],[aria-hidden="true"]',
+};
+const NO_MATCH_REGEX = /a^/i;
+// --- State Cache ---
+let cachedContext;
+let lastConfigRef;
+// --- Helpers Inlined/Optimized ---
+function escapeRegexLiteral(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function buildTokenRegex(tokens) {
+    if (tokens.size === 0)
+        return NO_MATCH_REGEX;
+    return new RegExp(`(?:^|[^a-z0-9])(?:${[...tokens].map(escapeRegexLiteral).join('|')})(?:$|[^a-z0-9])`, 'i');
+}
+function getPromoMatchers(currentConfig, flags) {
+    const baseTokens = new Set(PROMO_TOKENS_ALWAYS);
+    const aggressiveTokens = new Set();
+    if (currentConfig.aggressiveMode) {
+        for (const t of PROMO_TOKENS_AGGRESSIVE)
+            aggressiveTokens.add(t);
+    }
+    if (flags.cookieBanners)
+        for (const t of PROMO_TOKENS_BY_CATEGORY['cookie-banners'])
+            baseTokens.add(t);
+    if (flags.newsletters)
+        for (const t of PROMO_TOKENS_BY_CATEGORY['newsletters'])
+            baseTokens.add(t);
+    if (flags.socialShare)
+        for (const t of PROMO_TOKENS_BY_CATEGORY['social-share'])
+            baseTokens.add(t);
+    for (const t of currentConfig.extraTokens) {
+        const n = t.toLowerCase().trim();
+        if (n)
+            baseTokens.add(n);
+    }
+    return {
+        base: buildTokenRegex(baseTokens),
+        aggressive: buildTokenRegex(aggressiveTokens),
+    };
+}
+function getContext() {
+    const currentConfig = config.noiseRemoval;
+    if (cachedContext && lastConfigRef === currentConfig) {
+        return cachedContext;
+    }
+    const enabled = new Set(currentConfig.enabledCategories
+        .map((c) => {
+        const s = c.toLowerCase().trim();
+        const { locale } = config.i18n;
+        return locale ? s.toLocaleLowerCase(locale) : s;
+    })
+        .filter(Boolean));
+    const isEnabled = (cat) => enabled.has(cat);
+    const flags = {
+        navFooter: isEnabled('nav-footer'),
+        cookieBanners: isEnabled('cookie-banners'),
+        newsletters: isEnabled('newsletters'),
+        socialShare: isEnabled('social-share'),
+    };
+    const structuralTags = new Set(BASE_STRUCTURAL_TAGS);
+    if (!currentConfig.preserveSvgCanvas) {
+        structuralTags.add('svg');
+        structuralTags.add('canvas');
+    }
+    const promoMatchers = getPromoMatchers(currentConfig, flags);
+    const extraSelectors = currentConfig.extraSelectors
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+    // Pre-build selectors
+    const selectors = [BASE_NOISE_SELECTORS.hidden];
+    if (flags.navFooter)
+        selectors.push(BASE_NOISE_SELECTORS.navFooter);
+    if (flags.cookieBanners)
+        selectors.push(BASE_NOISE_SELECTORS.cookieBanners);
+    const baseSelector = selectors.join(',');
+    const candidateSelector = [
+        ...structuralTags,
+        ...ALWAYS_NOISE_TAGS,
+        'aside',
+        'header',
+        '[class]',
+        '[id]',
+        '[role]',
+        '[style]',
+    ].join(',');
+    cachedContext = {
+        flags,
+        structuralTags,
+        weights: currentConfig.weights,
+        promoMatchers,
+        promoEnabled: flags.cookieBanners || flags.newsletters || flags.socialShare,
+        extraSelectors,
+        baseSelector,
+        candidateSelector,
+    };
+    lastConfigRef = currentConfig;
+    return cachedContext;
+}
+// --- Hot Path Logic ---
+function isInteractive(element, role) {
+    if (role && INTERACTIVE_CONTENT_ROLES.has(role))
+        return true;
+    const ds = element.getAttribute('data-state');
+    if (ds === 'inactive' || ds === 'closed')
+        return true;
+    const dataOrientation = element.getAttribute('data-orientation');
+    if (dataOrientation === 'horizontal' || dataOrientation === 'vertical')
+        return true;
+    return (element.hasAttribute('data-accordion-item') ||
+        element.hasAttribute('data-radix-collection-item'));
+}
+function isWithinPrimaryContent(element) {
+    let current = element;
+    while (current) {
+        const tagName = current.tagName.toLowerCase();
+        if (tagName === 'article' || tagName === 'main')
+            return true;
+        if (current.getAttribute('role') === 'main')
+            return true;
+        current = current.parentElement;
+    }
+    return false;
+}
+function shouldPreserve(element, tagName) {
+    // Check Dialog
+    const role = element.getAttribute('role');
+    if (role === 'dialog' || role === 'alertdialog') {
+        if (isWithinPrimaryContent(element))
+            return true;
+        const textLen = (element.textContent || '').length;
+        if (textLen > DIALOG_MIN_CHARS_FOR_PRESERVATION)
+            return true;
+        return element.querySelector('h1,h2,h3,h4,h5,h6') !== null;
+    }
+    // Check Nav/Footer
+    if (tagName === 'nav' || tagName === 'footer') {
+        if (element.querySelector('article,main,section,[role="main"]'))
+            return true;
+        return ((element.textContent || '').trim().length >=
+            NAV_FOOTER_MIN_CHARS_FOR_PRESERVATION);
+    }
+    return false;
+}
+function removeNodes(nodes) {
+    for (let i = nodes.length - 1; i >= 0; i--) {
+        const node = nodes[i];
+        if (node?.parentNode && !shouldPreserve(node, node.tagName.toLowerCase())) {
+            node.remove();
+        }
+    }
+}
+function scoreNavFooter(tagName, role, className, id, weights) {
+    let score = 0;
+    if (ALWAYS_NOISE_TAGS.has(tagName))
+        score += weights.structural;
+    // Header Boilerplate
+    if (tagName === 'header') {
+        if ((role && NAVIGATION_ROLES.has(role)) ||
+            HEADER_NOISE_PATTERN.test(`${className} ${id}`)) {
+            score += weights.structural;
+        }
+    }
+    // Role Noise
+    if (role && NAVIGATION_ROLES.has(role)) {
+        if (tagName !== 'aside' || role !== 'complementary') {
+            score += weights.structural;
+        }
+    }
+    return score;
+}
+function extractElementMetadata(element) {
+    const tagName = element.tagName.toLowerCase();
+    const className = element.getAttribute('class') ?? '';
+    const id = element.getAttribute('id') ?? '';
+    const role = element.getAttribute('role');
+    const style = element.getAttribute('style');
+    const _isInteractive = isInteractive(element, role);
+    const isHidden = element.hasAttribute('hidden') ||
+        element.getAttribute('aria-hidden') === 'true' ||
+        (style !== null &&
+            /\b(?:display\s*:\s*none|visibility\s*:\s*hidden)\b/i.test(style));
+    return {
+        tagName,
+        className,
+        id,
+        role,
+        style,
+        isInteractive: _isInteractive,
+        isHidden,
+    };
+}
+function isNoiseElement(element, context) {
+    const meta = extractElementMetadata(element);
+    let score = 0;
+    const { weights } = context;
+    // Structural
+    if (context.structuralTags.has(meta.tagName) && !meta.isInteractive) {
+        score += weights.structural;
+    }
+    // Nav/Footer Scoring
+    if (context.flags.navFooter) {
+        score += scoreNavFooter(meta.tagName, meta.role, meta.className, meta.id, weights);
+    }
+    // Hidden
+    if (meta.isHidden && !meta.isInteractive) {
+        score += weights.hidden;
+    }
+    // Sticky/Fixed
+    if (FIXED_OR_HIGH_Z_PATTERN.test(meta.className)) {
+        score += weights.stickyFixed;
+    }
+    // Promo
+    if (context.promoEnabled) {
+        const aggTest = context.promoMatchers.aggressive.test(meta.className) ||
+            context.promoMatchers.aggressive.test(meta.id);
+        const isAggressiveMatch = aggTest && !isWithinPrimaryContent(element);
+        const isBaseMatch = !aggTest &&
+            (context.promoMatchers.base.test(meta.className) ||
+                context.promoMatchers.base.test(meta.id));
+        if (isAggressiveMatch || isBaseMatch) {
+            score += weights.promo;
+        }
+    }
+    return score >= weights.threshold;
+}
+function cleanHeadingWrapperDivs(h) {
+    const divs = h.querySelectorAll('div');
+    for (let j = divs.length - 1; j >= 0; j--) {
+        const d = divs[j];
+        if (!d?.parentNode)
+            continue;
+        const cls = d.getAttribute('class') ?? '';
+        const stl = d.getAttribute('style') ?? '';
+        if (cls.includes('absolute') ||
+            stl.includes('position') ||
+            d.getAttribute('tabindex') === '-1') {
+            d.remove();
+        }
+    }
+}
+function cleanHeadingAnchors(h) {
+    const anchors = h.querySelectorAll('a');
+    for (let j = anchors.length - 1; j >= 0; j--) {
+        const a = anchors[j];
+        if (!a?.parentNode)
+            continue;
+        const href = a.getAttribute('href') ?? '';
+        const txt = (a.textContent || '').replace(/[\u200B\s]/g, '');
+        if (href.startsWith('#') && txt.length === 0) {
+            a.remove();
+        }
+    }
+}
+function cleanHeadingZeroWidth(h, document) {
+    const walker = document.createTreeWalker(h, 4); // SHOW_TEXT
+    let node;
+    while ((node = walker.nextNode())) {
+        if (node.textContent?.includes('\u200B')) {
+            node.textContent = node.textContent.replace(/\u200B/g, '');
+        }
+    }
+}
+function cleanHeadings(document) {
+    // Clean Heading Anchors
+    const headings = document.querySelectorAll('h1,h2,h3,h4,h5,h6');
+    for (const h of headings) {
+        if (!h.parentNode)
+            continue;
+        cleanHeadingWrapperDivs(h);
+        cleanHeadingAnchors(h);
+        cleanHeadingZeroWidth(h, document);
+    }
+}
+function stripNoise(document, context) {
+    cleanHeadings(document);
+    // Remove Base & Extra
+    const { baseSelector, extraSelectors } = context;
+    // Base
+    const baseNodes = document.querySelectorAll(baseSelector);
+    removeNodes(baseNodes);
+    // Extra
+    if (extraSelectors.length > 0) {
+        const combinedExtra = extraSelectors.join(',');
+        const extraNodes = document.querySelectorAll(combinedExtra);
+        removeNodes(extraNodes);
+    }
+    // Candidates
+    const candidates = document.querySelectorAll(context.candidateSelector);
+    for (let i = candidates.length - 1; i >= 0; i--) {
+        const node = candidates[i];
+        if (!node)
+            continue;
+        if (!node.parentNode)
+            continue;
+        if (shouldPreserve(node, node.tagName.toLowerCase()))
+            continue;
+        if (isNoiseElement(node, context)) {
+            node.remove();
+        }
+    }
+}
+function processUrlElement(el, attr, base, isSrcset) {
+    if (!el.parentNode)
+        return;
+    if (isSrcset) {
+        const val = el.getAttribute(attr);
+        if (val) {
+            const newVal = val
+                .split(',')
+                .map((entry) => {
+                const parts = entry.trim().split(/\s+/);
+                if (!parts[0])
+                    return entry;
+                try {
+                    parts[0] = new URL(parts[0], base).href;
+                }
+                catch {
+                    /* ignore */
+                }
+                return parts.join(' ');
+            })
+                .join(', ');
+            el.setAttribute(attr, newVal);
+        }
+        return;
+    }
+    const val = el.getAttribute(attr);
+    if (val &&
+        !SKIP_URL_PREFIXES.some((p) => val.trim().toLowerCase().startsWith(p))) {
+        try {
+            el.setAttribute(attr, new URL(val, base).href);
+        }
+        catch {
+            /* ignore */
+        }
+    }
+}
+function resolveUrls(document, baseUrlStr) {
+    let base;
+    try {
+        base = new URL(baseUrlStr);
+    }
+    catch {
+        return;
+    }
+    const elements = document.querySelectorAll('a[href],img[src],source[srcset]');
+    for (const el of Array.from(elements)) {
+        const tag = el.tagName.toLowerCase();
+        if (tag === 'a')
+            processUrlElement(el, 'href', base, false);
+        else if (tag === 'img')
+            processUrlElement(el, 'src', base, false);
+        else if (tag === 'source')
+            processUrlElement(el, 'srcset', base, true);
+    }
+}
+function serialize(document, fallback) {
+    const bodyHtml = document.body.innerHTML;
+    if (bodyHtml.trim().length > MIN_BODY_CONTENT_LENGTH)
+        return bodyHtml;
+    const outerHtml = document.documentElement.outerHTML;
+    if (outerHtml.trim().length > MIN_BODY_CONTENT_LENGTH)
+        return outerHtml;
+    return fallback;
+}
+function isFullDocumentHtml(html) {
+    return HTML_DOCUMENT_MARKERS.test(html);
+}
+function mayContainNoise(html) {
+    const sample = html.length <= NOISE_SCAN_LIMIT
+        ? html
+        : `${html.substring(0, NOISE_SCAN_LIMIT)}\n${html.substring(html.length - NOISE_SCAN_LIMIT)}`;
+    return NOISE_PATTERNS.some((re) => re.test(sample));
+}
+export function removeNoiseFromHtml(html, document, baseUrl) {
+    const shouldParse = isFullDocumentHtml(html) ||
+        mayContainNoise(html) ||
+        HTML_FRAGMENT_MARKERS.test(html);
+    if (!shouldParse)
+        return html;
+    try {
+        const context = getContext();
+        if (config.noiseRemoval.debug) {
+            logDebug('Noise removal audit enabled', {
+                categories: [...(context.flags.navFooter ? ['nav-footer'] : [])],
+            });
+        }
+        const doc = document ?? parseHTML(html).document;
+        stripNoise(doc, context);
+        if (baseUrl)
+            resolveUrls(doc, baseUrl);
+        return serialize(doc, html);
+    }
+    catch {
+        return html;
+    }
+}

package/dist/errors.d.ts

@@ -0,0 +1,10 @@
+export declare class FetchError extends Error {
+    readonly url: string;
+    readonly statusCode: number;
+    readonly code: string;
+    readonly details: Readonly<Record<string, unknown>>;
+    constructor(message: string, url: string, httpStatus?: number, details?: Record<string, unknown>, options?: ErrorOptions);
+}
+export declare function getErrorMessage(error: unknown): string;
+export declare function createErrorWithCode(message: string, code: string, options?: ErrorOptions): NodeJS.ErrnoException;
+export declare function isSystemError(error: unknown): error is NodeJS.ErrnoException;

package/dist/errors.js

@@ -0,0 +1,61 @@
+import { inspect } from 'node:util';
+import { isError, isObject } from './type-guards.js';
+const DEFAULT_HTTP_STATUS = 502;
+export class FetchError extends Error {
+    url;
+    statusCode;
+    code;
+    details;
+    constructor(message, url, httpStatus, details = {}, options) {
+        super(message, options);
+        this.url = url;
+        this.name = 'FetchError';
+        this.statusCode = httpStatus ?? DEFAULT_HTTP_STATUS;
+        this.code = httpStatus ? `HTTP_${httpStatus}` : 'FETCH_ERROR';
+        this.details = Object.freeze({ url, httpStatus, ...details });
+        Error.captureStackTrace(this, this.constructor);
+    }
+}
+export function getErrorMessage(error) {
+    if (isError(error))
+        return error.message;
+    if (typeof error === 'string' && error.length > 0)
+        return error;
+    if (isErrorWithMessage(error))
+        return error.message;
+    return formatUnknownError(error);
+}
+function isErrorWithMessage(error) {
+    if (!isObject(error))
+        return false;
+    const { message } = error;
+    return typeof message === 'string' && message.length > 0;
+}
+function formatUnknownError(error) {
+    if (error === null || error === undefined)
+        return 'Unknown error';
+    try {
+        return inspect(error, {
+            depth: 2,
+            maxStringLength: 200,
+            breakLength: Infinity,
+            compact: true,
+            colors: false,
+        });
+    }
+    catch {
+        return 'Unknown error';
+    }
+}
+export function createErrorWithCode(message, code, options) {
+    const error = new Error(message, options);
+    return Object.assign(error, { code });
+}
+export function isSystemError(error) {
+    if (!isError(error))
+        return false;
+    if (!('code' in error))
+        return false;
+    const { code } = error;
+    return typeof code === 'string';
+}

package/dist/fetch.d.ts

@@ -0,0 +1,42 @@
+export interface FetchOptions {
+    signal?: AbortSignal;
+}
+export interface TransformResult {
+    readonly url: string;
+    readonly transformed: boolean;
+    readonly platform?: string;
+}
+export interface FetchTelemetryContext {
+    requestId: string;
+    startTime: number;
+    url: string;
+    method: string;
+    contextRequestId?: string;
+    operationId?: string;
+}
+export declare function isBlockedIp(ip: string): boolean;
+export declare function normalizeUrl(urlString: string): {
+    normalizedUrl: string;
+    hostname: string;
+};
+export declare function validateAndNormalizeUrl(urlString: string): string;
+export declare function transformToRawUrl(url: string): TransformResult;
+export declare function isRawTextContentUrl(url: string): boolean;
+export declare function startFetchTelemetry(url: string, method: string): FetchTelemetryContext;
+export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
+export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
+export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
+    response: Response;
+    url: string;
+}>;
+export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal, encoding?: string): Promise<{
+    text: string;
+    size: number;
+}>;
+export declare function fetchNormalizedUrl(normalizedUrl: string, options?: FetchOptions): Promise<string>;
+export declare function fetchNormalizedUrlBuffer(normalizedUrl: string, options?: FetchOptions): Promise<{
+    buffer: Uint8Array;
+    encoding: string;
+    truncated: boolean;
+    finalUrl: string;
+}>;