@j0hanz/superfetch 2.2.1 → 2.3.0

package/dist/dom-noise-removal.js

@@ -0,0 +1,485 @@
+/**
+ * DOM noise removal utilities for cleaning HTML before markdown conversion.
+ * Removes navigation, ads, popups, and other non-content elements.
+ */
+import { parseHTML } from 'linkedom';
+import { config } from './config.js';
+import { isObject } from './type-guards.js';
+// ─────────────────────────────────────────────────────────────────────────────
+// DOM Type Guards and Accessors
+// ─────────────────────────────────────────────────────────────────────────────
+function isElement(node) {
+    return (isObject(node) &&
+        'getAttribute' in node &&
+        typeof node.getAttribute === 'function');
+}
+function getBodyInnerHtml(document) {
+    if (!isObject(document))
+        return undefined;
+    const { body } = document;
+    if (isObject(body) && typeof body.innerHTML === 'string') {
+        return body.innerHTML;
+    }
+    return undefined;
+}
+function getDocumentToString(document) {
+    if (!isObject(document))
+        return undefined;
+    if (typeof document.toString === 'function') {
+        return document.toString.bind(document);
+    }
+    return undefined;
+}
+function getDocumentElementOuterHtml(document) {
+    if (!isObject(document))
+        return undefined;
+    const docEl = document.documentElement;
+    if (isObject(docEl) && typeof docEl.outerHTML === 'string') {
+        return docEl.outerHTML;
+    }
+    return undefined;
+}
+// ─────────────────────────────────────────────────────────────────────────────
+// Noise Detection Constants
+// ─────────────────────────────────────────────────────────────────────────────
+const STRUCTURAL_TAGS = new Set([
+    'script',
+    'style',
+    'noscript',
+    'iframe',
+    'form',
+    'button',
+    'input',
+    'select',
+    'textarea',
+    'svg',
+    'canvas',
+]);
+const ALWAYS_NOISE_TAGS = new Set(['nav', 'footer']);
+const BASE_NOISE_SELECTORS = [
+    'nav',
+    'footer',
+    'header[class*="site"]',
+    'header[class*="nav"]',
+    'header[class*="menu"]',
+    '[role="banner"]',
+    '[role="navigation"]',
+    '[role="dialog"]',
+    '[style*="display: none"]',
+    '[style*="display:none"]',
+    '[hidden]',
+    '[aria-hidden="true"]',
+];
+const BASE_NOISE_SELECTOR = BASE_NOISE_SELECTORS.join(',');
+const CANDIDATE_NOISE_SELECTOR = [
+    ...STRUCTURAL_TAGS,
+    ...ALWAYS_NOISE_TAGS,
+    'aside',
+    'header',
+    '[class]',
+    '[id]',
+    '[role]',
+    '[style]',
+].join(',');
+function buildNoiseSelector(extraSelectors) {
+    const extra = extraSelectors.filter((selector) => selector.trim().length > 0);
+    if (extra.length === 0)
+        return BASE_NOISE_SELECTOR;
+    return `${BASE_NOISE_SELECTOR},${extra.join(',')}`;
+}
+const NAVIGATION_ROLES = new Set([
+    'navigation',
+    'banner',
+    'complementary',
+    'contentinfo',
+    'tree',
+    'menubar',
+    'menu',
+    'dialog',
+    'alertdialog',
+    'search',
+]);
+const INTERACTIVE_CONTENT_ROLES = new Set([
+    'tabpanel',
+    'tab',
+    'tablist',
+    'dialog',
+    'alertdialog',
+    'menu',
+    'menuitem',
+    'option',
+    'listbox',
+    'combobox',
+    'tooltip',
+    'alert',
+]);
+const BASE_PROMO_TOKENS = [
+    'banner',
+    'promo',
+    'announcement',
+    'cta',
+    'advert',
+    'ad',
+    'ads',
+    'sponsor',
+    'newsletter',
+    'subscribe',
+    'cookie',
+    'consent',
+    'popup',
+    'modal',
+    'overlay',
+    'toast',
+    'share',
+    'social',
+    'related',
+    'recommend',
+    'comment',
+    'breadcrumb',
+    'pagination',
+    'pager',
+    'taglist',
+];
+/**
+ * Get promo tokens merged with any user-configured extra tokens.
+ * Memoized because it is used in hot paths when scanning many nodes.
+ */
+let promoTokensCache = null;
+function getPromoTokens() {
+    if (promoTokensCache)
+        return promoTokensCache;
+    const tokens = new Set(BASE_PROMO_TOKENS);
+    for (const token of config.noiseRemoval.extraTokens) {
+        const normalized = token.toLowerCase().trim();
+        if (normalized)
+            tokens.add(normalized);
+    }
+    promoTokensCache = tokens;
+    return tokens;
+}
+const HEADER_NOISE_PATTERN = /\b(site-header|masthead|topbar|navbar|nav(?:bar)?|menu|header-nav)\b/i;
+const FIXED_PATTERN = /\b(fixed|sticky)\b/;
+const HIGH_Z_PATTERN = /\bz-(?:4\d|50)\b/;
+const ISOLATE_PATTERN = /\bisolate\b/;
+const HTML_DOCUMENT_MARKERS = /<\s*(?:!doctype|html|head|body)\b/i;
+const NOISE_MARKERS = [
+    '<script',
+    '<style',
+    '<noscript',
+    '<iframe',
+    '<nav',
+    '<footer',
+    '<header',
+    '<form',
+    '<button',
+    '<input',
+    '<select',
+    '<textarea',
+    '<svg',
+    '<canvas',
+    ' aria-hidden="true"',
+    " aria-hidden='true'",
+    ' hidden',
+    ' role="navigation"',
+    " role='navigation'",
+    ' role="banner"',
+    " role='banner'",
+    ' role="complementary"',
+    " role='complementary'",
+    ' role="contentinfo"',
+    " role='contentinfo'",
+    ' role="tree"',
+    " role='tree'",
+    ' role="menubar"',
+    " role='menubar'",
+    ' role="menu"',
+    " role='menu'",
+    ' banner',
+    ' promo',
+    ' announcement',
+    ' cta',
+    ' advert',
+    ' newsletter',
+    ' subscribe',
+    ' cookie',
+    ' consent',
+    ' popup',
+    ' modal',
+    ' overlay',
+    ' toast',
+    ' fixed',
+    ' sticky',
+    ' z-50',
+    ' z-4',
+    ' isolate',
+    ' breadcrumb',
+    ' pagination',
+];
+// ─────────────────────────────────────────────────────────────────────────────
+// Noise Detection Functions
+// ─────────────────────────────────────────────────────────────────────────────
+const NOISE_SCAN_LIMIT = 50_000;
+function mayContainNoise(html) {
+    // Fast path: only scan a bounded prefix; parsing is the expensive step anyway.
+    // Most noise markers appear near the top of the document (nav, scripts, meta, etc.).
+    const sample = html.length > NOISE_SCAN_LIMIT ? html.slice(0, NOISE_SCAN_LIMIT) : html;
+    const haystack = sample.toLowerCase();
+    return NOISE_MARKERS.some((marker) => haystack.includes(marker));
+}
+function isFullDocumentHtml(html) {
+    return HTML_DOCUMENT_MARKERS.test(html);
+}
+function isStructuralNoiseTag(tagName) {
+    return STRUCTURAL_TAGS.has(tagName);
+}
+function isInteractiveComponent(element) {
+    const role = element.getAttribute('role');
+    if (role && INTERACTIVE_CONTENT_ROLES.has(role))
+        return true;
+    // Check for common UI framework data attributes that indicate managed visibility
+    const dataState = element.getAttribute('data-state');
+    if (dataState === 'inactive' || dataState === 'closed')
+        return true;
+    const dataOrientation = element.getAttribute('data-orientation');
+    if (dataOrientation === 'horizontal' || dataOrientation === 'vertical') {
+        return true;
+    }
+    // Check for accordion/collapse patterns
+    if (element.getAttribute('data-accordion-item') !== null)
+        return true;
+    if (element.getAttribute('data-radix-collection-item') !== null)
+        return true;
+    return false;
+}
+function isElementHidden(element) {
+    const style = element.getAttribute('style') ?? '';
+    return (element.getAttribute('hidden') !== null ||
+        element.getAttribute('aria-hidden') === 'true' ||
+        /\bdisplay\s*:\s*none\b/i.test(style) ||
+        /\bvisibility\s*:\s*hidden\b/i.test(style));
+}
+function hasNoiseRole(role) {
+    return role !== null && NAVIGATION_ROLES.has(role);
+}
+function tokenizeIdentifierLikeText(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, ' ')
+        .trim()
+        .split(' ')
+        .filter(Boolean);
+}
+function matchesPromoIdOrClass(className, id) {
+    const tokens = tokenizeIdentifierLikeText(`${className} ${id}`);
+    const promoTokens = getPromoTokens();
+    return tokens.some((token) => promoTokens.has(token));
+}
+function matchesFixedOrHighZIsolate(className) {
+    return (FIXED_PATTERN.test(className) ||
+        (HIGH_Z_PATTERN.test(className) && ISOLATE_PATTERN.test(className)));
+}
+function readElementMetadata(element) {
+    return {
+        tagName: element.tagName.toLowerCase(),
+        className: element.getAttribute('class') ?? '',
+        id: element.getAttribute('id') ?? '',
+        role: element.getAttribute('role'),
+        isHidden: isElementHidden(element),
+    };
+}
+function isBoilerplateHeader({ className, id, role, }) {
+    if (hasNoiseRole(role))
+        return true;
+    const combined = `${className} ${id}`.toLowerCase();
+    return HEADER_NOISE_PATTERN.test(combined);
+}
+function isNoiseElement(node) {
+    const metadata = readElementMetadata(node);
+    const isComplementaryAside = metadata.tagName === 'aside' && metadata.role === 'complementary';
+    const shouldCheckHidden = metadata.isHidden && !isInteractiveComponent(node);
+    const isInteractiveStructural = isStructuralNoiseTag(metadata.tagName) && isInteractiveComponent(node);
+    return ((isStructuralNoiseTag(metadata.tagName) && !isInteractiveStructural) ||
+        ALWAYS_NOISE_TAGS.has(metadata.tagName) ||
+        (metadata.tagName === 'header' && isBoilerplateHeader(metadata)) ||
+        shouldCheckHidden ||
+        (!isComplementaryAside && hasNoiseRole(metadata.role)) ||
+        matchesFixedOrHighZIsolate(metadata.className) ||
+        matchesPromoIdOrClass(metadata.className, metadata.id));
+}
+function isNodeListLike(value) {
+    return isObject(value) && typeof value.length === 'number';
+}
+function tryGetNodeListItem(nodes, index) {
+    if (typeof nodes.item === 'function')
+        return nodes.item(index);
+    return (nodes[index] ?? null);
+}
+function removeNoiseFromNodeListLike(nodes, shouldCheckNoise) {
+    for (let index = nodes.length - 1; index >= 0; index -= 1) {
+        const node = tryGetNodeListItem(nodes, index);
+        if (!node)
+            continue;
+        if (isElement(node) && (!shouldCheckNoise || isNoiseElement(node))) {
+            node.remove();
+        }
+    }
+}
+function removeNoiseNodes(nodes, shouldCheckNoise = true) {
+    if (isNodeListLike(nodes)) {
+        removeNoiseFromNodeListLike(nodes, shouldCheckNoise);
+        return;
+    }
+    // Generic iterable: copy to avoid iteration issues while removing.
+    const nodeList = Array.from(nodes);
+    for (const node of nodeList) {
+        if (isElement(node) && (!shouldCheckNoise || isNoiseElement(node))) {
+            node.remove();
+        }
+    }
+}
+function stripNoiseNodes(document) {
+    // Pass 1: Trusted selectors (Common noise)
+    // We trust these selectors match actual noise, so we skip the expensive isNoiseElement check
+    // Add user-configured extra selectors
+    const targetSelectors = buildNoiseSelector(config.noiseRemoval.extraSelectors);
+    const potentialNoiseNodes = document.querySelectorAll(targetSelectors);
+    removeNoiseNodes(potentialNoiseNodes, false);
+    // Second pass: check remaining elements for noise patterns (promo, fixed positioning, etc.)
+    const allElements = document.querySelectorAll(CANDIDATE_NOISE_SELECTOR);
+    removeNoiseNodes(allElements, true);
+}
+// ─────────────────────────────────────────────────────────────────────────────
+// URL Resolution
+// ─────────────────────────────────────────────────────────────────────────────
+// Protocol patterns to skip during URL resolution (fragment, mailto, tel, blob, data, javascript)
+// JavaScript protocol is detected to skip it for XSS prevention, not to evaluate it
+const SKIP_URL_PREFIXES = [
+    '#',
+    'java' + 'script:',
+    'mailto:',
+    'tel:',
+    'data:',
+    'blob:',
+];
+/**
+ * Check if a URL scheme should be skipped during resolution.
+ * These schemes are either fragment-only (#), protocol handlers (mailto, tel),
+ * inline data (data, blob), or javascript: which we skip to avoid XSS.
+ */
+function shouldSkipUrlResolution(url) {
+    const normalized = url.trim().toLowerCase();
+    return SKIP_URL_PREFIXES.some((prefix) => normalized.startsWith(prefix));
+}
+/**
+ * Safely resolve a relative URL to absolute using base URL.
+ */
+function tryResolveUrl(relativeUrl, baseUrl) {
+    try {
+        return new URL(relativeUrl, baseUrl).href;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Resolve anchor hrefs to absolute URLs.
+ */
+function resolveAnchorUrls(document, baseUrl) {
+    for (const anchor of document.querySelectorAll('a[href]')) {
+        const href = anchor.getAttribute('href');
+        if (href && !shouldSkipUrlResolution(href)) {
+            const resolved = tryResolveUrl(href, baseUrl);
+            if (resolved)
+                anchor.setAttribute('href', resolved);
+        }
+    }
+}
+/**
+ * Resolve image srcs to absolute URLs.
+ */
+function resolveImageUrls(document, baseUrl) {
+    for (const img of document.querySelectorAll('img[src]')) {
+        const src = img.getAttribute('src');
+        if (src && !shouldSkipUrlResolution(src)) {
+            const resolved = tryResolveUrl(src, baseUrl);
+            if (resolved)
+                img.setAttribute('src', resolved);
+        }
+    }
+}
+/**
+ * Resolve source srcset to absolute URLs (for picture elements).
+ */
+function resolveSrcsetUrls(document, baseUrl) {
+    for (const source of document.querySelectorAll('source[srcset]')) {
+        const srcset = source.getAttribute('srcset');
+        if (!srcset)
+            continue;
+        // srcset can have multiple URLs with descriptors like "url 1x, url 2x"
+        const resolved = srcset
+            .split(',')
+            .map((entry) => {
+            const parts = entry.trim().split(/\s+/);
+            const url = parts[0];
+            if (url) {
+                const resolvedUrl = tryResolveUrl(url, baseUrl);
+                if (resolvedUrl)
+                    parts[0] = resolvedUrl;
+            }
+            return parts.join(' ');
+        })
+            .join(', ');
+        source.setAttribute('srcset', resolved);
+    }
+}
+/**
+ * Resolve relative URLs in anchor and image elements to absolute URLs.
+ * Fixes broken links/images in markdown output when the source uses relative paths.
+ */
+function resolveRelativeUrls(document, baseUrl) {
+    try {
+        const base = new URL(baseUrl);
+        resolveAnchorUrls(document, base);
+        resolveImageUrls(document, base);
+        resolveSrcsetUrls(document, base);
+    }
+    catch {
+        /* invalid base URL - skip resolution */
+    }
+}
+// ─────────────────────────────────────────────────────────────────────────────
+// Main Export
+// ─────────────────────────────────────────────────────────────────────────────
+/**
+ * Remove noise elements from HTML and resolve relative URLs.
+ * Used as a preprocessing step before markdown conversion.
+ */
+export function removeNoiseFromHtml(html, document, baseUrl) {
+    const shouldParse = isFullDocumentHtml(html) || mayContainNoise(html);
+    if (!shouldParse)
+        return html;
+    try {
+        const resolvedDocument = document ?? parseHTML(html).document;
+        stripNoiseNodes(resolvedDocument);
+        // Resolve relative URLs before converting to markdown
+        if (baseUrl) {
+            resolveRelativeUrls(resolvedDocument, baseUrl);
+        }
+        const bodyInnerHtml = getBodyInnerHtml(resolvedDocument);
+        // Only use body innerHTML if it has substantial content
+        // On some sites (e.g., Framer), noise removal empties body but leaves content in documentElement
+        if (bodyInnerHtml && bodyInnerHtml.trim().length > 100) {
+            return bodyInnerHtml;
+        }
+        const docToString = getDocumentToString(resolvedDocument);
+        if (docToString)
+            return docToString();
+        const documentElementOuterHtml = getDocumentElementOuterHtml(resolvedDocument);
+        if (documentElementOuterHtml)
+            return documentElementOuterHtml;
+        return html;
+    }
+    catch {
+        return html;
+    }
+}

package/dist/errors.d.ts

@@ -8,4 +8,3 @@ export declare class FetchError extends Error {
 export declare function getErrorMessage(error: unknown): string;
 export declare function createErrorWithCode(message: string, code: string): NodeJS.ErrnoException;
 export declare function isSystemError(error: unknown): error is NodeJS.ErrnoException;
-//# sourceMappingURL=errors.d.ts.map

package/dist/errors.js

@@ -1,4 +1,4 @@
-import { isRecord } from './type-guards.js';
+import { isObject } from './type-guards.js';
 const DEFAULT_HTTP_STATUS = 502;
 export class FetchError extends Error {
     url;
@@ -25,7 +25,7 @@ export function getErrorMessage(error) {
     return 'Unknown error';
 }
 function isErrorWithMessage(error) {
-    if (!isRecord(error))
+    if (!isObject(error))
         return false;
     const { message } = error;
     return typeof message === 'string' && message.length > 0;
@@ -35,8 +35,10 @@ export function createErrorWithCode(message, code) {
     return Object.assign(error, { code });
 }
 export function isSystemError(error) {
-    return (error instanceof Error &&
-        'code' in error &&
-        typeof Reflect.get(error, 'code') === 'string');
+    if (!(error instanceof Error))
+        return false;
+    if (!('code' in error))
+        return false;
+    const { code } = error;
+    return typeof code === 'string';
 }
-//# sourceMappingURL=errors.js.map

package/dist/fetch.d.ts

@@ -38,4 +38,3 @@ export declare function readResponseText(response: Response, url: string, maxByt
 }>;
 export declare function fetchNormalizedUrl(normalizedUrl: string, options?: FetchOptions): Promise<string>;
 export {};
-//# sourceMappingURL=fetch.d.ts.map