@j0hanz/superfetch 2.0.0 → 2.1.0

package/dist/transformers/markdown/fenced-code-rule.d.ts

@@ -0,0 +1,2 @@
+import type TurndownService from 'turndown';
+export declare function addFencedCodeRule(instance: TurndownService): void;

package/dist/transformers/markdown/fenced-code-rule.js

@@ -0,0 +1,38 @@
+import { CODE_BLOCK } from '../../config/formatting.js';
+import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../../utils/code-language.js';
+import { isRecord } from '../../utils/guards.js';
+function isElement(node) {
+    return (isRecord(node) &&
+        'getAttribute' in node &&
+        typeof node.getAttribute === 'function');
+}
+function isFencedCodeBlock(node, options) {
+    return (options.codeBlockStyle === 'fenced' &&
+        node.nodeName === 'PRE' &&
+        node.firstChild?.nodeName === 'CODE');
+}
+function formatFencedCodeBlock(node) {
+    const codeNode = node.firstChild;
+    if (!isElement(codeNode))
+        return '';
+    const code = codeNode.textContent || '';
+    const language = resolveCodeLanguage(codeNode, code);
+    return CODE_BLOCK.format(code, language);
+}
+function resolveCodeLanguage(codeNode, code) {
+    const { className, dataLanguage } = readCodeAttributes(codeNode);
+    const attributeLanguage = resolveLanguageFromAttributes(className, dataLanguage);
+    return attributeLanguage ?? detectLanguageFromCode(code) ?? '';
+}
+function readCodeAttributes(codeNode) {
+    return {
+        className: codeNode.getAttribute('class') ?? '',
+        dataLanguage: codeNode.getAttribute('data-language') ?? '',
+    };
+}
+export function addFencedCodeRule(instance) {
+    instance.addRule('fencedCodeBlockWithLanguage', {
+        filter: (node, options) => isFencedCodeBlock(node, options),
+        replacement: (_content, node) => formatFencedCodeBlock(node),
+    });
+}

package/dist/transformers/markdown/frontmatter.d.ts

@@ -0,0 +1,2 @@
+import type { MetadataBlock } from '../../config/types/content.js';
+export declare function buildFrontmatter(metadata?: MetadataBlock): string;

package/dist/transformers/markdown/frontmatter.js

@@ -0,0 +1,45 @@
+import { FRONTMATTER_DELIMITER, joinLines } from '../../config/formatting.js';
+const YAML_SPECIAL_CHARS = /[:[\]{}"\r\t'|>&*!?,#]|\n/;
+const YAML_NUMERIC = /^[\d.]+$/;
+const YAML_RESERVED_WORDS = /^(true|false|null|yes|no|on|off)$/i;
+const ESCAPE_PATTERNS = {
+    backslash: /\\/g,
+    quote: /"/g,
+    newline: /\n/g,
+    tab: /\t/g,
+};
+const YAML_QUOTE_CHECKS = [
+    (input) => YAML_SPECIAL_CHARS.test(input),
+    (input) => input.startsWith(' ') || input.endsWith(' '),
+    (input) => input === '',
+    (input) => YAML_NUMERIC.test(input),
+    (input) => YAML_RESERVED_WORDS.test(input),
+];
+function needsYamlQuotes(value) {
+    return YAML_QUOTE_CHECKS.some((check) => check(value));
+}
+function escapeYamlValue(value) {
+    if (!needsYamlQuotes(value)) {
+        return value;
+    }
+    const escaped = value
+        .replace(ESCAPE_PATTERNS.backslash, '\\\\')
+        .replace(ESCAPE_PATTERNS.quote, '\\"')
+        .replace(ESCAPE_PATTERNS.newline, '\\n')
+        .replace(ESCAPE_PATTERNS.tab, '\\t');
+    return `"${escaped}"`;
+}
+function appendFrontmatterField(lines, key, value) {
+    if (!value)
+        return;
+    lines.push(`${key}: ${escapeYamlValue(value)}`);
+}
+export function buildFrontmatter(metadata) {
+    if (!metadata)
+        return '';
+    const lines = [FRONTMATTER_DELIMITER];
+    appendFrontmatterField(lines, 'title', metadata.title);
+    appendFrontmatterField(lines, 'source', metadata.url);
+    lines.push(FRONTMATTER_DELIMITER);
+    return joinLines(lines);
+}

package/dist/transformers/markdown/noise-rule.d.ts

@@ -0,0 +1,2 @@
+import type TurndownService from 'turndown';
+export declare function addNoiseRule(instance: TurndownService): void;

package/dist/transformers/markdown/noise-rule.js

@@ -0,0 +1,80 @@
+import { isRecord } from '../../utils/guards.js';
+const STRUCTURAL_TAGS = new Set([
+    'script',
+    'style',
+    'noscript',
+    'iframe',
+    'nav',
+    'footer',
+    'aside',
+    'header',
+    'form',
+    'button',
+    'input',
+    'select',
+    'textarea',
+]);
+const NAVIGATION_ROLES = new Set([
+    'navigation',
+    'banner',
+    'complementary',
+    'contentinfo',
+    'tree',
+    'menubar',
+    'menu',
+]);
+const PROMO_PATTERN = /banner|promo|announcement|cta|callout|advert|newsletter|subscribe|cookie|consent|popup|modal|overlay|toast/;
+const FIXED_PATTERN = /\b(fixed|sticky)\b/;
+const HIGH_Z_PATTERN = /\bz-(?:4\d|50)\b/;
+const ISOLATE_PATTERN = /\bisolate\b/;
+function isElement(node) {
+    return (isRecord(node) &&
+        'getAttribute' in node &&
+        typeof node.getAttribute === 'function');
+}
+function isStructuralNoiseTag(tagName) {
+    return (STRUCTURAL_TAGS.has(tagName) || tagName === 'svg' || tagName === 'canvas');
+}
+function isElementHidden(element) {
+    return (element.getAttribute('hidden') !== null ||
+        element.getAttribute('aria-hidden') === 'true');
+}
+function hasNoiseRole(role) {
+    return role !== null && NAVIGATION_ROLES.has(role);
+}
+function matchesPromoIdOrClass(className, id) {
+    const combined = `${className} ${id}`.toLowerCase();
+    return PROMO_PATTERN.test(combined);
+}
+function matchesHighZIsolate(className) {
+    return HIGH_Z_PATTERN.test(className) && ISOLATE_PATTERN.test(className);
+}
+function matchesFixedOrHighZIsolate(className) {
+    return FIXED_PATTERN.test(className) || matchesHighZIsolate(className);
+}
+function readElementMetadata(element) {
+    return {
+        tagName: element.tagName.toLowerCase(),
+        className: element.getAttribute('class') ?? '',
+        id: element.getAttribute('id') ?? '',
+        role: element.getAttribute('role'),
+        isHidden: isElementHidden(element),
+    };
+}
+function isNoiseElement(node) {
+    const metadata = readElementMetadata(node);
+    return (isStructuralNoiseTag(metadata.tagName) ||
+        metadata.isHidden ||
+        hasNoiseRole(metadata.role) ||
+        matchesFixedOrHighZIsolate(metadata.className) ||
+        matchesPromoIdOrClass(metadata.className, metadata.id));
+}
+function isNoiseNode(node) {
+    return isElement(node) && isNoiseElement(node);
+}
+export function addNoiseRule(instance) {
+    instance.addRule('removeNoise', {
+        filter: (node) => isNoiseNode(node),
+        replacement: () => '',
+    });
+}

package/dist/transformers/markdown/turndown-instance.d.ts

@@ -0,0 +1,2 @@
+import TurndownService from 'turndown';
+export declare function getTurndown(): TurndownService;

package/dist/transformers/markdown/turndown-instance.js

@@ -0,0 +1,19 @@
+import TurndownService from 'turndown';
+import { addFencedCodeRule } from './fenced-code-rule.js';
+import { addNoiseRule } from './noise-rule.js';
+let turndownInstance = null;
+function createTurndownInstance() {
+    const instance = new TurndownService({
+        headingStyle: 'atx',
+        codeBlockStyle: 'fenced',
+        emDelimiter: '_',
+        bulletListMarker: '-',
+    });
+    addNoiseRule(instance);
+    addFencedCodeRule(instance);
+    return instance;
+}
+export function getTurndown() {
+    turndownInstance ??= createTurndownInstance();
+    return turndownInstance;
+}

package/dist/transformers/markdown.d.ts

@@ -0,0 +1,5 @@
+import type { MetadataBlock } from '../config/types/content.js';
+export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock, options?: {
+    url?: string;
+    signal?: AbortSignal;
+}): string;

package/dist/transformers/markdown.js

@@ -0,0 +1,314 @@
+import { parseHTML } from 'linkedom';
+import { NodeHtmlMarkdown, } from 'node-html-markdown';
+import { CODE_BLOCK, FRONTMATTER_DELIMITER, joinLines, } from '../config/formatting.js';
+import { FetchError } from '../errors/app-error.js';
+import { endTransformStage, startTransformStage, } from '../services/telemetry.js';
+import { throwIfAborted } from '../utils/cancellation.js';
+import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../utils/code-language.js';
+import { isRecord } from '../utils/guards.js';
+const YAML_SPECIAL_CHARS = /[:[\]{}"\r\t'|>&*!?,#]|\n/;
+const YAML_NUMERIC = /^[\d.]+$/;
+const YAML_RESERVED_WORDS = /^(true|false|null|yes|no|on|off)$/i;
+const ESCAPE_PATTERNS = {
+    backslash: /\\/g,
+    quote: /"/g,
+    newline: /\n/g,
+    tab: /\t/g,
+};
+const YAML_QUOTE_CHECKS = [
+    (input) => YAML_SPECIAL_CHARS.test(input),
+    (input) => input.startsWith(' ') || input.endsWith(' '),
+    (input) => input === '',
+    (input) => YAML_NUMERIC.test(input),
+    (input) => YAML_RESERVED_WORDS.test(input),
+];
+function needsYamlQuotes(value) {
+    return YAML_QUOTE_CHECKS.some((check) => check(value));
+}
+function escapeYamlValue(value) {
+    if (!needsYamlQuotes(value)) {
+        return value;
+    }
+    const escaped = value
+        .replace(ESCAPE_PATTERNS.backslash, '\\\\')
+        .replace(ESCAPE_PATTERNS.quote, '\\"')
+        .replace(ESCAPE_PATTERNS.newline, '\\n')
+        .replace(ESCAPE_PATTERNS.tab, '\\t');
+    return `"${escaped}"`;
+}
+function appendFrontmatterField(lines, key, value) {
+    if (!value)
+        return;
+    lines.push(`${key}: ${escapeYamlValue(value)}`);
+}
+function buildFrontmatter(metadata) {
+    if (!metadata)
+        return '';
+    const lines = [FRONTMATTER_DELIMITER];
+    appendFrontmatterField(lines, 'title', metadata.title);
+    appendFrontmatterField(lines, 'source', metadata.url);
+    appendFrontmatterField(lines, 'author', metadata.author);
+    appendFrontmatterField(lines, 'description', metadata.description);
+    appendFrontmatterField(lines, 'fetchedAt', metadata.fetchedAt);
+    lines.push(FRONTMATTER_DELIMITER);
+    return joinLines(lines);
+}
+function isElement(node) {
+    return (isRecord(node) &&
+        'getAttribute' in node &&
+        typeof node.getAttribute === 'function');
+}
+const STRUCTURAL_TAGS = new Set([
+    'script',
+    'style',
+    'noscript',
+    'iframe',
+    'nav',
+    'footer',
+    'aside',
+    'header',
+    'form',
+    'button',
+    'input',
+    'select',
+    'textarea',
+]);
+const NAVIGATION_ROLES = new Set([
+    'navigation',
+    'banner',
+    'complementary',
+    'contentinfo',
+    'tree',
+    'menubar',
+    'menu',
+]);
+const PROMO_PATTERN = /banner|promo|announcement|cta|callout|advert|newsletter|subscribe|cookie|consent|popup|modal|overlay|toast/;
+const FIXED_PATTERN = /\b(fixed|sticky)\b/;
+const HIGH_Z_PATTERN = /\bz-(?:4\d|50)\b/;
+const ISOLATE_PATTERN = /\bisolate\b/;
+const HTML_DOCUMENT_MARKERS = /<\s*(?:!doctype|html|head|body)\b/i;
+const NOISE_MARKERS = [
+    '<script',
+    '<style',
+    '<noscript',
+    '<iframe',
+    '<nav',
+    '<footer',
+    '<aside',
+    '<header',
+    '<form',
+    '<button',
+    '<input',
+    '<select',
+    '<textarea',
+    '<svg',
+    '<canvas',
+    ' aria-hidden="true"',
+    " aria-hidden='true'",
+    ' hidden',
+    ' role="navigation"',
+    " role='navigation'",
+    ' role="banner"',
+    " role='banner'",
+    ' role="complementary"',
+    " role='complementary'",
+    ' role="contentinfo"',
+    " role='contentinfo'",
+    ' role="tree"',
+    " role='tree'",
+    ' role="menubar"',
+    " role='menubar'",
+    ' role="menu"',
+    " role='menu'",
+    ' banner',
+    ' promo',
+    ' announcement',
+    ' cta',
+    ' callout',
+    ' advert',
+    ' newsletter',
+    ' subscribe',
+    ' cookie',
+    ' consent',
+    ' popup',
+    ' modal',
+    ' overlay',
+    ' toast',
+    ' fixed',
+    ' sticky',
+    ' z-50',
+    ' z-4',
+    ' isolate',
+];
+function mayContainNoise(html) {
+    const haystack = html.toLowerCase();
+    return NOISE_MARKERS.some((marker) => haystack.includes(marker));
+}
+function isFullDocumentHtml(html) {
+    return HTML_DOCUMENT_MARKERS.test(html);
+}
+function isStructuralNoiseTag(tagName) {
+    return (STRUCTURAL_TAGS.has(tagName) || tagName === 'svg' || tagName === 'canvas');
+}
+function isElementHidden(element) {
+    return (element.getAttribute('hidden') !== null ||
+        element.getAttribute('aria-hidden') === 'true');
+}
+function hasNoiseRole(role) {
+    return role !== null && NAVIGATION_ROLES.has(role);
+}
+function matchesPromoIdOrClass(className, id) {
+    const combined = `${className} ${id}`.toLowerCase();
+    return PROMO_PATTERN.test(combined);
+}
+function matchesHighZIsolate(className) {
+    return HIGH_Z_PATTERN.test(className) && ISOLATE_PATTERN.test(className);
+}
+function matchesFixedOrHighZIsolate(className) {
+    return FIXED_PATTERN.test(className) || matchesHighZIsolate(className);
+}
+function readElementMetadata(element) {
+    return {
+        tagName: element.tagName.toLowerCase(),
+        className: element.getAttribute('class') ?? '',
+        id: element.getAttribute('id') ?? '',
+        role: element.getAttribute('role'),
+        isHidden: isElementHidden(element),
+    };
+}
+function isNoiseElement(node) {
+    const metadata = readElementMetadata(node);
+    return (isStructuralNoiseTag(metadata.tagName) ||
+        metadata.isHidden ||
+        hasNoiseRole(metadata.role) ||
+        matchesFixedOrHighZIsolate(metadata.className) ||
+        matchesPromoIdOrClass(metadata.className, metadata.id));
+}
+function removeNoiseFromHtml(html) {
+    const shouldParse = isFullDocumentHtml(html) || mayContainNoise(html);
+    if (!shouldParse)
+        return html;
+    const shouldRemove = mayContainNoise(html);
+    try {
+        const { document } = parseHTML(html);
+        if (shouldRemove) {
+            const nodes = Array.from(document.querySelectorAll('*'));
+            for (let index = nodes.length - 1; index >= 0; index -= 1) {
+                const node = nodes[index];
+                if (!node)
+                    continue;
+                if (isElement(node) && isNoiseElement(node)) {
+                    node.remove();
+                }
+            }
+        }
+        const { body } = document;
+        if (body?.innerHTML)
+            return body.innerHTML;
+        if (typeof document.toString ===
+            'function') {
+            return document.toString();
+        }
+        const { documentElement } = document;
+        if (documentElement?.outerHTML)
+            return documentElement.outerHTML;
+        return html;
+    }
+    catch {
+        return html;
+    }
+}
+function buildInlineCode(content) {
+    const runs = content.match(/`+/g);
+    const longest = runs?.sort((a, b) => b.length - a.length)[0] ?? '';
+    const delimiter = `\`${longest}`;
+    const padding = delimiter.length > 1 ? ' ' : '';
+    return `${delimiter}${padding}${content}${padding}${delimiter}`;
+}
+function isCodeBlock(parent) {
+    if (!isRecord(parent))
+        return false;
+    const tagName = typeof parent.tagName === 'string' ? parent.tagName.toUpperCase() : '';
+    return ['PRE', 'WRAPPED-PRE'].includes(tagName);
+}
+function createCodeTranslator() {
+    return {
+        code: (ctx) => {
+            if (!isRecord(ctx)) {
+                return {
+                    spaceIfRepeatingChar: true,
+                    noEscape: true,
+                    postprocess: ({ content }) => buildInlineCode(content),
+                };
+            }
+            const { node, parent, visitor } = ctx;
+            const getAttribute = isRecord(node) && typeof node.getAttribute === 'function'
+                ? node.getAttribute.bind(node)
+                : undefined;
+            if (!isCodeBlock(parent)) {
+                return {
+                    spaceIfRepeatingChar: true,
+                    noEscape: true,
+                    postprocess: ({ content }) => buildInlineCode(content),
+                };
+            }
+            const className = getAttribute?.('class') ?? '';
+            const dataLanguage = getAttribute?.('data-language') ?? '';
+            const attributeLanguage = resolveLanguageFromAttributes(className, dataLanguage);
+            const childTranslators = isRecord(visitor) ? visitor.instance : null;
+            const codeBlockTranslators = isRecord(childTranslators) &&
+                isRecord(childTranslators
+                    .codeBlockTranslators)
+                ? childTranslators.codeBlockTranslators
+                : null;
+            return {
+                noEscape: true,
+                preserveWhitespace: true,
+                ...(codeBlockTranslators
+                    ? { childTranslators: codeBlockTranslators }
+                    : null),
+                postprocess: ({ content }) => {
+                    const language = attributeLanguage ?? detectLanguageFromCode(content) ?? '';
+                    return CODE_BLOCK.format(content, language);
+                },
+            };
+        },
+    };
+}
+let markdownInstance = null;
+function createMarkdownInstance() {
+    return new NodeHtmlMarkdown({
+        codeFence: CODE_BLOCK.fence,
+        codeBlockStyle: 'fenced',
+        emDelimiter: '_',
+        bulletMarker: '-',
+    }, createCodeTranslator());
+}
+function getMarkdownConverter() {
+    markdownInstance ??= createMarkdownInstance();
+    return markdownInstance;
+}
+export function htmlToMarkdown(html, metadata, options) {
+    const url = options?.url ?? metadata?.url ?? '';
+    const frontmatter = buildFrontmatter(metadata);
+    if (!html)
+        return frontmatter;
+    try {
+        throwIfAborted(options?.signal, url, 'markdown:begin');
+        const noiseStage = startTransformStage(url, 'markdown:noise');
+        const cleanedHtml = removeNoiseFromHtml(html);
+        endTransformStage(noiseStage);
+        throwIfAborted(options?.signal, url, 'markdown:cleaned');
+        const translateStage = startTransformStage(url, 'markdown:translate');
+        const content = getMarkdownConverter().translate(cleanedHtml).trim();
+        endTransformStage(translateStage);
+        throwIfAborted(options?.signal, url, 'markdown:translated');
+        return frontmatter ? `${frontmatter}\n${content}` : content;
+    }
+    catch (error) {
+        if (error instanceof FetchError) {
+            throw error;
+        }
+        return frontmatter;
+    }
+}