@j0hanz/superfetch 2.0.0 → 2.0.1

package/dist/tools/utils/frontmatter.js

@@ -0,0 +1,73 @@
+function detectLineEnding(content) {
+    return content.includes('\r\n') ? '\r\n' : '\n';
+}
+function findFrontmatterLines(content) {
+    const lineEnding = detectLineEnding(content);
+    const lines = content.split(lineEnding);
+    if (lines[0] !== '---')
+        return null;
+    const endIndex = lines.indexOf('---', 1);
+    if (endIndex === -1)
+        return null;
+    return { lineEnding, lines, endIndex };
+}
+function stripOptionalQuotes(value) {
+    const trimmed = value.trim();
+    if (trimmed.length < 2)
+        return trimmed;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+function parseFrontmatterEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const separatorIndex = trimmed.indexOf(':');
+    if (separatorIndex <= 0)
+        return null;
+    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
+    const value = trimmed.slice(separatorIndex + 1);
+    return { key, value };
+}
+function isTitleKey(key) {
+    return key === 'title' || key === 'name';
+}
+export function extractTitleFromRawMarkdown(content) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter)
+        return undefined;
+    const { lines, endIndex } = frontmatter;
+    const entry = lines
+        .slice(1, endIndex)
+        .map((line) => parseFrontmatterEntry(line))
+        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
+    if (!entry)
+        return undefined;
+    const value = stripOptionalQuotes(entry.value);
+    return value || undefined;
+}
+export function addSourceToMarkdown(content, url) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter) {
+        return `---\nsource: "${url}"\n---\n\n${content}`;
+    }
+    const { lineEnding, lines, endIndex } = frontmatter;
+    const bodyLines = lines.slice(1, endIndex);
+    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
+    if (hasSource)
+        return content;
+    const updatedLines = [
+        lines[0],
+        ...bodyLines,
+        `source: "${url}"`,
+        ...lines.slice(endIndex),
+    ];
+    return updatedLines.join(lineEnding);
+}
+export function hasFrontmatter(trimmed) {
+    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+}

package/dist/tools/utils/markdown-heuristics.d.ts

@@ -0,0 +1 @@
+export declare function looksLikeMarkdown(content: string): boolean;

package/dist/tools/utils/markdown-heuristics.js

@@ -0,0 +1,19 @@
+const HEADING_PATTERN = /^#{1,6}\s/m;
+const LIST_PATTERN = /^(?:[-*+])\s/m;
+export function looksLikeMarkdown(content) {
+    return (containsMarkdownHeading(content) ||
+        containsMarkdownList(content) ||
+        containsFencedCodeBlock(content));
+}
+function containsMarkdownHeading(content) {
+    return HEADING_PATTERN.test(content);
+}
+function containsMarkdownList(content) {
+    return LIST_PATTERN.test(content);
+}
+function containsFencedCodeBlock(content) {
+    const first = content.indexOf('```');
+    if (first === -1)
+        return false;
+    return content.includes('```', first + 3);
+}

package/dist/tools/utils/markdown-signals.d.ts

@@ -0,0 +1 @@
+export declare function looksLikeMarkdown(content: string): boolean;

package/dist/tools/utils/markdown-signals.js

@@ -0,0 +1,19 @@
+const HEADING_PATTERN = /^#{1,6}\s/m;
+const LIST_PATTERN = /^(?:[-*+])\s/m;
+export function looksLikeMarkdown(content) {
+    return (containsMarkdownHeading(content) ||
+        containsMarkdownList(content) ||
+        containsFencedCodeBlock(content));
+}
+function containsMarkdownHeading(content) {
+    return HEADING_PATTERN.test(content);
+}
+function containsMarkdownList(content) {
+    return LIST_PATTERN.test(content);
+}
+function containsFencedCodeBlock(content) {
+    const first = content.indexOf('```');
+    if (first === -1)
+        return false;
+    return content.includes('```', first + 3);
+}

package/dist/tools/utils/raw-markdown-frontmatter.d.ts

@@ -0,0 +1,3 @@
+export declare function extractTitleFromRawMarkdown(content: string): string | undefined;
+export declare function addSourceToMarkdown(content: string, url: string): string;
+export declare function hasFrontmatter(trimmed: string): boolean;

package/dist/tools/utils/raw-markdown-frontmatter.js

@@ -0,0 +1,73 @@
+function detectLineEnding(content) {
+    return content.includes('\r\n') ? '\r\n' : '\n';
+}
+function findFrontmatterLines(content) {
+    const lineEnding = detectLineEnding(content);
+    const lines = content.split(lineEnding);
+    if (lines[0] !== '---')
+        return null;
+    const endIndex = lines.indexOf('---', 1);
+    if (endIndex === -1)
+        return null;
+    return { lineEnding, lines, endIndex };
+}
+function stripOptionalQuotes(value) {
+    const trimmed = value.trim();
+    if (trimmed.length < 2)
+        return trimmed;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+function parseFrontmatterEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const separatorIndex = trimmed.indexOf(':');
+    if (separatorIndex <= 0)
+        return null;
+    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
+    const value = trimmed.slice(separatorIndex + 1);
+    return { key, value };
+}
+function isTitleKey(key) {
+    return key === 'title' || key === 'name';
+}
+export function extractTitleFromRawMarkdown(content) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter)
+        return undefined;
+    const { lines, endIndex } = frontmatter;
+    const entry = lines
+        .slice(1, endIndex)
+        .map((line) => parseFrontmatterEntry(line))
+        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
+    if (!entry)
+        return undefined;
+    const value = stripOptionalQuotes(entry.value);
+    return value || undefined;
+}
+export function addSourceToMarkdown(content, url) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter) {
+        return `---\nsource: "${url}"\n---\n\n${content}`;
+    }
+    const { lineEnding, lines, endIndex } = frontmatter;
+    const bodyLines = lines.slice(1, endIndex);
+    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
+    if (hasSource)
+        return content;
+    const updatedLines = [
+        lines[0],
+        ...bodyLines,
+        `source: "${url}"`,
+        ...lines.slice(endIndex),
+    ];
+    return updatedLines.join(lineEnding);
+}
+export function hasFrontmatter(trimmed) {
+    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+}

package/dist/tools/utils/raw-markdown.d.ts

@@ -0,0 +1,6 @@
+import type { MarkdownTransformResult } from '../../config/types/content.js';
+export declare function tryTransformRawContent({ html, url, includeMetadata, }: {
+    html: string;
+    url: string;
+    includeMetadata: boolean;
+}): MarkdownTransformResult | null;

package/dist/tools/utils/raw-markdown.js

@@ -0,0 +1,135 @@
+import { logDebug } from '../../services/logger.js';
+import { isRawTextContentUrl } from '../../utils/url-transformer.js';
+const HEADING_PATTERN = /^#{1,6}\s/m;
+const LIST_PATTERN = /^(?:[-*+])\s/m;
+const HTML_DOCUMENT_PATTERN = /^(<!doctype|<html)/i;
+function containsMarkdownHeading(content) {
+    return HEADING_PATTERN.test(content);
+}
+function containsMarkdownList(content) {
+    return LIST_PATTERN.test(content);
+}
+function containsFencedCodeBlock(content) {
+    const first = content.indexOf('```');
+    if (first === -1)
+        return false;
+    return content.includes('```', first + 3);
+}
+function looksLikeMarkdown(content) {
+    return (containsMarkdownHeading(content) ||
+        containsMarkdownList(content) ||
+        containsFencedCodeBlock(content));
+}
+function detectLineEnding(content) {
+    return content.includes('\r\n') ? '\r\n' : '\n';
+}
+function findFrontmatterLines(content) {
+    const lineEnding = detectLineEnding(content);
+    const lines = content.split(lineEnding);
+    if (lines[0] !== '---')
+        return null;
+    const endIndex = lines.indexOf('---', 1);
+    if (endIndex === -1)
+        return null;
+    return { lineEnding, lines, endIndex };
+}
+function stripOptionalQuotes(value) {
+    const trimmed = value.trim();
+    if (trimmed.length < 2)
+        return trimmed;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+function parseFrontmatterEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const separatorIndex = trimmed.indexOf(':');
+    if (separatorIndex <= 0)
+        return null;
+    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
+    const value = trimmed.slice(separatorIndex + 1);
+    return { key, value };
+}
+function isTitleKey(key) {
+    return key === 'title' || key === 'name';
+}
+function extractTitleFromRawMarkdown(content) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter)
+        return undefined;
+    const { lines, endIndex } = frontmatter;
+    const entry = lines
+        .slice(1, endIndex)
+        .map((line) => parseFrontmatterEntry(line))
+        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
+    if (!entry)
+        return undefined;
+    const value = stripOptionalQuotes(entry.value);
+    return value || undefined;
+}
+function addSourceToMarkdown(content, url) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter) {
+        return `---\nsource: "${url}"\n---\n\n${content}`;
+    }
+    const { lineEnding, lines, endIndex } = frontmatter;
+    const bodyLines = lines.slice(1, endIndex);
+    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
+    if (hasSource)
+        return content;
+    const updatedLines = [
+        lines[0],
+        ...bodyLines,
+        `source: "${url}"`,
+        ...lines.slice(endIndex),
+    ];
+    return updatedLines.join(lineEnding);
+}
+function hasFrontmatter(trimmed) {
+    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+}
+function looksLikeHtmlDocument(trimmed) {
+    return HTML_DOCUMENT_PATTERN.test(trimmed);
+}
+function countCommonHtmlTags(content) {
+    const matches = content.match(/<(html|head|body|div|span|script|style|meta|link)\b/gi) ??
+        [];
+    return matches.length;
+}
+function isRawTextContent(content) {
+    const trimmed = content.trim();
+    const isHtmlDocument = looksLikeHtmlDocument(trimmed);
+    const hasMarkdownFrontmatter = hasFrontmatter(trimmed);
+    const hasTooManyHtmlTags = countCommonHtmlTags(content) > 2;
+    const isMarkdown = looksLikeMarkdown(content);
+    return (!isHtmlDocument &&
+        (hasMarkdownFrontmatter || (!hasTooManyHtmlTags && isMarkdown)));
+}
+function buildRawMarkdownPayload({ rawContent, url, includeMetadata, }) {
+    const title = extractTitleFromRawMarkdown(rawContent);
+    const content = includeMetadata
+        ? addSourceToMarkdown(rawContent, url)
+        : rawContent;
+    return { content, title };
+}
+export function tryTransformRawContent({ html, url, includeMetadata, }) {
+    if (!isRawTextContentUrl(url) && !isRawTextContent(html)) {
+        return null;
+    }
+    logDebug('Preserving raw markdown content', { url: url.substring(0, 80) });
+    const { content, title } = buildRawMarkdownPayload({
+        rawContent: html,
+        url,
+        includeMetadata,
+    });
+    return {
+        markdown: content,
+        title,
+        truncated: false,
+    };
+}

package/dist/transformers/markdown/fenced-code-rule.d.ts

@@ -0,0 +1,2 @@
+import type TurndownService from 'turndown';
+export declare function addFencedCodeRule(instance: TurndownService): void;

package/dist/transformers/markdown/fenced-code-rule.js

@@ -0,0 +1,38 @@
+import { CODE_BLOCK } from '../../config/formatting.js';
+import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../../utils/code-language.js';
+import { isRecord } from '../../utils/guards.js';
+function isElement(node) {
+    return (isRecord(node) &&
+        'getAttribute' in node &&
+        typeof node.getAttribute === 'function');
+}
+function isFencedCodeBlock(node, options) {
+    return (options.codeBlockStyle === 'fenced' &&
+        node.nodeName === 'PRE' &&
+        node.firstChild?.nodeName === 'CODE');
+}
+function formatFencedCodeBlock(node) {
+    const codeNode = node.firstChild;
+    if (!isElement(codeNode))
+        return '';
+    const code = codeNode.textContent || '';
+    const language = resolveCodeLanguage(codeNode, code);
+    return CODE_BLOCK.format(code, language);
+}
+function resolveCodeLanguage(codeNode, code) {
+    const { className, dataLanguage } = readCodeAttributes(codeNode);
+    const attributeLanguage = resolveLanguageFromAttributes(className, dataLanguage);
+    return attributeLanguage ?? detectLanguageFromCode(code) ?? '';
+}
+function readCodeAttributes(codeNode) {
+    return {
+        className: codeNode.getAttribute('class') ?? '',
+        dataLanguage: codeNode.getAttribute('data-language') ?? '',
+    };
+}
+export function addFencedCodeRule(instance) {
+    instance.addRule('fencedCodeBlockWithLanguage', {
+        filter: (node, options) => isFencedCodeBlock(node, options),
+        replacement: (_content, node) => formatFencedCodeBlock(node),
+    });
+}

package/dist/transformers/markdown/frontmatter.d.ts

@@ -0,0 +1,2 @@
+import type { MetadataBlock } from '../../config/types/content.js';
+export declare function buildFrontmatter(metadata?: MetadataBlock): string;

package/dist/transformers/markdown/frontmatter.js

@@ -0,0 +1,45 @@
+import { FRONTMATTER_DELIMITER, joinLines } from '../../config/formatting.js';
+const YAML_SPECIAL_CHARS = /[:[\]{}"\r\t'|>&*!?,#]|\n/;
+const YAML_NUMERIC = /^[\d.]+$/;
+const YAML_RESERVED_WORDS = /^(true|false|null|yes|no|on|off)$/i;
+const ESCAPE_PATTERNS = {
+    backslash: /\\/g,
+    quote: /"/g,
+    newline: /\n/g,
+    tab: /\t/g,
+};
+const YAML_QUOTE_CHECKS = [
+    (input) => YAML_SPECIAL_CHARS.test(input),
+    (input) => input.startsWith(' ') || input.endsWith(' '),
+    (input) => input === '',
+    (input) => YAML_NUMERIC.test(input),
+    (input) => YAML_RESERVED_WORDS.test(input),
+];
+function needsYamlQuotes(value) {
+    return YAML_QUOTE_CHECKS.some((check) => check(value));
+}
+function escapeYamlValue(value) {
+    if (!needsYamlQuotes(value)) {
+        return value;
+    }
+    const escaped = value
+        .replace(ESCAPE_PATTERNS.backslash, '\\\\')
+        .replace(ESCAPE_PATTERNS.quote, '\\"')
+        .replace(ESCAPE_PATTERNS.newline, '\\n')
+        .replace(ESCAPE_PATTERNS.tab, '\\t');
+    return `"${escaped}"`;
+}
+function appendFrontmatterField(lines, key, value) {
+    if (!value)
+        return;
+    lines.push(`${key}: ${escapeYamlValue(value)}`);
+}
+export function buildFrontmatter(metadata) {
+    if (!metadata)
+        return '';
+    const lines = [FRONTMATTER_DELIMITER];
+    appendFrontmatterField(lines, 'title', metadata.title);
+    appendFrontmatterField(lines, 'source', metadata.url);
+    lines.push(FRONTMATTER_DELIMITER);
+    return joinLines(lines);
+}

package/dist/transformers/markdown/noise-rule.d.ts

@@ -0,0 +1,2 @@
+import type TurndownService from 'turndown';
+export declare function addNoiseRule(instance: TurndownService): void;

package/dist/transformers/markdown/noise-rule.js

@@ -0,0 +1,80 @@
+import { isRecord } from '../../utils/guards.js';
+const STRUCTURAL_TAGS = new Set([
+    'script',
+    'style',
+    'noscript',
+    'iframe',
+    'nav',
+    'footer',
+    'aside',
+    'header',
+    'form',
+    'button',
+    'input',
+    'select',
+    'textarea',
+]);
+const NAVIGATION_ROLES = new Set([
+    'navigation',
+    'banner',
+    'complementary',
+    'contentinfo',
+    'tree',
+    'menubar',
+    'menu',
+]);
+const PROMO_PATTERN = /banner|promo|announcement|cta|callout|advert|newsletter|subscribe|cookie|consent|popup|modal|overlay|toast/;
+const FIXED_PATTERN = /\b(fixed|sticky)\b/;
+const HIGH_Z_PATTERN = /\bz-(?:4\d|50)\b/;
+const ISOLATE_PATTERN = /\bisolate\b/;
+function isElement(node) {
+    return (isRecord(node) &&
+        'getAttribute' in node &&
+        typeof node.getAttribute === 'function');
+}
+function isStructuralNoiseTag(tagName) {
+    return (STRUCTURAL_TAGS.has(tagName) || tagName === 'svg' || tagName === 'canvas');
+}
+function isElementHidden(element) {
+    return (element.getAttribute('hidden') !== null ||
+        element.getAttribute('aria-hidden') === 'true');
+}
+function hasNoiseRole(role) {
+    return role !== null && NAVIGATION_ROLES.has(role);
+}
+function matchesPromoIdOrClass(className, id) {
+    const combined = `${className} ${id}`.toLowerCase();
+    return PROMO_PATTERN.test(combined);
+}
+function matchesHighZIsolate(className) {
+    return HIGH_Z_PATTERN.test(className) && ISOLATE_PATTERN.test(className);
+}
+function matchesFixedOrHighZIsolate(className) {
+    return FIXED_PATTERN.test(className) || matchesHighZIsolate(className);
+}
+function readElementMetadata(element) {
+    return {
+        tagName: element.tagName.toLowerCase(),
+        className: element.getAttribute('class') ?? '',
+        id: element.getAttribute('id') ?? '',
+        role: element.getAttribute('role'),
+        isHidden: isElementHidden(element),
+    };
+}
+function isNoiseElement(node) {
+    const metadata = readElementMetadata(node);
+    return (isStructuralNoiseTag(metadata.tagName) ||
+        metadata.isHidden ||
+        hasNoiseRole(metadata.role) ||
+        matchesFixedOrHighZIsolate(metadata.className) ||
+        matchesPromoIdOrClass(metadata.className, metadata.id));
+}
+function isNoiseNode(node) {
+    return isElement(node) && isNoiseElement(node);
+}
+export function addNoiseRule(instance) {
+    instance.addRule('removeNoise', {
+        filter: (node) => isNoiseNode(node),
+        replacement: () => '',
+    });
+}

package/dist/transformers/markdown/turndown-instance.d.ts

@@ -0,0 +1,2 @@
+import TurndownService from 'turndown';
+export declare function getTurndown(): TurndownService;

package/dist/transformers/markdown/turndown-instance.js

@@ -0,0 +1,19 @@
+import TurndownService from 'turndown';
+import { addFencedCodeRule } from './fenced-code-rule.js';
+import { addNoiseRule } from './noise-rule.js';
+let turndownInstance = null;
+function createTurndownInstance() {
+    const instance = new TurndownService({
+        headingStyle: 'atx',
+        codeBlockStyle: 'fenced',
+        emDelimiter: '_',
+        bulletListMarker: '-',
+    });
+    addNoiseRule(instance);
+    addFencedCodeRule(instance);
+    return instance;
+}
+export function getTurndown() {
+    turndownInstance ??= createTurndownInstance();
+    return turndownInstance;
+}

package/dist/transformers/markdown.d.ts

@@ -0,0 +1,2 @@
+import type { MetadataBlock } from '../config/types/content.js';
+export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock): string;