@j0hanz/superfetch 1.2.5 → 2.0.1

package/dist/tools/utils/fetch-pipeline.js

@@ -1,27 +1,23 @@
 import * as cache from '../../services/cache.js';
-import { fetchNormalizedUrlWithRetry } from '../../services/fetcher.js';
+import { createCacheKey } from '../../services/cache-keys.js';
+import { fetchNormalizedUrl } from '../../services/fetcher.js';
 import { logDebug } from '../../services/logger.js';
-import { assertResolvedAddressesAllowed, normalizeUrl, } from '../../utils/url-validator.js';
-import { appendHeaderVary } from './cache-vary.js';
-function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalizedUrl) {
+import { isRecord } from '../../utils/guards.js';
+import { transformToRawUrl } from '../../utils/url-transformer.js';
+import { normalizeUrl } from '../../utils/url-validator.js';
+function attemptCacheRetrieval({ cacheKey, deserialize, cacheNamespace, normalizedUrl, }) {
     if (!cacheKey)
         return null;
     const cached = cache.get(cacheKey);
     if (!cached)
         return null;
     if (!deserialize) {
-        logDebug('Cache miss due to missing deserializer', {
-            namespace: cacheNamespace,
-            url: normalizedUrl,
-        });
+        logCacheMiss('missing deserializer', cacheNamespace, normalizedUrl);
         return null;
     }
     const data = deserialize(cached.content);
     if (data === undefined) {
-        logDebug('Cache miss due to deserialize failure', {
-            namespace: cacheNamespace,
-            url: normalizedUrl,
-        });
+        logCacheMiss('deserialize failure', cacheNamespace, normalizedUrl);
         return null;
     }
     logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
@@ -33,72 +29,70 @@ function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalized
         cacheKey,
     };
 }
-/**
- * Unified fetch pipeline that handles caching, fetching, and transformation.
- * Implements cache-first strategy with automatic serialization.
- *
- * @template T - Type of the transformed result
- * @param options - Pipeline configuration options
- * @returns Promise resolving to the pipeline result
- */
+function resolveNormalizedUrl(url) {
+    const { normalizedUrl: validatedUrl } = normalizeUrl(url);
+    const { url: normalizedUrl, transformed } = transformToRawUrl(validatedUrl);
+    return { normalizedUrl, originalUrl: validatedUrl, transformed };
+}
 export async function executeFetchPipeline(options) {
-    const { normalizedUrl, hostname } = normalizeUrl(options.url);
-    const cacheKey = resolveCacheKey(options, normalizedUrl);
-    const cachedResult = attemptCacheRetrieval(cacheKey, options.deserialize, options.cacheNamespace, normalizedUrl);
+    const resolvedUrl = resolveNormalizedUrl(options.url);
+    logRawUrlTransformation(resolvedUrl);
+    const cacheKey = createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
+    const cachedResult = attemptCacheRetrieval({
+        cacheKey,
+        deserialize: options.deserialize,
+        cacheNamespace: options.cacheNamespace,
+        normalizedUrl: resolvedUrl.normalizedUrl,
+    });
     if (cachedResult)
         return cachedResult;
-    await assertResolvedAddressesAllowed(hostname);
-    const fetchOptions = buildFetchOptions(options);
-    logDebug('Fetching URL', { url: normalizedUrl, retries: options.retries });
-    const html = await fetchNormalizedUrlWithRetry(normalizedUrl, fetchOptions, options.retries);
-    const data = await options.transform(html, normalizedUrl);
+    logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl });
+    const fetchOptions = options.signal === undefined ? {} : { signal: options.signal };
+    const html = await fetchNormalizedUrl(resolvedUrl.normalizedUrl, fetchOptions);
+    const data = await options.transform(html, resolvedUrl.normalizedUrl);
     if (cache.isEnabled()) {
-        persistCache(cacheKey, data, options.serialize, normalizedUrl);
-    }
-    return buildPipelineResult(normalizedUrl, data, cacheKey);
-}
-function resolveCacheKey(options, normalizedUrl) {
-    const cacheVary = appendHeaderVary(options.cacheVary, options.customHeaders);
-    return cache.createCacheKey(options.cacheNamespace, normalizedUrl, cacheVary);
-}
-function buildFetchOptions(options) {
-    const fetchOptions = {};
-    if (options.customHeaders !== undefined) {
-        fetchOptions.customHeaders = options.customHeaders;
-    }
-    if (options.signal !== undefined) {
-        fetchOptions.signal = options.signal;
-    }
-    if (options.timeout !== undefined) {
-        fetchOptions.timeout = options.timeout;
+        persistCache({
+            cacheKey,
+            data,
+            serialize: options.serialize,
+            normalizedUrl: resolvedUrl.normalizedUrl,
+        });
     }
-    return fetchOptions;
+    return {
+        data,
+        fromCache: false,
+        url: resolvedUrl.normalizedUrl,
+        fetchedAt: new Date().toISOString(),
+        cacheKey,
+    };
 }
-function persistCache(cacheKey, data, serialize, normalizedUrl) {
+function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
     if (!cacheKey)
         return;
     const serializer = serialize ?? JSON.stringify;
-    const metadata = { url: normalizedUrl };
     const title = extractTitle(data);
-    if (title !== undefined) {
-        metadata.title = title;
-    }
+    const metadata = {
+        url: normalizedUrl,
+        ...(title === undefined ? {} : { title }),
+    };
     cache.set(cacheKey, serializer(data), metadata);
 }
 function extractTitle(value) {
-    if (!value || typeof value !== 'object')
-        return undefined;
-    if (!('title' in value))
+    if (!isRecord(value))
         return undefined;
     const { title } = value;
     return typeof title === 'string' ? title : undefined;
 }
-function buildPipelineResult(url, data, cacheKey) {
-    return {
-        data,
-        fromCache: false,
-        url,
-        fetchedAt: new Date().toISOString(),
-        cacheKey,
-    };
+function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
+    logDebug(`Cache miss due to ${reason}`, {
+        namespace: cacheNamespace,
+        url: normalizedUrl,
+    });
+}
+function logRawUrlTransformation(resolvedUrl) {
+    if (!resolvedUrl.transformed)
+        return;
+    logDebug('Using transformed raw content URL', {
+        original: resolvedUrl.originalUrl,
+    });
 }

package/dist/tools/utils/frontmatter.d.ts

@@ -0,0 +1,3 @@
+export declare function extractTitleFromRawMarkdown(content: string): string | undefined;
+export declare function addSourceToMarkdown(content: string, url: string): string;
+export declare function hasFrontmatter(trimmed: string): boolean;

package/dist/tools/utils/frontmatter.js

@@ -0,0 +1,73 @@
+function detectLineEnding(content) {
+    return content.includes('\r\n') ? '\r\n' : '\n';
+}
+function findFrontmatterLines(content) {
+    const lineEnding = detectLineEnding(content);
+    const lines = content.split(lineEnding);
+    if (lines[0] !== '---')
+        return null;
+    const endIndex = lines.indexOf('---', 1);
+    if (endIndex === -1)
+        return null;
+    return { lineEnding, lines, endIndex };
+}
+function stripOptionalQuotes(value) {
+    const trimmed = value.trim();
+    if (trimmed.length < 2)
+        return trimmed;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+function parseFrontmatterEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const separatorIndex = trimmed.indexOf(':');
+    if (separatorIndex <= 0)
+        return null;
+    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
+    const value = trimmed.slice(separatorIndex + 1);
+    return { key, value };
+}
+function isTitleKey(key) {
+    return key === 'title' || key === 'name';
+}
+export function extractTitleFromRawMarkdown(content) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter)
+        return undefined;
+    const { lines, endIndex } = frontmatter;
+    const entry = lines
+        .slice(1, endIndex)
+        .map((line) => parseFrontmatterEntry(line))
+        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
+    if (!entry)
+        return undefined;
+    const value = stripOptionalQuotes(entry.value);
+    return value || undefined;
+}
+export function addSourceToMarkdown(content, url) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter) {
+        return `---\nsource: "${url}"\n---\n\n${content}`;
+    }
+    const { lineEnding, lines, endIndex } = frontmatter;
+    const bodyLines = lines.slice(1, endIndex);
+    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
+    if (hasSource)
+        return content;
+    const updatedLines = [
+        lines[0],
+        ...bodyLines,
+        `source: "${url}"`,
+        ...lines.slice(endIndex),
+    ];
+    return updatedLines.join(lineEnding);
+}
+export function hasFrontmatter(trimmed) {
+    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+}

package/dist/tools/utils/inline-content.d.ts

@@ -1,4 +1,3 @@
-type InlineContentFormat = 'jsonl' | 'markdown';
 interface InlineContentResult {
     content?: string;
     contentSize: number;
@@ -7,5 +6,5 @@ interface InlineContentResult {
     error?: string;
     truncated?: boolean;
 }
-export declare function applyInlineContentLimit(content: string, cacheKey: string | null, format: InlineContentFormat): InlineContentResult;
+export declare function applyInlineContentLimit(content: string, cacheKey: string | null): InlineContentResult;
 export {};

package/dist/tools/utils/inline-content.js

@@ -1,7 +1,7 @@
 import { TRUNCATION_MARKER } from '../../config/formatting.js';
 import { config } from '../../config/index.js';
-import * as cache from '../../services/cache.js';
-export function applyInlineContentLimit(content, cacheKey, format) {
+import { toResourceUri } from '../../services/cache-keys.js';
+export function applyInlineContentLimit(content, cacheKey) {
     const contentSize = content.length;
     const inlineLimit = config.constants.maxInlineContentChars;
     if (contentSize <= inlineLimit) {
@@ -14,16 +14,13 @@ export function applyInlineContentLimit(content, cacheKey, format) {
     return {
         contentSize,
         resourceUri,
-        resourceMimeType: resolveResourceMimeType(format),
+        resourceMimeType: 'text/markdown',
     };
 }
 function resolveResourceUri(cacheKey) {
     if (!config.cache.enabled || !cacheKey)
         return null;
-    return cache.toResourceUri(cacheKey);
-}
-function resolveResourceMimeType(format) {
-    return format === 'markdown' ? 'text/markdown' : 'application/jsonl';
+    return toResourceUri(cacheKey);
 }
 function buildTruncatedFallback(content, contentSize, inlineLimit) {
     const maxContentLength = Math.max(0, inlineLimit - TRUNCATION_MARKER.length);

package/dist/tools/utils/markdown-heuristics.d.ts

@@ -0,0 +1 @@
+export declare function looksLikeMarkdown(content: string): boolean;

package/dist/tools/utils/markdown-heuristics.js

@@ -0,0 +1,19 @@
+const HEADING_PATTERN = /^#{1,6}\s/m;
+const LIST_PATTERN = /^(?:[-*+])\s/m;
+export function looksLikeMarkdown(content) {
+    return (containsMarkdownHeading(content) ||
+        containsMarkdownList(content) ||
+        containsFencedCodeBlock(content));
+}
+function containsMarkdownHeading(content) {
+    return HEADING_PATTERN.test(content);
+}
+function containsMarkdownList(content) {
+    return LIST_PATTERN.test(content);
+}
+function containsFencedCodeBlock(content) {
+    const first = content.indexOf('```');
+    if (first === -1)
+        return false;
+    return content.includes('```', first + 3);
+}

package/dist/tools/utils/markdown-signals.d.ts

@@ -0,0 +1 @@
+export declare function looksLikeMarkdown(content: string): boolean;

package/dist/tools/utils/markdown-signals.js

@@ -0,0 +1,19 @@
+const HEADING_PATTERN = /^#{1,6}\s/m;
+const LIST_PATTERN = /^(?:[-*+])\s/m;
+export function looksLikeMarkdown(content) {
+    return (containsMarkdownHeading(content) ||
+        containsMarkdownList(content) ||
+        containsFencedCodeBlock(content));
+}
+function containsMarkdownHeading(content) {
+    return HEADING_PATTERN.test(content);
+}
+function containsMarkdownList(content) {
+    return LIST_PATTERN.test(content);
+}
+function containsFencedCodeBlock(content) {
+    const first = content.indexOf('```');
+    if (first === -1)
+        return false;
+    return content.includes('```', first + 3);
+}

package/dist/tools/utils/raw-markdown-frontmatter.d.ts

@@ -0,0 +1,3 @@
+export declare function extractTitleFromRawMarkdown(content: string): string | undefined;
+export declare function addSourceToMarkdown(content: string, url: string): string;
+export declare function hasFrontmatter(trimmed: string): boolean;

package/dist/tools/utils/raw-markdown-frontmatter.js

@@ -0,0 +1,73 @@
+function detectLineEnding(content) {
+    return content.includes('\r\n') ? '\r\n' : '\n';
+}
+function findFrontmatterLines(content) {
+    const lineEnding = detectLineEnding(content);
+    const lines = content.split(lineEnding);
+    if (lines[0] !== '---')
+        return null;
+    const endIndex = lines.indexOf('---', 1);
+    if (endIndex === -1)
+        return null;
+    return { lineEnding, lines, endIndex };
+}
+function stripOptionalQuotes(value) {
+    const trimmed = value.trim();
+    if (trimmed.length < 2)
+        return trimmed;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+function parseFrontmatterEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const separatorIndex = trimmed.indexOf(':');
+    if (separatorIndex <= 0)
+        return null;
+    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
+    const value = trimmed.slice(separatorIndex + 1);
+    return { key, value };
+}
+function isTitleKey(key) {
+    return key === 'title' || key === 'name';
+}
+export function extractTitleFromRawMarkdown(content) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter)
+        return undefined;
+    const { lines, endIndex } = frontmatter;
+    const entry = lines
+        .slice(1, endIndex)
+        .map((line) => parseFrontmatterEntry(line))
+        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
+    if (!entry)
+        return undefined;
+    const value = stripOptionalQuotes(entry.value);
+    return value || undefined;
+}
+export function addSourceToMarkdown(content, url) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter) {
+        return `---\nsource: "${url}"\n---\n\n${content}`;
+    }
+    const { lineEnding, lines, endIndex } = frontmatter;
+    const bodyLines = lines.slice(1, endIndex);
+    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
+    if (hasSource)
+        return content;
+    const updatedLines = [
+        lines[0],
+        ...bodyLines,
+        `source: "${url}"`,
+        ...lines.slice(endIndex),
+    ];
+    return updatedLines.join(lineEnding);
+}
+export function hasFrontmatter(trimmed) {
+    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+}

package/dist/tools/utils/raw-markdown.d.ts

@@ -0,0 +1,6 @@
+import type { MarkdownTransformResult } from '../../config/types/content.js';
+export declare function tryTransformRawContent({ html, url, includeMetadata, }: {
+    html: string;
+    url: string;
+    includeMetadata: boolean;
+}): MarkdownTransformResult | null;

package/dist/tools/utils/raw-markdown.js

@@ -0,0 +1,135 @@
+import { logDebug } from '../../services/logger.js';
+import { isRawTextContentUrl } from '../../utils/url-transformer.js';
+const HEADING_PATTERN = /^#{1,6}\s/m;
+const LIST_PATTERN = /^(?:[-*+])\s/m;
+const HTML_DOCUMENT_PATTERN = /^(<!doctype|<html)/i;
+function containsMarkdownHeading(content) {
+    return HEADING_PATTERN.test(content);
+}
+function containsMarkdownList(content) {
+    return LIST_PATTERN.test(content);
+}
+function containsFencedCodeBlock(content) {
+    const first = content.indexOf('```');
+    if (first === -1)
+        return false;
+    return content.includes('```', first + 3);
+}
+function looksLikeMarkdown(content) {
+    return (containsMarkdownHeading(content) ||
+        containsMarkdownList(content) ||
+        containsFencedCodeBlock(content));
+}
+function detectLineEnding(content) {
+    return content.includes('\r\n') ? '\r\n' : '\n';
+}
+function findFrontmatterLines(content) {
+    const lineEnding = detectLineEnding(content);
+    const lines = content.split(lineEnding);
+    if (lines[0] !== '---')
+        return null;
+    const endIndex = lines.indexOf('---', 1);
+    if (endIndex === -1)
+        return null;
+    return { lineEnding, lines, endIndex };
+}
+function stripOptionalQuotes(value) {
+    const trimmed = value.trim();
+    if (trimmed.length < 2)
+        return trimmed;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+function parseFrontmatterEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const separatorIndex = trimmed.indexOf(':');
+    if (separatorIndex <= 0)
+        return null;
+    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
+    const value = trimmed.slice(separatorIndex + 1);
+    return { key, value };
+}
+function isTitleKey(key) {
+    return key === 'title' || key === 'name';
+}
+function extractTitleFromRawMarkdown(content) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter)
+        return undefined;
+    const { lines, endIndex } = frontmatter;
+    const entry = lines
+        .slice(1, endIndex)
+        .map((line) => parseFrontmatterEntry(line))
+        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
+    if (!entry)
+        return undefined;
+    const value = stripOptionalQuotes(entry.value);
+    return value || undefined;
+}
+function addSourceToMarkdown(content, url) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter) {
+        return `---\nsource: "${url}"\n---\n\n${content}`;
+    }
+    const { lineEnding, lines, endIndex } = frontmatter;
+    const bodyLines = lines.slice(1, endIndex);
+    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
+    if (hasSource)
+        return content;
+    const updatedLines = [
+        lines[0],
+        ...bodyLines,
+        `source: "${url}"`,
+        ...lines.slice(endIndex),
+    ];
+    return updatedLines.join(lineEnding);
+}
+function hasFrontmatter(trimmed) {
+    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+}
+function looksLikeHtmlDocument(trimmed) {
+    return HTML_DOCUMENT_PATTERN.test(trimmed);
+}
+function countCommonHtmlTags(content) {
+    const matches = content.match(/<(html|head|body|div|span|script|style|meta|link)\b/gi) ??
+        [];
+    return matches.length;
+}
+function isRawTextContent(content) {
+    const trimmed = content.trim();
+    const isHtmlDocument = looksLikeHtmlDocument(trimmed);
+    const hasMarkdownFrontmatter = hasFrontmatter(trimmed);
+    const hasTooManyHtmlTags = countCommonHtmlTags(content) > 2;
+    const isMarkdown = looksLikeMarkdown(content);
+    return (!isHtmlDocument &&
+        (hasMarkdownFrontmatter || (!hasTooManyHtmlTags && isMarkdown)));
+}
+function buildRawMarkdownPayload({ rawContent, url, includeMetadata, }) {
+    const title = extractTitleFromRawMarkdown(rawContent);
+    const content = includeMetadata
+        ? addSourceToMarkdown(rawContent, url)
+        : rawContent;
+    return { content, title };
+}
+export function tryTransformRawContent({ html, url, includeMetadata, }) {
+    if (!isRawTextContentUrl(url) && !isRawTextContent(html)) {
+        return null;
+    }
+    logDebug('Preserving raw markdown content', { url: url.substring(0, 80) });
+    const { content, title } = buildRawMarkdownPayload({
+        rawContent: html,
+        url,
+        includeMetadata,
+    });
+    return {
+        markdown: content,
+        title,
+        truncated: false,
+    };
+}

package/dist/transformers/markdown/fenced-code-rule.d.ts

@@ -0,0 +1,2 @@
+import type TurndownService from 'turndown';
+export declare function addFencedCodeRule(instance: TurndownService): void;

package/dist/transformers/markdown/fenced-code-rule.js

@@ -0,0 +1,38 @@
+import { CODE_BLOCK } from '../../config/formatting.js';
+import { detectLanguageFromCode, resolveLanguageFromAttributes, } from '../../utils/code-language.js';
+import { isRecord } from '../../utils/guards.js';
+function isElement(node) {
+    return (isRecord(node) &&
+        'getAttribute' in node &&
+        typeof node.getAttribute === 'function');
+}
+function isFencedCodeBlock(node, options) {
+    return (options.codeBlockStyle === 'fenced' &&
+        node.nodeName === 'PRE' &&
+        node.firstChild?.nodeName === 'CODE');
+}
+function formatFencedCodeBlock(node) {
+    const codeNode = node.firstChild;
+    if (!isElement(codeNode))
+        return '';
+    const code = codeNode.textContent || '';
+    const language = resolveCodeLanguage(codeNode, code);
+    return CODE_BLOCK.format(code, language);
+}
+function resolveCodeLanguage(codeNode, code) {
+    const { className, dataLanguage } = readCodeAttributes(codeNode);
+    const attributeLanguage = resolveLanguageFromAttributes(className, dataLanguage);
+    return attributeLanguage ?? detectLanguageFromCode(code) ?? '';
+}
+function readCodeAttributes(codeNode) {
+    return {
+        className: codeNode.getAttribute('class') ?? '',
+        dataLanguage: codeNode.getAttribute('data-language') ?? '',
+    };
+}
+export function addFencedCodeRule(instance) {
+    instance.addRule('fencedCodeBlockWithLanguage', {
+        filter: (node, options) => isFencedCodeBlock(node, options),
+        replacement: (_content, node) => formatFencedCodeBlock(node),
+    });
+}

package/dist/transformers/markdown/frontmatter.d.ts

@@ -0,0 +1,2 @@
+import type { MetadataBlock } from '../../config/types/content.js';
+export declare function buildFrontmatter(metadata?: MetadataBlock): string;

package/dist/transformers/markdown/frontmatter.js

@@ -0,0 +1,45 @@
+import { FRONTMATTER_DELIMITER, joinLines } from '../../config/formatting.js';
+const YAML_SPECIAL_CHARS = /[:[\]{}"\r\t'|>&*!?,#]|\n/;
+const YAML_NUMERIC = /^[\d.]+$/;
+const YAML_RESERVED_WORDS = /^(true|false|null|yes|no|on|off)$/i;
+const ESCAPE_PATTERNS = {
+    backslash: /\\/g,
+    quote: /"/g,
+    newline: /\n/g,
+    tab: /\t/g,
+};
+const YAML_QUOTE_CHECKS = [
+    (input) => YAML_SPECIAL_CHARS.test(input),
+    (input) => input.startsWith(' ') || input.endsWith(' '),
+    (input) => input === '',
+    (input) => YAML_NUMERIC.test(input),
+    (input) => YAML_RESERVED_WORDS.test(input),
+];
+function needsYamlQuotes(value) {
+    return YAML_QUOTE_CHECKS.some((check) => check(value));
+}
+function escapeYamlValue(value) {
+    if (!needsYamlQuotes(value)) {
+        return value;
+    }
+    const escaped = value
+        .replace(ESCAPE_PATTERNS.backslash, '\\\\')
+        .replace(ESCAPE_PATTERNS.quote, '\\"')
+        .replace(ESCAPE_PATTERNS.newline, '\\n')
+        .replace(ESCAPE_PATTERNS.tab, '\\t');
+    return `"${escaped}"`;
+}
+function appendFrontmatterField(lines, key, value) {
+    if (!value)
+        return;
+    lines.push(`${key}: ${escapeYamlValue(value)}`);
+}
+export function buildFrontmatter(metadata) {
+    if (!metadata)
+        return '';
+    const lines = [FRONTMATTER_DELIMITER];
+    appendFrontmatterField(lines, 'title', metadata.title);
+    appendFrontmatterField(lines, 'source', metadata.url);
+    lines.push(FRONTMATTER_DELIMITER);
+    return joinLines(lines);
+}

package/dist/transformers/markdown/noise-rule.d.ts

@@ -0,0 +1,2 @@
+import type TurndownService from 'turndown';
+export declare function addNoiseRule(instance: TurndownService): void;