@j0hanz/superfetch 2.0.0 → 2.0.1

package/dist/tools/handlers/fetch-single.shared.d.ts

@@ -1,6 +1,4 @@
-import type { PipelineResult, ToolContentBlock } from '../../config/types/runtime.js';
-import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
-import { applyInlineContentLimit } from '../utils/inline-content.js';
+import type { FetchPipelineOptions, PipelineResult, ToolContentBlock } from '../../config/types/runtime.js';
 interface SharedFetchOptions<T extends {
     content: string;
 }> {
@@ -20,4 +18,14 @@ export declare function performSharedFetch<T extends {
 }>;
 export type InlineResult = ReturnType<typeof applyInlineContentLimit>;
 export declare function buildToolContentBlocks(structuredContent: Record<string, unknown>, fromCache: boolean, inlineResult: InlineResult, resourceName: string, cacheKey?: string | null, fullContent?: string, url?: string, title?: string): ToolContentBlock[];
+interface InlineContentResult {
+    content?: string;
+    contentSize: number;
+    resourceUri?: string;
+    resourceMimeType?: string;
+    error?: string;
+    truncated?: boolean;
+}
+declare function applyInlineContentLimit(content: string, cacheKey: string | null): InlineContentResult;
+export declare function executeFetchPipeline<T>(options: FetchPipelineOptions<T>): Promise<PipelineResult<T>>;
 export {};

package/dist/tools/handlers/fetch-single.shared.js

@@ -1,7 +1,13 @@
+import { TRUNCATION_MARKER } from '../../config/formatting.js';
 import { config } from '../../config/index.js';
+import * as cache from '../../services/cache.js';
+import { createCacheKey, toResourceUri } from '../../services/cache-keys.js';
+import { fetchNormalizedUrl } from '../../services/fetcher.js';
+import { logDebug } from '../../services/logger.js';
 import { generateSafeFilename } from '../../utils/filename-generator.js';
-import { executeFetchPipeline } from '../utils/fetch-pipeline.js';
-import { applyInlineContentLimit } from '../utils/inline-content.js';
+import { isRecord } from '../../utils/guards.js';
+import { transformToRawUrl } from '../../utils/url-transformer.js';
+import { normalizeUrl } from '../../utils/url-validator.js';
 function applyOptionalPipelineSerialization(pipelineOptions, options) {
     if (options.serialize !== undefined) {
         pipelineOptions.serialize = options.serialize;
@@ -92,3 +98,126 @@ export function buildToolContentBlocks(structuredContent, fromCache, inlineResul
     maybeAppendResourceLink(blocks, inlineResult, resourceName);
     return blocks;
 }
+function applyInlineContentLimit(content, cacheKey) {
+    const contentSize = content.length;
+    const inlineLimit = config.constants.maxInlineContentChars;
+    if (contentSize <= inlineLimit) {
+        return { content, contentSize };
+    }
+    const resourceUri = resolveResourceUri(cacheKey);
+    if (!resourceUri) {
+        return buildTruncatedFallback(content, contentSize, inlineLimit);
+    }
+    return {
+        contentSize,
+        resourceUri,
+        resourceMimeType: 'text/markdown',
+    };
+}
+function resolveResourceUri(cacheKey) {
+    if (!config.cache.enabled || !cacheKey)
+        return null;
+    return toResourceUri(cacheKey);
+}
+function buildTruncatedFallback(content, contentSize, inlineLimit) {
+    const maxContentLength = Math.max(0, inlineLimit - TRUNCATION_MARKER.length);
+    const truncatedContent = content.length > inlineLimit
+        ? `${content.substring(0, maxContentLength)}${TRUNCATION_MARKER}`
+        : content;
+    return {
+        content: truncatedContent,
+        contentSize,
+        truncated: true,
+    };
+}
+function attemptCacheRetrieval({ cacheKey, deserialize, cacheNamespace, normalizedUrl, }) {
+    if (!cacheKey)
+        return null;
+    const cached = cache.get(cacheKey);
+    if (!cached)
+        return null;
+    if (!deserialize) {
+        logCacheMiss('missing deserializer', cacheNamespace, normalizedUrl);
+        return null;
+    }
+    const data = deserialize(cached.content);
+    if (data === undefined) {
+        logCacheMiss('deserialize failure', cacheNamespace, normalizedUrl);
+        return null;
+    }
+    logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
+    return {
+        data,
+        fromCache: true,
+        url: normalizedUrl,
+        fetchedAt: cached.fetchedAt,
+        cacheKey,
+    };
+}
+function resolveNormalizedUrl(url) {
+    const { normalizedUrl: validatedUrl } = normalizeUrl(url);
+    const { url: normalizedUrl, transformed } = transformToRawUrl(validatedUrl);
+    return { normalizedUrl, originalUrl: validatedUrl, transformed };
+}
+export async function executeFetchPipeline(options) {
+    const resolvedUrl = resolveNormalizedUrl(options.url);
+    logRawUrlTransformation(resolvedUrl);
+    const cacheKey = createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
+    const cachedResult = attemptCacheRetrieval({
+        cacheKey,
+        deserialize: options.deserialize,
+        cacheNamespace: options.cacheNamespace,
+        normalizedUrl: resolvedUrl.normalizedUrl,
+    });
+    if (cachedResult)
+        return cachedResult;
+    logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl });
+    const fetchOptions = options.signal === undefined ? {} : { signal: options.signal };
+    const html = await fetchNormalizedUrl(resolvedUrl.normalizedUrl, fetchOptions);
+    const data = await options.transform(html, resolvedUrl.normalizedUrl);
+    if (cache.isEnabled()) {
+        persistCache({
+            cacheKey,
+            data,
+            serialize: options.serialize,
+            normalizedUrl: resolvedUrl.normalizedUrl,
+        });
+    }
+    return {
+        data,
+        fromCache: false,
+        url: resolvedUrl.normalizedUrl,
+        fetchedAt: new Date().toISOString(),
+        cacheKey,
+    };
+}
+function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
+    if (!cacheKey)
+        return;
+    const serializer = serialize ?? JSON.stringify;
+    const title = extractTitle(data);
+    const metadata = {
+        url: normalizedUrl,
+        ...(title === undefined ? {} : { title }),
+    };
+    cache.set(cacheKey, serializer(data), metadata);
+}
+function extractTitle(value) {
+    if (!isRecord(value))
+        return undefined;
+    const { title } = value;
+    return typeof title === 'string' ? title : undefined;
+}
+function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
+    logDebug(`Cache miss due to ${reason}`, {
+        namespace: cacheNamespace,
+        url: normalizedUrl,
+    });
+}
+function logRawUrlTransformation(resolvedUrl) {
+    if (!resolvedUrl.transformed)
+        return;
+    logDebug('Using transformed raw content URL', {
+        original: resolvedUrl.originalUrl,
+    });
+}

package/dist/tools/handlers/fetch-url.tool.d.ts

@@ -1,4 +1,10 @@
+import type { MarkdownTransformResult } from '../../config/types/content.js';
 import type { FetchUrlInput, ToolResponseBase } from '../../config/types/tools.js';
 export declare const FETCH_URL_TOOL_NAME = "fetch-url";
 export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format";
+type MarkdownPipelineResult = MarkdownTransformResult & {
+    readonly content: string;
+};
+export declare function parseCachedMarkdownResult(cached: string): MarkdownPipelineResult | undefined;
 export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<ToolResponseBase>;
+export {};

package/dist/tools/handlers/fetch-url.tool.js

@@ -1,10 +1,55 @@
 import { logDebug, logError } from '../../services/logger.js';
+import { isRecord } from '../../utils/guards.js';
 import { createToolErrorResponse, handleToolError, } from '../../utils/tool-error-handler.js';
-import { parseCachedMarkdownResult } from '../utils/cached-markdown.js';
 import { transformHtmlToMarkdown } from '../utils/content-transform.js';
 import { buildToolContentBlocks, performSharedFetch, } from './fetch-single.shared.js';
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
 export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format';
+function parseJsonRecord(input) {
+    try {
+        const parsed = JSON.parse(input);
+        return isRecord(parsed) ? parsed : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function resolveMarkdownContent(parsed) {
+    const { markdown } = parsed;
+    if (typeof markdown === 'string')
+        return markdown;
+    const { content } = parsed;
+    if (typeof content === 'string')
+        return content;
+    return undefined;
+}
+function resolveOptionalTitle(parsed) {
+    const { title } = parsed;
+    if (title === undefined)
+        return undefined;
+    return typeof title === 'string' ? title : undefined;
+}
+function resolveTruncatedFlag(parsed) {
+    const { truncated } = parsed;
+    return typeof truncated === 'boolean' ? truncated : false;
+}
+export function parseCachedMarkdownResult(cached) {
+    const parsed = parseJsonRecord(cached);
+    if (!parsed)
+        return undefined;
+    const resolvedContent = resolveMarkdownContent(parsed);
+    if (resolvedContent === undefined)
+        return undefined;
+    const title = resolveOptionalTitle(parsed);
+    if (parsed.title !== undefined && title === undefined)
+        return undefined;
+    return {
+        content: resolvedContent,
+        markdown: resolvedContent,
+        title,
+        truncated: resolveTruncatedFlag(parsed),
+    };
+}
 function deserializeMarkdownResult(cached) {
     return parseCachedMarkdownResult(cached);
 }
@@ -53,13 +98,10 @@ function buildResponse(pipeline, inlineResult) {
     };
 }
 export async function fetchUrlToolHandler(input) {
-    try {
-        return await executeFetch(input);
-    }
-    catch (error) {
+    return executeFetch(input).catch((error) => {
         logError('fetch-url tool error', error instanceof Error ? error : undefined);
         return handleToolError(error, input.url, 'Failed to fetch URL');
-    }
+    });
 }
 async function executeFetch(input) {
     const { url } = input;

package/dist/tools/utils/content-shaping.js

@@ -1,10 +1,25 @@
 const MIN_CONTENT_RATIO = 0.3;
 const MIN_HTML_LENGTH_FOR_GATE = 100;
+function stripHtmlTags(html) {
+    const parts = [];
+    let inTag = false;
+    for (const char of html) {
+        if (char === '<') {
+            inTag = true;
+            continue;
+        }
+        if (char === '>') {
+            inTag = false;
+            continue;
+        }
+        if (!inTag) {
+            parts.push(char);
+        }
+    }
+    return parts.join('');
+}
 function estimateTextLength(html) {
-    return html
-        .replace(/<[^>]*>/g, '')
-        .replace(/\s+/g, ' ')
-        .trim().length;
+    return stripHtmlTags(html).replace(/\s+/g, ' ').trim().length;
 }
 export function isExtractionSufficient(article, originalHtml) {
     if (!article)

package/dist/tools/utils/content-transform.d.ts

@@ -1,2 +1,5 @@
-import type { MarkdownTransformResult, TransformOptions } from '../../config/types/content.js';
+import type { ExtractedArticle, ExtractedMetadata, MarkdownTransformResult, MetadataBlock, TransformOptions } from '../../config/types/content.js';
+export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtml: string): boolean;
+export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
+export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
 export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): MarkdownTransformResult;

package/dist/tools/utils/content-transform.js

@@ -1,9 +1,75 @@
 import { extractContent } from '../../services/extractor.js';
 import { logDebug } from '../../services/logger.js';
-import { isRawTextContentUrl } from '../../utils/url-transformer.js';
-import { htmlToMarkdown } from '../../transformers/markdown.transformer.js';
-import { createContentMetadataBlock, determineContentExtractionSource, isExtractionSufficient, } from './content-shaping.js';
-function buildArticleContentSource(url, article, extractedMeta, includeMetadata) {
+import { htmlToMarkdown } from '../../transformers/markdown.js';
+import { tryTransformRawContent } from './raw-markdown.js';
+const MIN_CONTENT_RATIO = 0.3;
+const MIN_HTML_LENGTH_FOR_GATE = 100;
+function stripHtmlTags(html) {
+    const parts = [];
+    let inTag = false;
+    for (const char of html) {
+        if (char === '<') {
+            inTag = true;
+            continue;
+        }
+        if (char === '>') {
+            inTag = false;
+            continue;
+        }
+        if (!inTag) {
+            parts.push(char);
+        }
+    }
+    return parts.join('');
+}
+function estimateTextLength(html) {
+    return stripHtmlTags(html).replace(/\s+/g, ' ').trim().length;
+}
+export function isExtractionSufficient(article, originalHtml) {
+    if (!article)
+        return false;
+    const articleLength = article.textContent.length;
+    const originalLength = estimateTextLength(originalHtml);
+    if (originalLength < MIN_HTML_LENGTH_FOR_GATE)
+        return true;
+    return articleLength / originalLength >= MIN_CONTENT_RATIO;
+}
+export function determineContentExtractionSource(article) {
+    return !!article;
+}
+function applyArticleMetadata(metadata, article) {
+    if (article.title !== undefined)
+        metadata.title = article.title;
+    if (article.byline !== undefined)
+        metadata.author = article.byline;
+}
+function applyExtractedMetadata(metadata, extractedMeta) {
+    if (extractedMeta.title !== undefined)
+        metadata.title = extractedMeta.title;
+    if (extractedMeta.description !== undefined) {
+        metadata.description = extractedMeta.description;
+    }
+    if (extractedMeta.author !== undefined) {
+        metadata.author = extractedMeta.author;
+    }
+}
+export function createContentMetadataBlock(url, article, extractedMeta, shouldExtractFromArticle, includeMetadata) {
+    if (!includeMetadata)
+        return undefined;
+    const now = new Date().toISOString();
+    const metadata = {
+        type: 'metadata',
+        url,
+        fetchedAt: now,
+    };
+    if (shouldExtractFromArticle && article) {
+        applyArticleMetadata(metadata, article);
+        return metadata;
+    }
+    applyExtractedMetadata(metadata, extractedMeta);
+    return metadata;
+}
+function buildArticleContentSource({ url, article, extractedMeta, includeMetadata, }) {
     const metadata = createContentMetadataBlock(url, article, extractedMeta, true, includeMetadata);
     return {
         sourceHtml: article.content,
@@ -11,7 +77,7 @@ function buildArticleContentSource(url, article, extractedMeta, includeMetadata)
         metadata,
     };
 }
-function buildFullHtmlContentSource(html, url, article, extractedMeta, includeMetadata) {
+function buildFullHtmlContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
     const metadata = createContentMetadataBlock(url, article, extractedMeta, false, includeMetadata);
     return {
         sourceHtml: html,
@@ -19,119 +85,67 @@ function buildFullHtmlContentSource(html, url, article, extractedMeta, includeMe
         metadata,
     };
 }
-function logQualityGateFallback(url, article) {
+function logQualityGateFallback({ url, articleLength, }) {
     logDebug('Quality gate: Readability extraction below threshold, using full HTML', {
         url: url.substring(0, 80),
-        articleLength: article.textContent.length,
+        articleLength,
     });
 }
-function tryBuildExtractedArticleContentSource(html, url, article, extractedMeta, options) {
+function tryBuildExtractedArticleContentSource({ html, url, article, extractedMeta, includeMetadata, }) {
     if (!article)
         return null;
     const shouldExtractFromArticle = determineContentExtractionSource(article);
     if (shouldExtractFromArticle && isExtractionSufficient(article, html)) {
-        return buildArticleContentSource(url, article, extractedMeta, options.includeMetadata);
+        return buildArticleContentSource({
+            url,
+            article,
+            extractedMeta,
+            includeMetadata,
+        });
     }
     if (shouldExtractFromArticle) {
-        logQualityGateFallback(url, article);
+        logQualityGateFallback({
+            url,
+            articleLength: article.textContent.length,
+        });
     }
     return null;
 }
-function resolveContentSource(html, url, options) {
+function resolveContentSource({ html, url, includeMetadata, }) {
     const { article, metadata: extractedMeta } = extractContent(html, url, {
         extractArticle: true,
     });
-    const extracted = tryBuildExtractedArticleContentSource(html, url, article, extractedMeta, options);
+    const extracted = tryBuildExtractedArticleContentSource({
+        html,
+        url,
+        article,
+        extractedMeta,
+        includeMetadata,
+    });
     if (extracted)
         return extracted;
-    return buildFullHtmlContentSource(html, url, article, extractedMeta, options.includeMetadata);
-}
-function buildMarkdownPayload(context) {
-    return htmlToMarkdown(context.sourceHtml, context.metadata);
-}
-function buildRawMarkdownPayload(rawContent, url, includeMetadata) {
-    const title = extractTitleFromRawMarkdown(rawContent);
-    const content = includeMetadata
-        ? addSourceToMarkdown(rawContent, url)
-        : rawContent;
-    return { content, title };
-}
-function extractTitleFromRawMarkdown(content) {
-    const frontmatterMatch = /^---\r?\n([\s\S]*?)\r?\n---/.exec(content);
-    if (!frontmatterMatch)
-        return undefined;
-    const frontmatter = frontmatterMatch[1] ?? '';
-    const titleMatch = /^(?:title|name):\s*["']?(.+?)["']?\s*$/im.exec(frontmatter);
-    return titleMatch?.[1]?.trim();
-}
-function addSourceToMarkdown(content, url) {
-    const frontmatterMatch = /^(---\r?\n)([\s\S]*?)(\r?\n---)/.exec(content);
-    if (frontmatterMatch) {
-        const start = frontmatterMatch[1] ?? '---\n';
-        const existingFields = frontmatterMatch[2] ?? '';
-        const end = frontmatterMatch[3] ?? '\n---';
-        const rest = content.slice(frontmatterMatch[0].length);
-        if (/^source:/im.test(existingFields)) {
-            return content;
-        }
-        return `${start}${existingFields}\nsource: "${url}"${end}${rest}`;
-    }
-    return `---\nsource: "${url}"\n---\n\n${content}`;
-}
-function looksLikeHtmlDocument(trimmed) {
-    return (trimmed.startsWith('<!DOCTYPE') ||
-        trimmed.startsWith('<!doctype') ||
-        trimmed.startsWith('<html') ||
-        trimmed.startsWith('<HTML'));
-}
-function hasFrontmatter(trimmed) {
-    return /^---\r?\n/.test(trimmed);
-}
-function countCommonHtmlTags(content) {
-    const matches = content.match(/<(html|head|body|div|span|script|style|meta|link)\b/gi) ??
-        [];
-    return matches.length;
-}
-function looksLikeMarkdown(content) {
-    const hasMarkdownHeadings = /^#{1,6}\s+/m.test(content);
-    const hasMarkdownLists = /^[\s]*[-*+]\s+/m.test(content);
-    const hasMarkdownCodeBlocks = /```[\s\S]*?```/.test(content);
-    return hasMarkdownHeadings || hasMarkdownLists || hasMarkdownCodeBlocks;
-}
-function isRawTextContent(content) {
-    const trimmed = content.trim();
-    if (looksLikeHtmlDocument(trimmed)) {
-        return false;
-    }
-    if (hasFrontmatter(trimmed)) {
-        return true;
-    }
-    if (countCommonHtmlTags(content) > 2) {
-        return false;
-    }
-    if (looksLikeMarkdown(content)) {
-        return true;
-    }
-    return false;
-}
-function tryTransformRawContent(html, url, options) {
-    if (!isRawTextContentUrl(url) && !isRawTextContent(html)) {
-        return null;
-    }
-    logDebug('Preserving raw markdown content', { url: url.substring(0, 80) });
-    const { content, title } = buildRawMarkdownPayload(html, url, options.includeMetadata);
-    return {
-        markdown: content,
-        title,
-        truncated: false,
-    };
+    return buildFullHtmlContentSource({
+        html,
+        url,
+        article,
+        extractedMeta,
+        includeMetadata,
+    });
 }
 export function transformHtmlToMarkdown(html, url, options) {
-    const raw = tryTransformRawContent(html, url, options);
+    const raw = tryTransformRawContent({
+        html,
+        url,
+        includeMetadata: options.includeMetadata,
+    });
     if (raw)
         return raw;
-    const context = resolveContentSource(html, url, options);
-    const content = buildMarkdownPayload(context);
+    const context = resolveContentSource({
+        html,
+        url,
+        includeMetadata: options.includeMetadata,
+    });
+    const content = htmlToMarkdown(context.sourceHtml, context.metadata);
     return {
         markdown: content,
         title: context.title,

package/dist/tools/utils/fetch-pipeline.js

@@ -5,19 +5,29 @@ import { logDebug } from '../../services/logger.js';
 import { isRecord } from '../../utils/guards.js';
 import { transformToRawUrl } from '../../utils/url-transformer.js';
 import { normalizeUrl } from '../../utils/url-validator.js';
-function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalizedUrl) {
+function attemptCacheRetrieval({ cacheKey, deserialize, cacheNamespace, normalizedUrl, }) {
     if (!cacheKey)
         return null;
     const cached = cache.get(cacheKey);
     if (!cached)
         return null;
-    if (!deserialize)
-        return logCacheMiss('missing deserializer', cacheNamespace, normalizedUrl);
+    if (!deserialize) {
+        logCacheMiss('missing deserializer', cacheNamespace, normalizedUrl);
+        return null;
+    }
     const data = deserialize(cached.content);
-    if (data === undefined)
-        return logCacheMiss('deserialize failure', cacheNamespace, normalizedUrl);
+    if (data === undefined) {
+        logCacheMiss('deserialize failure', cacheNamespace, normalizedUrl);
+        return null;
+    }
     logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
-    return buildCacheHitResult(data, cached.fetchedAt, normalizedUrl, cacheKey);
+    return {
+        data,
+        fromCache: true,
+        url: normalizedUrl,
+        fetchedAt: cached.fetchedAt,
+        cacheKey,
+    };
 }
 function resolveNormalizedUrl(url) {
     const { normalizedUrl: validatedUrl } = normalizeUrl(url);
@@ -27,44 +37,44 @@ function resolveNormalizedUrl(url) {
 export async function executeFetchPipeline(options) {
     const resolvedUrl = resolveNormalizedUrl(options.url);
     logRawUrlTransformation(resolvedUrl);
-    const cacheKey = resolveCacheKey(options, resolvedUrl.normalizedUrl);
-    const cachedResult = attemptCacheRetrieval(cacheKey, options.deserialize, options.cacheNamespace, resolvedUrl.normalizedUrl);
+    const cacheKey = createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
+    const cachedResult = attemptCacheRetrieval({
+        cacheKey,
+        deserialize: options.deserialize,
+        cacheNamespace: options.cacheNamespace,
+        normalizedUrl: resolvedUrl.normalizedUrl,
+    });
     if (cachedResult)
         return cachedResult;
-    const data = await fetchAndTransform(options, resolvedUrl.normalizedUrl);
+    logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl });
+    const fetchOptions = options.signal === undefined ? {} : { signal: options.signal };
+    const html = await fetchNormalizedUrl(resolvedUrl.normalizedUrl, fetchOptions);
+    const data = await options.transform(html, resolvedUrl.normalizedUrl);
     if (cache.isEnabled()) {
-        persistCache(cacheKey, data, options.serialize, resolvedUrl.normalizedUrl);
-    }
-    return buildPipelineResult(resolvedUrl.normalizedUrl, data, cacheKey);
-}
-function resolveCacheKey(options, normalizedUrl) {
-    return createCacheKey(options.cacheNamespace, normalizedUrl, options.cacheVary);
-}
-async function fetchAndTransform(options, normalizedUrl) {
-    const fetchOptions = buildFetchOptions(options);
-    logDebug('Fetching URL', { url: normalizedUrl });
-    const html = await fetchNormalizedUrl(normalizedUrl, fetchOptions);
-    return options.transform(html, normalizedUrl);
-}
-function buildFetchOptions(options) {
-    return options.signal === undefined ? {} : { signal: options.signal };
-}
-function resolveCacheMetadata(data, normalizedUrl) {
-    const metadata = { url: normalizedUrl };
-    const title = extractTitle(data);
-    if (title !== undefined) {
-        metadata.title = title;
+        persistCache({
+            cacheKey,
+            data,
+            serialize: options.serialize,
+            normalizedUrl: resolvedUrl.normalizedUrl,
+        });
     }
-    return metadata;
-}
-function resolveSerializer(serialize) {
-    return serialize ?? JSON.stringify;
+    return {
+        data,
+        fromCache: false,
+        url: resolvedUrl.normalizedUrl,
+        fetchedAt: new Date().toISOString(),
+        cacheKey,
+    };
 }
-function persistCache(cacheKey, data, serialize, normalizedUrl) {
+function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
     if (!cacheKey)
         return;
-    const serializer = resolveSerializer(serialize);
-    const metadata = resolveCacheMetadata(data, normalizedUrl);
+    const serializer = serialize ?? JSON.stringify;
+    const title = extractTitle(data);
+    const metadata = {
+        url: normalizedUrl,
+        ...(title === undefined ? {} : { title }),
+    };
     cache.set(cacheKey, serializer(data), metadata);
 }
 function extractTitle(value) {
@@ -78,7 +88,6 @@ function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
         namespace: cacheNamespace,
         url: normalizedUrl,
     });
-    return null;
 }
 function logRawUrlTransformation(resolvedUrl) {
     if (!resolvedUrl.transformed)
@@ -87,21 +96,3 @@ function logRawUrlTransformation(resolvedUrl) {
         original: resolvedUrl.originalUrl,
     });
 }
-function buildCacheHitResult(data, fetchedAt, url, cacheKey) {
-    return {
-        data,
-        fromCache: true,
-        url,
-        fetchedAt,
-        cacheKey,
-    };
-}
-function buildPipelineResult(url, data, cacheKey) {
-    return {
-        data,
-        fromCache: false,
-        url,
-        fetchedAt: new Date().toISOString(),
-        cacheKey,
-    };
-}

package/dist/tools/utils/frontmatter.d.ts

@@ -0,0 +1,3 @@
+export declare function extractTitleFromRawMarkdown(content: string): string | undefined;
+export declare function addSourceToMarkdown(content: string, url: string): string;
+export declare function hasFrontmatter(trimmed: string): boolean;