@j0hanz/superfetch 2.0.0 → 2.1.0

package/dist/tools/utils/fetch-pipeline.js

@@ -5,19 +5,29 @@ import { logDebug } from '../../services/logger.js';
 import { isRecord } from '../../utils/guards.js';
 import { transformToRawUrl } from '../../utils/url-transformer.js';
 import { normalizeUrl } from '../../utils/url-validator.js';
-function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalizedUrl) {
+function attemptCacheRetrieval({ cacheKey, deserialize, cacheNamespace, normalizedUrl, }) {
     if (!cacheKey)
         return null;
     const cached = cache.get(cacheKey);
     if (!cached)
         return null;
-    if (!deserialize)
-        return logCacheMiss('missing deserializer', cacheNamespace, normalizedUrl);
+    if (!deserialize) {
+        logCacheMiss('missing deserializer', cacheNamespace, normalizedUrl);
+        return null;
+    }
     const data = deserialize(cached.content);
-    if (data === undefined)
-        return logCacheMiss('deserialize failure', cacheNamespace, normalizedUrl);
+    if (data === undefined) {
+        logCacheMiss('deserialize failure', cacheNamespace, normalizedUrl);
+        return null;
+    }
     logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
-    return buildCacheHitResult(data, cached.fetchedAt, normalizedUrl, cacheKey);
+    return {
+        data,
+        fromCache: true,
+        url: normalizedUrl,
+        fetchedAt: cached.fetchedAt,
+        cacheKey,
+    };
 }
 function resolveNormalizedUrl(url) {
     const { normalizedUrl: validatedUrl } = normalizeUrl(url);
@@ -27,44 +37,44 @@ function resolveNormalizedUrl(url) {
 export async function executeFetchPipeline(options) {
     const resolvedUrl = resolveNormalizedUrl(options.url);
     logRawUrlTransformation(resolvedUrl);
-    const cacheKey = resolveCacheKey(options, resolvedUrl.normalizedUrl);
-    const cachedResult = attemptCacheRetrieval(cacheKey, options.deserialize, options.cacheNamespace, resolvedUrl.normalizedUrl);
+    const cacheKey = createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
+    const cachedResult = attemptCacheRetrieval({
+        cacheKey,
+        deserialize: options.deserialize,
+        cacheNamespace: options.cacheNamespace,
+        normalizedUrl: resolvedUrl.normalizedUrl,
+    });
     if (cachedResult)
         return cachedResult;
-    const data = await fetchAndTransform(options, resolvedUrl.normalizedUrl);
+    logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl });
+    const fetchOptions = options.signal === undefined ? {} : { signal: options.signal };
+    const html = await fetchNormalizedUrl(resolvedUrl.normalizedUrl, fetchOptions);
+    const data = await options.transform(html, resolvedUrl.normalizedUrl);
     if (cache.isEnabled()) {
-        persistCache(cacheKey, data, options.serialize, resolvedUrl.normalizedUrl);
-    }
-    return buildPipelineResult(resolvedUrl.normalizedUrl, data, cacheKey);
-}
-function resolveCacheKey(options, normalizedUrl) {
-    return createCacheKey(options.cacheNamespace, normalizedUrl, options.cacheVary);
-}
-async function fetchAndTransform(options, normalizedUrl) {
-    const fetchOptions = buildFetchOptions(options);
-    logDebug('Fetching URL', { url: normalizedUrl });
-    const html = await fetchNormalizedUrl(normalizedUrl, fetchOptions);
-    return options.transform(html, normalizedUrl);
-}
-function buildFetchOptions(options) {
-    return options.signal === undefined ? {} : { signal: options.signal };
-}
-function resolveCacheMetadata(data, normalizedUrl) {
-    const metadata = { url: normalizedUrl };
-    const title = extractTitle(data);
-    if (title !== undefined) {
-        metadata.title = title;
+        persistCache({
+            cacheKey,
+            data,
+            serialize: options.serialize,
+            normalizedUrl: resolvedUrl.normalizedUrl,
+        });
     }
-    return metadata;
-}
-function resolveSerializer(serialize) {
-    return serialize ?? JSON.stringify;
+    return {
+        data,
+        fromCache: false,
+        url: resolvedUrl.normalizedUrl,
+        fetchedAt: new Date().toISOString(),
+        cacheKey,
+    };
 }
-function persistCache(cacheKey, data, serialize, normalizedUrl) {
+function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
     if (!cacheKey)
         return;
-    const serializer = resolveSerializer(serialize);
-    const metadata = resolveCacheMetadata(data, normalizedUrl);
+    const serializer = serialize ?? JSON.stringify;
+    const title = extractTitle(data);
+    const metadata = {
+        url: normalizedUrl,
+        ...(title === undefined ? {} : { title }),
+    };
     cache.set(cacheKey, serializer(data), metadata);
 }
 function extractTitle(value) {
@@ -78,7 +88,6 @@ function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
         namespace: cacheNamespace,
         url: normalizedUrl,
     });
-    return null;
 }
 function logRawUrlTransformation(resolvedUrl) {
     if (!resolvedUrl.transformed)
@@ -87,21 +96,3 @@ function logRawUrlTransformation(resolvedUrl) {
         original: resolvedUrl.originalUrl,
     });
 }
-function buildCacheHitResult(data, fetchedAt, url, cacheKey) {
-    return {
-        data,
-        fromCache: true,
-        url,
-        fetchedAt,
-        cacheKey,
-    };
-}
-function buildPipelineResult(url, data, cacheKey) {
-    return {
-        data,
-        fromCache: false,
-        url,
-        fetchedAt: new Date().toISOString(),
-        cacheKey,
-    };
-}

package/dist/tools/utils/frontmatter.d.ts

@@ -0,0 +1,3 @@
+export declare function extractTitleFromRawMarkdown(content: string): string | undefined;
+export declare function addSourceToMarkdown(content: string, url: string): string;
+export declare function hasFrontmatter(trimmed: string): boolean;

package/dist/tools/utils/frontmatter.js

@@ -0,0 +1,73 @@
+function detectLineEnding(content) {
+    return content.includes('\r\n') ? '\r\n' : '\n';
+}
+function findFrontmatterLines(content) {
+    const lineEnding = detectLineEnding(content);
+    const lines = content.split(lineEnding);
+    if (lines[0] !== '---')
+        return null;
+    const endIndex = lines.indexOf('---', 1);
+    if (endIndex === -1)
+        return null;
+    return { lineEnding, lines, endIndex };
+}
+function stripOptionalQuotes(value) {
+    const trimmed = value.trim();
+    if (trimmed.length < 2)
+        return trimmed;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+function parseFrontmatterEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const separatorIndex = trimmed.indexOf(':');
+    if (separatorIndex <= 0)
+        return null;
+    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
+    const value = trimmed.slice(separatorIndex + 1);
+    return { key, value };
+}
+function isTitleKey(key) {
+    return key === 'title' || key === 'name';
+}
+export function extractTitleFromRawMarkdown(content) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter)
+        return undefined;
+    const { lines, endIndex } = frontmatter;
+    const entry = lines
+        .slice(1, endIndex)
+        .map((line) => parseFrontmatterEntry(line))
+        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
+    if (!entry)
+        return undefined;
+    const value = stripOptionalQuotes(entry.value);
+    return value || undefined;
+}
+export function addSourceToMarkdown(content, url) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter) {
+        return `---\nsource: "${url}"\n---\n\n${content}`;
+    }
+    const { lineEnding, lines, endIndex } = frontmatter;
+    const bodyLines = lines.slice(1, endIndex);
+    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
+    if (hasSource)
+        return content;
+    const updatedLines = [
+        lines[0],
+        ...bodyLines,
+        `source: "${url}"`,
+        ...lines.slice(endIndex),
+    ];
+    return updatedLines.join(lineEnding);
+}
+export function hasFrontmatter(trimmed) {
+    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+}

package/dist/tools/utils/markdown-heuristics.d.ts

@@ -0,0 +1 @@
+export declare function looksLikeMarkdown(content: string): boolean;

package/dist/tools/utils/markdown-heuristics.js

@@ -0,0 +1,19 @@
+const HEADING_PATTERN = /^#{1,6}\s/m;
+const LIST_PATTERN = /^(?:[-*+])\s/m;
+export function looksLikeMarkdown(content) {
+    return (containsMarkdownHeading(content) ||
+        containsMarkdownList(content) ||
+        containsFencedCodeBlock(content));
+}
+function containsMarkdownHeading(content) {
+    return HEADING_PATTERN.test(content);
+}
+function containsMarkdownList(content) {
+    return LIST_PATTERN.test(content);
+}
+function containsFencedCodeBlock(content) {
+    const first = content.indexOf('```');
+    if (first === -1)
+        return false;
+    return content.includes('```', first + 3);
+}

package/dist/tools/utils/markdown-signals.d.ts

@@ -0,0 +1 @@
+export declare function looksLikeMarkdown(content: string): boolean;

package/dist/tools/utils/markdown-signals.js

@@ -0,0 +1,19 @@
+const HEADING_PATTERN = /^#{1,6}\s/m;
+const LIST_PATTERN = /^(?:[-*+])\s/m;
+export function looksLikeMarkdown(content) {
+    return (containsMarkdownHeading(content) ||
+        containsMarkdownList(content) ||
+        containsFencedCodeBlock(content));
+}
+function containsMarkdownHeading(content) {
+    return HEADING_PATTERN.test(content);
+}
+function containsMarkdownList(content) {
+    return LIST_PATTERN.test(content);
+}
+function containsFencedCodeBlock(content) {
+    const first = content.indexOf('```');
+    if (first === -1)
+        return false;
+    return content.includes('```', first + 3);
+}

package/dist/tools/utils/raw-markdown-frontmatter.d.ts

@@ -0,0 +1,3 @@
+export declare function extractTitleFromRawMarkdown(content: string): string | undefined;
+export declare function addSourceToMarkdown(content: string, url: string): string;
+export declare function hasFrontmatter(trimmed: string): boolean;

package/dist/tools/utils/raw-markdown-frontmatter.js

@@ -0,0 +1,73 @@
+function detectLineEnding(content) {
+    return content.includes('\r\n') ? '\r\n' : '\n';
+}
+function findFrontmatterLines(content) {
+    const lineEnding = detectLineEnding(content);
+    const lines = content.split(lineEnding);
+    if (lines[0] !== '---')
+        return null;
+    const endIndex = lines.indexOf('---', 1);
+    if (endIndex === -1)
+        return null;
+    return { lineEnding, lines, endIndex };
+}
+function stripOptionalQuotes(value) {
+    const trimmed = value.trim();
+    if (trimmed.length < 2)
+        return trimmed;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+function parseFrontmatterEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const separatorIndex = trimmed.indexOf(':');
+    if (separatorIndex <= 0)
+        return null;
+    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
+    const value = trimmed.slice(separatorIndex + 1);
+    return { key, value };
+}
+function isTitleKey(key) {
+    return key === 'title' || key === 'name';
+}
+export function extractTitleFromRawMarkdown(content) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter)
+        return undefined;
+    const { lines, endIndex } = frontmatter;
+    const entry = lines
+        .slice(1, endIndex)
+        .map((line) => parseFrontmatterEntry(line))
+        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
+    if (!entry)
+        return undefined;
+    const value = stripOptionalQuotes(entry.value);
+    return value || undefined;
+}
+export function addSourceToMarkdown(content, url) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter) {
+        return `---\nsource: "${url}"\n---\n\n${content}`;
+    }
+    const { lineEnding, lines, endIndex } = frontmatter;
+    const bodyLines = lines.slice(1, endIndex);
+    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
+    if (hasSource)
+        return content;
+    const updatedLines = [
+        lines[0],
+        ...bodyLines,
+        `source: "${url}"`,
+        ...lines.slice(endIndex),
+    ];
+    return updatedLines.join(lineEnding);
+}
+export function hasFrontmatter(trimmed) {
+    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+}

package/dist/tools/utils/raw-markdown.d.ts

@@ -0,0 +1,6 @@
+import type { MarkdownTransformResult } from '../../config/types/content.js';
+export declare function tryTransformRawContent({ html, url, includeMetadata, }: {
+    html: string;
+    url: string;
+    includeMetadata: boolean;
+}): MarkdownTransformResult | null;

package/dist/tools/utils/raw-markdown.js

@@ -0,0 +1,149 @@
+import { logDebug } from '../../services/logger.js';
+import { isRawTextContentUrl } from '../../utils/url-transformer.js';
+const HEADING_PATTERN = /^#{1,6}\s/m;
+const LIST_PATTERN = /^(?:[-*+])\s/m;
+const HTML_DOCUMENT_PATTERN = /^(<!doctype|<html)/i;
+function containsMarkdownHeading(content) {
+    return HEADING_PATTERN.test(content);
+}
+function containsMarkdownList(content) {
+    return LIST_PATTERN.test(content);
+}
+function containsFencedCodeBlock(content) {
+    const first = content.indexOf('```');
+    if (first === -1)
+        return false;
+    return content.includes('```', first + 3);
+}
+function looksLikeMarkdown(content) {
+    return (containsMarkdownHeading(content) ||
+        containsMarkdownList(content) ||
+        containsFencedCodeBlock(content));
+}
+function detectLineEnding(content) {
+    return content.includes('\r\n') ? '\r\n' : '\n';
+}
+function findFrontmatterLines(content) {
+    const lineEnding = detectLineEnding(content);
+    const lines = content.split(lineEnding);
+    if (lines[0] !== '---')
+        return null;
+    const endIndex = lines.indexOf('---', 1);
+    if (endIndex === -1)
+        return null;
+    return { lineEnding, lines, endIndex };
+}
+function stripOptionalQuotes(value) {
+    const trimmed = value.trim();
+    if (trimmed.length < 2)
+        return trimmed;
+    const first = trimmed[0];
+    const last = trimmed[trimmed.length - 1];
+    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+        return trimmed.slice(1, -1).trim();
+    }
+    return trimmed;
+}
+function parseFrontmatterEntry(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    const separatorIndex = trimmed.indexOf(':');
+    if (separatorIndex <= 0)
+        return null;
+    const key = trimmed.slice(0, separatorIndex).trim().toLowerCase();
+    const value = trimmed.slice(separatorIndex + 1);
+    return { key, value };
+}
+function isTitleKey(key) {
+    return key === 'title' || key === 'name';
+}
+function extractTitleFromRawMarkdown(content) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter)
+        return undefined;
+    const { lines, endIndex } = frontmatter;
+    const entry = lines
+        .slice(1, endIndex)
+        .map((line) => parseFrontmatterEntry(line))
+        .find((parsed) => parsed !== null && isTitleKey(parsed.key));
+    if (!entry)
+        return undefined;
+    const value = stripOptionalQuotes(entry.value);
+    return value || undefined;
+}
+function addSourceToMarkdown(content, url) {
+    const frontmatter = findFrontmatterLines(content);
+    if (!frontmatter) {
+        return `---\nsource: "${url}"\n---\n\n${content}`;
+    }
+    const { lineEnding, lines, endIndex } = frontmatter;
+    const bodyLines = lines.slice(1, endIndex);
+    const hasSource = bodyLines.some((line) => line.trimStart().toLowerCase().startsWith('source:'));
+    if (hasSource)
+        return content;
+    const updatedLines = [
+        lines[0],
+        ...bodyLines,
+        `source: "${url}"`,
+        ...lines.slice(endIndex),
+    ];
+    return updatedLines.join(lineEnding);
+}
+function hasFrontmatter(trimmed) {
+    return trimmed.startsWith('---\n') || trimmed.startsWith('---\r\n');
+}
+function looksLikeHtmlDocument(trimmed) {
+    return HTML_DOCUMENT_PATTERN.test(trimmed);
+}
+function countCommonHtmlTags(content) {
+    const matches = content.match(/<(html|head|body|div|span|script|style|meta|link)\b/gi) ??
+        [];
+    return matches.length;
+}
+function isRawTextContent(content) {
+    const trimmed = content.trim();
+    const isHtmlDocument = looksLikeHtmlDocument(trimmed);
+    const hasMarkdownFrontmatter = hasFrontmatter(trimmed);
+    const hasTooManyHtmlTags = countCommonHtmlTags(content) > 2;
+    const isMarkdown = looksLikeMarkdown(content);
+    return (!isHtmlDocument &&
+        (hasMarkdownFrontmatter || (!hasTooManyHtmlTags && isMarkdown)));
+}
+function isLikelyHtmlContent(content) {
+    const trimmed = content.trim();
+    if (!trimmed)
+        return false;
+    if (looksLikeHtmlDocument(trimmed))
+        return true;
+    return countCommonHtmlTags(content) > 2;
+}
+function shouldPreserveRawContent(url, content) {
+    if (isRawTextContentUrl(url)) {
+        return !isLikelyHtmlContent(content);
+    }
+    return isRawTextContent(content);
+}
+function buildRawMarkdownPayload({ rawContent, url, includeMetadata, }) {
+    const title = extractTitleFromRawMarkdown(rawContent);
+    const content = includeMetadata
+        ? addSourceToMarkdown(rawContent, url)
+        : rawContent;
+    return { content, title };
+}
+export function tryTransformRawContent({ html, url, includeMetadata, }) {
+    if (!shouldPreserveRawContent(url, html)) {
+        return null;
+    }
+    logDebug('Preserving raw markdown content', { url: url.substring(0, 80) });
+    const { content, title } = buildRawMarkdownPayload({
+        rawContent: html,
+        url,
+        includeMetadata,
+    });
+    return {
+        markdown: content,
+        title,
+        truncated: false,
+    };
+}

package/dist/tools.d.ts

@@ -0,0 +1,104 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { type MarkdownTransformResult } from './transform.js';
+export interface FetchUrlInput {
+    url: string;
+}
+export interface ToolContentBlock {
+    type: 'text';
+    text: string;
+}
+export interface ToolContentResourceLinkBlock {
+    type: 'resource_link';
+    uri: string;
+    name: string;
+    title?: string;
+    description?: string;
+    mimeType?: string;
+}
+export interface ToolContentResourceBlock {
+    type: 'resource';
+    resource: {
+        uri: string;
+        mimeType?: string;
+        text: string;
+    };
+}
+export type ToolContentBlockUnion = ToolContentBlock | ToolContentResourceLinkBlock | ToolContentResourceBlock;
+export interface ToolErrorResponse {
+    [x: string]: unknown;
+    content: ToolContentBlockUnion[];
+    structuredContent: {
+        error: string;
+        url: string;
+    };
+    isError: true;
+}
+export interface ToolResponseBase {
+    [x: string]: unknown;
+    content: ToolContentBlockUnion[];
+    structuredContent?: Record<string, unknown>;
+    isError?: boolean;
+}
+export interface FetchPipelineOptions<T> {
+    /** URL to fetch */
+    url: string;
+    /** Cache namespace (e.g., 'markdown') */
+    cacheNamespace: string;
+    /** Optional: AbortSignal for request cancellation */
+    signal?: AbortSignal;
+    /** Optional: cache variation input for headers/flags */
+    cacheVary?: Record<string, unknown> | string;
+    /** Transform function to process HTML into desired format */
+    transform: (html: string, url: string) => T | Promise<T>;
+    /** Optional: serialize result for caching (defaults to JSON.stringify) */
+    serialize?: (result: T) => string;
+    /** Optional: deserialize cached content */
+    deserialize?: (cached: string) => T | undefined;
+}
+export interface PipelineResult<T> {
+    data: T;
+    fromCache: boolean;
+    url: string;
+    fetchedAt: string;
+    cacheKey?: string | null;
+}
+export declare const FETCH_URL_TOOL_NAME = "fetch-url";
+export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format";
+interface InlineContentResult {
+    content?: string;
+    contentSize: number;
+    resourceUri?: string;
+    resourceMimeType?: string;
+    error?: string;
+    truncated?: boolean;
+}
+declare function applyInlineContentLimit(content: string, cacheKey: string | null): InlineContentResult;
+export type InlineResult = ReturnType<typeof applyInlineContentLimit>;
+export declare function executeFetchPipeline<T>(options: FetchPipelineOptions<T>): Promise<PipelineResult<T>>;
+interface SharedFetchOptions<T extends {
+    content: string;
+}> {
+    readonly url: string;
+    readonly transform: (html: string, normalizedUrl: string) => T | Promise<T>;
+    readonly serialize?: (result: T) => string;
+    readonly deserialize?: (cached: string) => T | undefined;
+}
+interface SharedFetchDeps {
+    readonly executeFetchPipeline?: typeof executeFetchPipeline;
+}
+export declare function performSharedFetch<T extends {
+    content: string;
+}>(options: SharedFetchOptions<T>, deps?: SharedFetchDeps): Promise<{
+    pipeline: PipelineResult<T>;
+    inlineResult: InlineResult;
+}>;
+export declare function createToolErrorResponse(message: string, url: string): ToolErrorResponse;
+export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string): ToolErrorResponse;
+type MarkdownPipelineResult = MarkdownTransformResult & {
+    readonly content: string;
+};
+export declare function parseCachedMarkdownResult(cached: string): MarkdownPipelineResult | undefined;
+export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<ToolResponseBase>;
+export declare function withRequestContextIfMissing<TParams, TResult>(handler: (params: TParams) => Promise<TResult>): (params: TParams) => Promise<TResult>;
+export declare function registerTools(server: McpServer): void;
+export {};