@j0hanz/superfetch 2.4.3 → 2.4.5

package/dist/tools.js

@@ -15,7 +15,7 @@ const fetchUrlInputSchema = z.strictObject({
         .url({ protocol: /^https?$/i })
         .min(1)
         .max(config.constants.maxUrlLength)
-        .describe('The URL to fetch'),
+        .describe('The URL of the webpage to fetch and convert to Markdown'),
 });
 const fetchUrlOutputSchema = z.strictObject({
     url: z
@@ -46,55 +46,130 @@ const fetchUrlOutputSchema = z.strictObject({
         .describe('Error message if the request failed'),
 });
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
-export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format';
+export const FETCH_URL_TOOL_DESCRIPTION = `
+Fetches a webpage and converts it to clean Markdown format optimized for LLM context.
+
+This tool is useful for:
+- Reading documentation, blog posts, or articles.
+- Extracting main content while removing navigation and ads (noise removal).
+- Caching content to speed up repeated queries.
+
+Limitations:
+- Returns truncated content if it exceeds ${config.constants.maxInlineContentChars} characters.
+- Does not execute complex client-side JavaScript interactions.
+`.trim();
+// Specific icon for the fetch-url tool (download cloud / web)
+const TOOL_ICON = {
+    src: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJjdXJyZW50Q29sb3IiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj48cGF0aCBkPSJNMjEgMTV2NGEyIDIgMCAwIDEtMiAySDVhMiAyIDAgMCAxLTItMnYtNCIvPjxwb2x5bGluZSBwb2ludHM9IjcgMTAgMTIgMTUgMTcgMTAiLz48bGluZSB4MT0iMTIiIHkxPSIxNSIgeDI9IjEyIiB5Mj0iMyIvPjwvc3ZnPg==',
+    mimeType: 'image/svg+xml',
+};
+/* -------------------------------------------------------------------------------------------------
+ * Progress reporting
+ * ------------------------------------------------------------------------------------------------- */
+class ToolProgressReporter {
+    token;
+    sendNotification;
+    constructor(token, sendNotification) {
+        this.token = token;
+        this.sendNotification = sendNotification;
+    }
+    static create(extra) {
+        const token = extra?._meta?.progressToken ?? null;
+        const sendNotification = extra?.sendNotification;
+        if (token === null || !sendNotification) {
+            return { report: async () => { } };
+        }
+        return new ToolProgressReporter(token, sendNotification);
+    }
+    async report(progress, message) {
+        try {
+            await Promise.race([
+                this.sendNotification({
+                    method: 'notifications/progress',
+                    params: {
+                        progressToken: this.token,
+                        progress,
+                        total: FETCH_PROGRESS_TOTAL,
+                        message,
+                    },
+                }),
+                new Promise((_, reject) => {
+                    setTimeout(() => {
+                        reject(new Error('Progress notification timeout'));
+                    }, PROGRESS_NOTIFICATION_TIMEOUT_MS);
+                }),
+            ]);
+        }
+        catch (error) {
+            const isTimeout = error instanceof Error &&
+                error.message === 'Progress notification timeout';
+            const logMessage = isTimeout
+                ? 'Progress notification timed out'
+                : 'Failed to send progress notification';
+            logWarn(logMessage, {
+                error: getErrorMessage(error),
+                progress,
+                message,
+            });
+        }
+    }
+}
 export function createProgressReporter(extra) {
-    const token = extra?._meta?.progressToken ?? null;
-    const sendNotification = extra?.sendNotification;
-    if (token === null || !sendNotification) {
-        return { report: async () => { } };
+    return ToolProgressReporter.create(extra);
+}
+class InlineContentLimiter {
+    apply(content, cacheKey) {
+        const contentSize = content.length;
+        const inlineLimit = config.constants.maxInlineContentChars;
+        if (contentSize <= inlineLimit) {
+            return { content, contentSize };
+        }
+        const resourceUri = this.resolveResourceUri(cacheKey);
+        if (!resourceUri) {
+            return this.buildTruncatedFallback(content, contentSize, inlineLimit);
+        }
+        return {
+            contentSize,
+            resourceUri,
+            resourceMimeType: 'text/markdown',
+        };
+    }
+    resolveResourceUri(cacheKey) {
+        if (!cache.isEnabled() || !cacheKey)
+            return null;
+        return cache.toResourceUri(cacheKey);
+    }
+    buildTruncatedFallback(content, contentSize, inlineLimit) {
+        const maxContentLength = Math.max(0, inlineLimit - TRUNCATION_MARKER.length);
+        const truncatedContent = content.length > inlineLimit
+            ? `${content.substring(0, maxContentLength)}${TRUNCATION_MARKER}`
+            : content;
+        return {
+            content: truncatedContent,
+            contentSize,
+            truncated: true,
+        };
     }
-    return {
-        report: async (progress, message) => {
-            try {
-                await Promise.race([
-                    sendNotification({
-                        method: 'notifications/progress',
-                        params: {
-                            progressToken: token,
-                            progress,
-                            total: FETCH_PROGRESS_TOTAL,
-                            message,
-                        },
-                    }),
-                    new Promise((_, reject) => {
-                        setTimeout(() => {
-                            reject(new Error('Progress notification timeout'));
-                        }, PROGRESS_NOTIFICATION_TIMEOUT_MS);
-                    }),
-                ]);
-            }
-            catch (error) {
-                const isTimeout = error instanceof Error &&
-                    error.message === 'Progress notification timeout';
-                const logMessage = isTimeout
-                    ? 'Progress notification timed out'
-                    : 'Failed to send progress notification';
-                logWarn(logMessage, {
-                    error: getErrorMessage(error),
-                    progress,
-                    message,
-                });
-            }
-        },
-    };
 }
+const inlineLimiter = new InlineContentLimiter();
+function applyInlineContentLimit(content, cacheKey) {
+    return inlineLimiter.apply(content, cacheKey);
+}
+/* -------------------------------------------------------------------------------------------------
+ * Tool response blocks (text + optional resource + optional link)
+ * ------------------------------------------------------------------------------------------------- */
 function serializeStructuredContent(structuredContent) {
     return JSON.stringify(structuredContent);
 }
+function buildTextBlock(structuredContent) {
+    return {
+        type: 'text',
+        text: serializeStructuredContent(structuredContent),
+    };
+}
 function buildResourceLink(inlineResult, name) {
-    if (!inlineResult.resourceUri) {
+    if (!inlineResult.resourceUri)
         return null;
-    }
     const block = {
         type: 'resource_link',
         uri: inlineResult.resourceUri,
@@ -107,9 +182,8 @@ function buildResourceLink(inlineResult, name) {
     return block;
 }
 function buildEmbeddedResource(content, url, title) {
-    if (!content) {
+    if (!content)
         return null;
-    }
     const filename = cache.generateSafeFilename(url, title, undefined, '.md');
     const uri = `file:///${filename}`;
     return {
@@ -121,28 +195,21 @@ function buildEmbeddedResource(content, url, title) {
         },
     };
 }
-function appendResourceBlocks({ blocks, inlineResult, resourceName, url, title, fullContent, }) {
+function appendResourceBlocks(params) {
+    const { blocks, inlineResult, resourceName, url, title, fullContent } = params;
     const contentToEmbed = config.runtime.httpMode
         ? inlineResult.content
         : (fullContent ?? inlineResult.content);
     if (contentToEmbed && url) {
-        const embeddedResource = buildEmbeddedResource(contentToEmbed, url, title);
-        if (embeddedResource) {
-            blocks.push(embeddedResource);
-        }
-    }
-    const resourceLink = buildResourceLink(inlineResult, resourceName);
-    if (resourceLink) {
-        blocks.push(resourceLink);
+        const embedded = buildEmbeddedResource(contentToEmbed, url, title);
+        if (embedded)
+            blocks.push(embedded);
     }
+    const link = buildResourceLink(inlineResult, resourceName);
+    if (link)
+        blocks.push(link);
 }
-function buildTextBlock(structuredContent) {
-    return {
-        type: 'text',
-        text: serializeStructuredContent(structuredContent),
-    };
-}
-function buildToolContentBlocks(structuredContent, fromCache, inlineResult, resourceName, cacheKey, fullContent, url, title) {
+function buildToolContentBlocks(structuredContent, _fromCache, inlineResult, resourceName, _cacheKey, fullContent, url, title) {
     const blocks = [buildTextBlock(structuredContent)];
     appendResourceBlocks({
         blocks,
@@ -154,39 +221,36 @@ function buildToolContentBlocks(structuredContent, fromCache, inlineResult, reso
     });
     return blocks;
 }
-function applyInlineContentLimit(content, cacheKey) {
-    const contentSize = content.length;
-    const inlineLimit = config.constants.maxInlineContentChars;
-    if (contentSize <= inlineLimit) {
-        return { content, contentSize };
-    }
-    const resourceUri = resolveResourceUri(cacheKey);
-    if (!resourceUri) {
-        return buildTruncatedFallback(content, contentSize, inlineLimit);
-    }
-    return {
-        contentSize,
-        resourceUri,
-        resourceMimeType: 'text/markdown',
-    };
+/* -------------------------------------------------------------------------------------------------
+ * Fetch pipeline executor (normalize → raw-transform → cache → fetch → transform → persist)
+ * ------------------------------------------------------------------------------------------------- */
+function resolveNormalizedUrl(url) {
+    const { normalizedUrl: validatedUrl } = normalizeUrl(url);
+    const { url: normalizedUrl, transformed } = transformToRawUrl(validatedUrl);
+    return { normalizedUrl, originalUrl: validatedUrl, transformed };
 }
-function resolveResourceUri(cacheKey) {
-    if (!cache.isEnabled() || !cacheKey)
-        return null;
-    return cache.toResourceUri(cacheKey);
+function logRawUrlTransformation(resolvedUrl) {
+    if (!resolvedUrl.transformed)
+        return;
+    logDebug('Using transformed raw content URL', {
+        original: resolvedUrl.originalUrl,
+    });
 }
-function buildTruncatedFallback(content, contentSize, inlineLimit) {
-    const maxContentLength = Math.max(0, inlineLimit - TRUNCATION_MARKER.length);
-    const truncatedContent = content.length > inlineLimit
-        ? `${content.substring(0, maxContentLength)}${TRUNCATION_MARKER}`
-        : content;
-    return {
-        content: truncatedContent,
-        contentSize,
-        truncated: true,
-    };
+function extractTitle(value) {
+    if (!isObject(value))
+        return undefined;
+    const { title } = value;
+    return typeof title === 'string' ? title : undefined;
+}
+function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
+    const log = reason === 'deserialize failure' ? logWarn : logDebug;
+    log(`Cache miss due to ${reason}`, {
+        namespace: cacheNamespace,
+        url: normalizedUrl,
+    });
 }
-function attemptCacheRetrieval({ cacheKey, deserialize, cacheNamespace, normalizedUrl, }) {
+function attemptCacheRetrieval(params) {
+    const { cacheKey, deserialize, cacheNamespace, normalizedUrl } = params;
     if (!cacheKey)
         return null;
     const cached = cache.get(cacheKey);
@@ -210,10 +274,17 @@ function attemptCacheRetrieval({ cacheKey, deserialize, cacheNamespace, normaliz
         cacheKey,
     };
 }
-function resolveNormalizedUrl(url) {
-    const { normalizedUrl: validatedUrl } = normalizeUrl(url);
-    const { url: normalizedUrl, transformed } = transformToRawUrl(validatedUrl);
-    return { normalizedUrl, originalUrl: validatedUrl, transformed };
+function persistCache(params) {
+    const { cacheKey, data, serialize, normalizedUrl } = params;
+    if (!cacheKey)
+        return;
+    const serializer = serialize ?? JSON.stringify;
+    const title = extractTitle(data);
+    const metadata = {
+        url: normalizedUrl,
+        ...(title === undefined ? {} : { title }),
+    };
+    cache.set(cacheKey, serializer(data), metadata);
 }
 export async function executeFetchPipeline(options) {
     const resolvedUrl = resolveNormalizedUrl(options.url);
@@ -247,38 +318,6 @@ export async function executeFetchPipeline(options) {
         cacheKey,
     };
 }
-function persistCache({ cacheKey, data, serialize, normalizedUrl, }) {
-    if (!cacheKey)
-        return;
-    const serializer = serialize ?? JSON.stringify;
-    const title = extractTitle(data);
-    const metadata = {
-        url: normalizedUrl,
-        ...(title === undefined ? {} : { title }),
-    };
-    cache.set(cacheKey, serializer(data), metadata);
-}
-function extractTitle(value) {
-    if (!isObject(value))
-        return undefined;
-    const { title } = value;
-    return typeof title === 'string' ? title : undefined;
-}
-function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
-    // Deserialize failures indicate unexpected data; surface at warn level.
-    const log = reason === 'deserialize failure' ? logWarn : logDebug;
-    log(`Cache miss due to ${reason}`, {
-        namespace: cacheNamespace,
-        url: normalizedUrl,
-    });
-}
-function logRawUrlTransformation(resolvedUrl) {
-    if (!resolvedUrl.transformed)
-        return;
-    logDebug('Using transformed raw content URL', {
-        original: resolvedUrl.originalUrl,
-    });
-}
 export async function performSharedFetch(options, deps = {}) {
     const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
     const pipelineOptions = {
@@ -293,6 +332,9 @@ export async function performSharedFetch(options, deps = {}) {
     const inlineResult = applyInlineContentLimit(pipeline.data.content, pipeline.cacheKey ?? null);
     return { pipeline, inlineResult };
 }
+/* -------------------------------------------------------------------------------------------------
+ * Tool error mapping
+ * ------------------------------------------------------------------------------------------------- */
 export function createToolErrorResponse(message, url) {
     const structuredContent = {
         error: message,
@@ -304,10 +346,6 @@ export function createToolErrorResponse(message, url) {
         isError: true,
     };
 }
-export function handleToolError(error, url, fallbackMessage = 'Operation failed') {
-    const message = resolveToolErrorMessage(error, fallbackMessage);
-    return createToolErrorResponse(message, url);
-}
 function isValidationError(error) {
     return (error instanceof Error &&
         isSystemError(error) &&
@@ -322,6 +360,10 @@ function resolveToolErrorMessage(error, fallbackMessage) {
     }
     return `${fallbackMessage}: Unknown error`;
 }
+export function handleToolError(error, url, fallbackMessage = 'Operation failed') {
+    const message = resolveToolErrorMessage(error, fallbackMessage);
+    return createToolErrorResponse(message, url);
+}
 function parseJsonRecord(input) {
     try {
         const parsed = JSON.parse(input);
@@ -381,6 +423,9 @@ function serializeMarkdownResult(result) {
         truncated: result.truncated,
     });
 }
+/* -------------------------------------------------------------------------------------------------
+ * fetch-url tool implementation
+ * ------------------------------------------------------------------------------------------------- */
 function buildStructuredContent(pipeline, inlineResult, inputUrl) {
     return {
         url: pipeline.url,
@@ -393,6 +438,14 @@ function buildStructuredContent(pipeline, inlineResult, inputUrl) {
 function buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult) {
     return buildToolContentBlocks(structuredContent, pipeline.fromCache, inlineResult, 'Fetched markdown', pipeline.cacheKey, pipeline.data.content, pipeline.url, pipeline.data.title);
 }
+function buildResponse(pipeline, inlineResult, inputUrl) {
+    const structuredContent = buildStructuredContent(pipeline, inlineResult, inputUrl);
+    const content = buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult);
+    return {
+        content,
+        structuredContent,
+    };
+}
 async function fetchPipeline(url, signal, progress) {
     return performSharedFetch({
         url,
@@ -407,14 +460,6 @@ async function fetchPipeline(url, signal, progress) {
         deserialize: parseCachedMarkdownResult,
     });
 }
-function buildResponse(pipeline, inlineResult, inputUrl) {
-    const structuredContent = buildStructuredContent(pipeline, inlineResult, inputUrl);
-    const content = buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult);
-    return {
-        content,
-        structuredContent,
-    };
-}
 async function executeFetch(input, extra) {
     const { url } = input;
     if (!url) {
@@ -432,6 +477,7 @@ async function executeFetch(input, extra) {
     await progress.report(1, 'Validating URL');
     logDebug('Fetching URL', { url });
     await progress.report(2, 'Fetching content');
+    await progress.report(2, 'Fetching content'); // preserve existing behavior
     const { pipeline, inlineResult } = await fetchPipeline(url, signal, progress);
     if (pipeline.fromCache) {
         await progress.report(3, 'Using cached content');
@@ -482,13 +528,16 @@ function resolveRequestIdFromExtra(extra) {
         return String(requestId);
     return undefined;
 }
-export function registerTools(server, serverIcons) {
-    server.registerTool(TOOL_DEFINITION.name, {
-        title: TOOL_DEFINITION.title,
-        description: TOOL_DEFINITION.description,
-        inputSchema: TOOL_DEFINITION.inputSchema,
-        outputSchema: TOOL_DEFINITION.outputSchema,
-        annotations: TOOL_DEFINITION.annotations,
-        ...(serverIcons ? { icons: serverIcons } : {}),
-    }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
+export function registerTools(server) {
+    if (config.tools.enabled.includes(FETCH_URL_TOOL_NAME)) {
+        server.registerTool(TOOL_DEFINITION.name, {
+            title: TOOL_DEFINITION.title,
+            description: TOOL_DEFINITION.description,
+            inputSchema: TOOL_DEFINITION.inputSchema,
+            outputSchema: TOOL_DEFINITION.outputSchema,
+            annotations: TOOL_DEFINITION.annotations,
+            // Use specific tool icon here
+            icons: [TOOL_ICON],
+        }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
+    }
 }

package/dist/transform.d.ts

@@ -3,10 +3,12 @@ export interface StageBudget {
     totalBudgetMs: number;
     elapsedMs: number;
 }
+/** Backwards-compatible exports */
 export declare function startTransformStage(url: string, stage: string, budget?: StageBudget): TransformStageContext | null;
 export declare function endTransformStage(context: TransformStageContext | null, options?: {
     truncated?: boolean;
 }): number;
+/** Backwards-compatible export */
 export declare function extractContent(html: string, url: string, options?: {
     extractArticle?: boolean;
     signal?: AbortSignal;
@@ -21,11 +23,11 @@ export declare function isExtractionSufficient(article: ExtractedArticle | null,
 export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
 export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
 export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;
-export declare function shutdownTransformWorkerPool(): Promise<void>;
 export interface TransformPoolStats {
     queueDepth: number;
     activeWorkers: number;
     capacity: number;
 }
 export declare function getTransformPoolStats(): TransformPoolStats | null;
+export declare function shutdownTransformWorkerPool(): Promise<void>;
 export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;