@j0hanz/fetch-url-mcp 1.3.1 → 1.5.0

package/dist/{tools.js → tools/fetch-url.js}

@@ -1,114 +1,17 @@
 import { randomUUID } from 'node:crypto';
+import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
 import { z } from 'zod';
-import * as cache from './cache.js';
-import { config } from './config.js';
-import { generateSafeFilename } from './download.js';
-import { getRequestId, logDebug, logError, logWarn, runWithRequestContext, } from './observability.js';
-import { handleToolError } from './tool-errors.js';
-import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord, readString, serializeMarkdownResult, TRUNCATION_MARKER, withSignal, } from './tool-pipeline.js';
-import { createProgressReporter, } from './tool-progress.js';
-import { isObject } from './type-guards.js';
-// Re-export public API so existing consumers keep working.
-export { createToolErrorResponse, handleToolError } from './tool-errors.js';
-export { executeFetchPipeline, parseCachedMarkdownResult, performSharedFetch, } from './tool-pipeline.js';
-export { createProgressReporter, } from './tool-progress.js';
-export const fetchUrlInputSchema = z.strictObject({
-    url: z
-        .url({ protocol: /^https?$/i })
-        .min(1)
-        .max(config.constants.maxUrlLength)
-        .describe(`Target URL. Max ${config.constants.maxUrlLength} chars.`),
-    skipNoiseRemoval: z
-        .boolean()
-        .optional()
-        .describe('Preserve navigation/footers (disable noise filtering).'),
-    forceRefresh: z
-        .boolean()
-        .optional()
-        .describe('Bypass cache and fetch fresh content.'),
-    maxInlineChars: z
-        .number()
-        .int()
-        .min(0)
-        .max(config.constants.maxHtmlSize)
-        .optional()
-        .describe(`Inline markdown limit (0-${config.constants.maxHtmlSize}, 0=unlimited). Lower of this or global limit applies.`),
-});
-export const fetchUrlOutputSchema = z.strictObject({
-    url: z
-        .string()
-        .min(1)
-        .max(config.constants.maxUrlLength)
-        .describe('Fetched URL.'),
-    inputUrl: z
-        .string()
-        .max(config.constants.maxUrlLength)
-        .optional()
-        .describe('Original requested URL.'),
-    resolvedUrl: z
-        .string()
-        .max(config.constants.maxUrlLength)
-        .optional()
-        .describe('Final URL after raw-content transformations.'),
-    finalUrl: z
-        .string()
-        .max(config.constants.maxUrlLength)
-        .optional()
-        .describe('Final URL after HTTP redirects.'),
-    cacheResourceUri: z
-        .string()
-        .max(config.constants.maxUrlLength)
-        .optional()
-        .describe('URI for resources/read to get full markdown.'),
-    title: z.string().max(512).optional().describe('Page title.'),
-    metadata: z
-        .strictObject({
-        title: z.string().max(512).optional().describe('Detected page title.'),
-        description: z
-            .string()
-            .max(2048)
-            .optional()
-            .describe('Detected page description.'),
-        author: z.string().max(512).optional().describe('Detected page author.'),
-        image: z
-            .string()
-            .max(config.constants.maxUrlLength)
-            .optional()
-            .describe('Detected page preview image URL.'),
-        favicon: z
-            .string()
-            .max(config.constants.maxUrlLength)
-            .optional()
-            .describe('Detected page favicon URL.'),
-        publishedAt: z
-            .string()
-            .max(64)
-            .optional()
-            .describe('Detected publication date.'),
-        modifiedAt: z
-            .string()
-            .max(64)
-            .optional()
-            .describe('Detected last modified date.'),
-    })
-        .optional()
-        .describe('Extracted page metadata.'),
-    markdown: (config.constants.maxInlineContentChars > 0
-        ? z.string().max(config.constants.maxInlineContentChars)
-        : z.string())
-        .optional()
-        .describe('Extracted Markdown. May be truncated (check truncated field).'),
-    fromCache: z.boolean().optional().describe('True if served from cache.'),
-    fetchedAt: z.string().max(64).optional().describe('ISO timestamp of fetch.'),
-    contentSize: z
-        .number()
-        .int()
-        .min(0)
-        .max(config.constants.maxHtmlSize * 4)
-        .optional()
-        .describe('Full markdown size before truncation.'),
-    truncated: z.boolean().optional().describe('True if markdown was truncated.'),
-});
+import * as cache from '../lib/core.js';
+import { config } from '../lib/core.js';
+import { getRequestId, logDebug, logError, logWarn, runWithRequestContext, } from '../lib/core.js';
+import { generateSafeFilename } from '../lib/http.js';
+import { handleToolError } from '../lib/mcp-tools.js';
+import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord, readString, serializeMarkdownResult, TRUNCATION_MARKER, withSignal, } from '../lib/mcp-tools.js';
+import { createProgressReporter, } from '../lib/mcp-tools.js';
+import { isAbortError, isObject, toError } from '../lib/utils.js';
+import { fetchUrlInputSchema } from '../schemas/inputs.js';
+import { fetchUrlOutputSchema } from '../schemas/outputs.js';
+import { registerTaskCapableTool, unregisterTaskCapableTool, } from '../tasks/tool-registry.js';
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
 const FETCH_URL_TOOL_DESCRIPTION = `
 <role>Web Content Extractor</role>
@@ -125,6 +28,7 @@ const TOOL_ICON = {
     src: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJjdXJyZW50Q29sb3IiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj48cGF0aCBkPSJNMjEgMTV2NGEyIDIgMCAwIDEtMiAySDVhMiAyIDAgMCAxLTItMnYtNCIvPjxwb2x5bGluZSBwb2ludHM9IjcgMTAgMTIgMTUgMTcgMTAiLz48bGluZSB4MT0iMTIiIHkxPSIxNSIgeDI9IjEyIiB5Mj0iMyIvPjwvc3ZnPg==',
     mimeType: 'image/svg+xml',
 };
+const JSON_SCHEMA_DRAFT_2020_12_URI = 'https://json-schema.org/draft/2020-12/schema';
 /* -------------------------------------------------------------------------------------------------
  * Tool response builders
  * ------------------------------------------------------------------------------------------------- */
@@ -138,7 +42,7 @@ function buildEmbeddedResource(content, url, title) {
     if (!content)
         return null;
     const filename = generateSafeFilename(url, title, undefined, '.md');
-    const uri = new URL(filename, 'file:///').href;
+    const uri = `internal://inline/${encodeURIComponent(filename)}`;
     const resource = {
         uri,
         mimeType: 'text/markdown',
@@ -178,11 +82,11 @@ function appendIfPresent(items, value) {
 /* -------------------------------------------------------------------------------------------------
  * Tool abort signal
  * ------------------------------------------------------------------------------------------------- */
+const HARD_TOOL_TIMEOUT_MS = 300_000;
 function buildToolAbortSignal(extraSignal) {
     const { timeoutMs } = config.tools;
-    if (timeoutMs <= 0)
-        return extraSignal;
-    const timeoutSignal = AbortSignal.timeout(timeoutMs);
+    const effectiveTimeout = timeoutMs > 0 ? timeoutMs : HARD_TOOL_TIMEOUT_MS;
+    const timeoutSignal = AbortSignal.timeout(effectiveTimeout);
     if (!extraSignal)
         return timeoutSignal;
     return AbortSignal.any([extraSignal, timeoutSignal]);
@@ -195,19 +99,21 @@ function truncateStr(value, max) {
         return value;
     return value.slice(0, max);
 }
+const METADATA_FIELD_LIMITS = {
+    title: 512,
+    description: 2048,
+    author: 512,
+};
 function truncateMetadata(metadata) {
-    return {
-        ...metadata,
-        ...(metadata.title !== undefined
-            ? { title: truncateStr(metadata.title, 512) }
-            : {}),
-        ...(metadata.description !== undefined
-            ? { description: truncateStr(metadata.description, 2048) }
-            : {}),
-        ...(metadata.author !== undefined
-            ? { author: truncateStr(metadata.author, 512) }
-            : {}),
-    };
+    const result = { ...metadata };
+    for (const [field, limit] of Object.entries(METADATA_FIELD_LIMITS)) {
+        const key = field;
+        const value = result[key];
+        if (typeof value === 'string' && value.length > limit) {
+            result[key] = value.slice(0, limit);
+        }
+    }
+    return result;
 }
 function buildStructuredContent(pipeline, inlineResult, inputUrl) {
     const cacheResourceUri = resolveCacheResourceUri(pipeline.cacheKey);
@@ -280,7 +186,7 @@ function buildResponse(pipeline, inlineResult, inputUrl) {
 /* -------------------------------------------------------------------------------------------------
  * fetch-url tool implementation
  * ------------------------------------------------------------------------------------------------- */
-export function getUrlContext(urlStr) {
+function getUrlContext(urlStr) {
     try {
         const u = new URL(urlStr);
         const host = u.hostname.replace(/^www\./, '');
@@ -321,11 +227,7 @@ export function getUrlContext(urlStr) {
     }
 }
 async function fetchPipeline(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
-    const reportProgress = (step, message) => {
-        if (!progress)
-            return;
-        progress.report(step, message);
-    };
+    const contextStr = getUrlContext(url);
     return performSharedFetch({
         url,
         ...withSignal(signal),
@@ -333,48 +235,67 @@ async function fetchPipeline(url, signal, progress, skipNoiseRemoval, forceRefre
         ...(forceRefresh ? { forceRefresh: true } : {}),
         ...(maxInlineChars !== undefined ? { maxInlineChars } : {}),
         transform: async ({ buffer, encoding, truncated }, normalizedUrl) => {
-            const contextStr = getUrlContext(url);
-            reportProgress(2, `fetch-url: ${contextStr} [converting to Markdown]`);
+            reportFetchProgress(progress, 2, contextStr, 'converting to Markdown');
             return markdownTransform({ buffer, encoding, ...(truncated ? { truncated } : {}) }, normalizedUrl, signal, skipNoiseRemoval);
         },
         serialize: serializeMarkdownResult,
         deserialize: parseCachedMarkdownResult,
     });
 }
+function buildFetchProgressMessage(context, state) {
+    if (state === 'completed' || state === 'cancelled' || state === 'failed') {
+        return `fetch-url: ${context} • ${state}`;
+    }
+    return `fetch-url: ${context} [${state}]`;
+}
+function reportFetchProgress(progress, step, context, state) {
+    if (!progress)
+        return;
+    progress.report(step, buildFetchProgressMessage(context, state));
+}
 async function executeFetch(input, extra) {
     const { url } = input;
     const signal = buildToolAbortSignal(extra?.signal);
     const progress = createProgressReporter(extra);
     const contextStr = getUrlContext(url);
-    progress.report(0, `fetch-url: ${contextStr} [starting]`);
+    reportFetchProgress(progress, 0, contextStr, 'starting');
     logDebug('Fetching URL', { url });
     try {
-        progress.report(1, `fetch-url: ${contextStr} [fetching HTML]`);
+        reportFetchProgress(progress, 1, contextStr, 'fetching HTML');
         const { pipeline, inlineResult } = await fetchPipeline(url, signal, progress, input.skipNoiseRemoval, input.forceRefresh, input.maxInlineChars);
         if (pipeline.fromCache) {
-            progress.report(3, `fetch-url: ${contextStr} [loaded from cache]`);
+            reportFetchProgress(progress, 3, contextStr, 'loaded from cache');
         }
-        progress.report(4, `fetch-url: ${contextStr} • completed`);
+        reportFetchProgress(progress, 4, contextStr, 'completed');
         return buildResponse(pipeline, inlineResult, url);
     }
     catch (error) {
-        const isAbort = error instanceof Error && error.name === 'AbortError';
-        progress.report(4, `fetch-url: ${contextStr} • ${isAbort ? 'cancelled' : 'failed'}`);
+        const isAbort = isAbortError(error);
+        reportFetchProgress(progress, 4, contextStr, isAbort ? 'cancelled' : 'failed');
         throw error;
     }
 }
 export async function fetchUrlToolHandler(input, extra) {
     return executeFetch(input, extra).catch((error) => {
-        logError('fetch-url tool error', error instanceof Error ? error : undefined);
+        logError('fetch-url tool error', toError(error));
         return handleToolError(error, input.url, 'Failed to fetch URL');
     });
 }
+function withJsonSchema202012(schema) {
+    if (typeof schema['$schema'] === 'string')
+        return schema;
+    return {
+        $schema: JSON_SCHEMA_DRAFT_2020_12_URI,
+        ...schema,
+    };
+}
 const TOOL_DEFINITION = {
     name: FETCH_URL_TOOL_NAME,
     title: 'Fetch URL',
     description: FETCH_URL_TOOL_DESCRIPTION,
     inputSchema: fetchUrlInputSchema,
-    outputSchema: z.toJSONSchema(fetchUrlOutputSchema),
+    // Explicitly mark JSON Schema dialect for MCP clients and static reviews.
+    outputSchema: withJsonSchema202012(z.toJSONSchema(fetchUrlOutputSchema)),
     handler: fetchUrlToolHandler,
     execution: {
         taskSupport: 'optional',
@@ -386,6 +307,18 @@ const TOOL_DEFINITION = {
         openWorldHint: true,
     },
 };
+function applyRegisteredToolExecutionMetadata(registeredTool, execution) {
+    // SDK workaround: RegisteredTool does not expose `execution` in its public type.
+    // Keep the mutation localized to one helper so future SDK upgrades touch one place.
+    registeredTool.execution = execution;
+}
+/**
+ * Stdio-path guard: ensures a request context (requestId, sessionId) is set
+ * in AsyncLocalStorage before invoking the handler. On the HTTP path the SDK
+ * populates `extra.requestId`/`extra.requestInfo`, so this is a no-op there.
+ * On the stdio path there is no SDK-provided context, so we derive one from
+ * the extra fields or generate a fresh UUID.
+ */
 export function withRequestContextIfMissing(handler) {
     return async (params, extra) => {
         const existingRequestId = getRequestId();
@@ -422,8 +355,25 @@ function resolveSessionIdFromExtra(extra) {
     return typeof headerValue === 'string' ? headerValue : undefined;
 }
 export function registerTools(server) {
-    if (!config.tools.enabled.includes(FETCH_URL_TOOL_NAME))
+    if (!config.tools.enabled.includes(FETCH_URL_TOOL_NAME)) {
+        unregisterTaskCapableTool(FETCH_URL_TOOL_NAME);
         return;
+    }
+    registerTaskCapableTool({
+        name: FETCH_URL_TOOL_NAME,
+        parseArguments: (args) => {
+            const parsed = fetchUrlInputSchema.safeParse(args);
+            if (!parsed.success) {
+                const flat = z.flattenError(parsed.error);
+                const details = Object.entries(flat.fieldErrors)
+                    .map(([k, v]) => `${k}: ${v.join(', ')}`)
+                    .join('; ') || flat.formErrors.join('; ');
+                throw new McpError(ErrorCode.InvalidParams, `Invalid arguments for ${FETCH_URL_TOOL_NAME}: ${details}`);
+            }
+            return parsed.data;
+        },
+        execute: fetchUrlToolHandler,
+    });
     const registeredTool = server.registerTool(TOOL_DEFINITION.name, {
         title: TOOL_DEFINITION.title,
         description: TOOL_DEFINITION.description,
@@ -433,9 +383,7 @@ export function registerTools(server) {
         execution: TOOL_DEFINITION.execution,
         icons: [TOOL_ICON],
     }, withRequestContextIfMissing(TOOL_DEFINITION.handler));
-    // SDK workaround: RegisteredTool does not expose `execution` in its public type, so we
-    // assign it directly post-registration to enable task-augmented tool calls (taskSupport).
-    // TODO: Remove when @modelcontextprotocol/sdk exposes `execution` in RegisteredTool type.
-    registeredTool.execution = TOOL_DEFINITION.execution;
+    // SDK typing gap workaround: preserve runtime `execution` metadata until the
+    // registered tool type includes this field.
+    applyRegisteredToolExecutionMetadata(registeredTool, TOOL_DEFINITION.execution);
 }
-//# sourceMappingURL=tools.js.map

package/dist/tools/index.d.ts

@@ -0,0 +1,2 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+export declare function registerAllTools(server: McpServer): void;

package/dist/tools/index.js

@@ -0,0 +1,4 @@
+import { registerTools as registerFetchUrlTool } from './fetch-url.js';
+export function registerAllTools(server) {
+    registerFetchUrlTool(server);
+}

package/dist/transform/html-translators.d.ts

@@ -0,0 +1 @@
+export declare function translateHtmlFragmentToMarkdown(html: string): string;