@j0hanz/fetch-url-mcp 0.0.1

package/dist/resources.js

@@ -0,0 +1,216 @@
+import { ResourceTemplate, } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { ErrorCode, McpError, } from '@modelcontextprotocol/sdk/types.js';
+import { get as getCacheEntry, keys as listCacheKeys, parseCachedPayload, parseCacheKey, resolveCachedPayloadContent, } from './cache.js';
+const CACHE_RESOURCE_TEMPLATE_URI = 'internal://cache/{namespace}/{hash}';
+const CACHE_RESOURCE_PREFIX = 'internal://cache/';
+const CACHE_NAMESPACE_PATTERN = /^[a-z0-9_-]{1,64}$/i;
+const CACHE_HASH_PATTERN = /^[a-f0-9.]{8,64}$/i;
+const RESOURCE_NOT_FOUND_ERROR_CODE = -32002;
+const MAX_COMPLETION_VALUES = 100;
+function isValidCacheResourceParts(parts) {
+    return (CACHE_NAMESPACE_PATTERN.test(parts.namespace) &&
+        CACHE_HASH_PATTERN.test(parts.hash));
+}
+function decodeSegment(value) {
+    try {
+        return decodeURIComponent(value);
+    }
+    catch {
+        return value;
+    }
+}
+function firstVariableValue(value) {
+    if (typeof value === 'string') {
+        const trimmed = value.trim();
+        return trimmed.length > 0 ? trimmed : undefined;
+    }
+    if (Array.isArray(value)) {
+        const first = value[0];
+        if (typeof first !== 'string')
+            return undefined;
+        const trimmed = first.trim();
+        return trimmed.length > 0 ? trimmed : undefined;
+    }
+    return undefined;
+}
+function parseCacheResourceFromVariables(variables) {
+    const namespace = firstVariableValue(variables.namespace);
+    const hash = firstVariableValue(variables.hash);
+    if (!namespace || !hash)
+        return null;
+    const decoded = {
+        namespace: decodeSegment(namespace),
+        hash: decodeSegment(hash),
+    };
+    return isValidCacheResourceParts(decoded) ? decoded : null;
+}
+function parseCacheResourceFromUri(uri) {
+    if (!uri.href.startsWith(CACHE_RESOURCE_PREFIX))
+        return null;
+    const rawPath = uri.pathname.startsWith('/')
+        ? uri.pathname.slice(1)
+        : uri.pathname;
+    const segments = rawPath.split('/');
+    if (segments.length !== 2)
+        return null;
+    const namespace = segments[0];
+    const hash = segments[1];
+    if (!namespace || !hash)
+        return null;
+    const decoded = {
+        namespace: decodeSegment(namespace),
+        hash: decodeSegment(hash),
+    };
+    return isValidCacheResourceParts(decoded) ? decoded : null;
+}
+function toCacheResourceUri(parts) {
+    const namespace = encodeURIComponent(parts.namespace);
+    const hash = encodeURIComponent(parts.hash);
+    return `${CACHE_RESOURCE_PREFIX}${namespace}/${hash}`;
+}
+function listCacheNamespaces() {
+    const namespaces = new Set();
+    for (const key of listCacheKeys()) {
+        const parsed = parseCacheKey(key);
+        if (!parsed)
+            continue;
+        namespaces.add(parsed.namespace);
+    }
+    return [...namespaces].sort((left, right) => left.localeCompare(right));
+}
+function completeCacheNamespaces(value) {
+    const normalized = value.trim().toLowerCase();
+    return listCacheNamespaces()
+        .filter((namespace) => namespace.toLowerCase().startsWith(normalized))
+        .slice(0, MAX_COMPLETION_VALUES);
+}
+function completeCacheHashes(value, context) {
+    const normalized = value.trim().toLowerCase();
+    const namespace = context?.arguments?.namespace?.trim();
+    const hashes = new Set();
+    for (const key of listCacheKeys()) {
+        const parsed = parseCacheKey(key);
+        if (!parsed)
+            continue;
+        if (namespace && parsed.namespace !== namespace)
+            continue;
+        if (!parsed.urlHash.toLowerCase().startsWith(normalized))
+            continue;
+        hashes.add(parsed.urlHash);
+    }
+    return [...hashes]
+        .sort((left, right) => left.localeCompare(right))
+        .slice(0, MAX_COMPLETION_VALUES);
+}
+function listCacheResources() {
+    const resources = listCacheKeys()
+        .map((key) => parseCacheKey(key))
+        .filter((parts) => Boolean(parts))
+        .slice(0, MAX_COMPLETION_VALUES)
+        .map((parts) => {
+        const cacheParts = {
+            namespace: parts.namespace,
+            hash: parts.urlHash,
+        };
+        return {
+            uri: toCacheResourceUri(cacheParts),
+            name: `${parts.namespace}:${parts.urlHash}`,
+            title: 'Cached Markdown',
+            description: 'Cached markdown output generated by fetch-url',
+            mimeType: 'text/markdown',
+            annotations: {
+                audience: ['assistant'],
+                priority: 0.6,
+            },
+        };
+    });
+    return { resources };
+}
+function resolveCacheResourceParts(uri, variables) {
+    const fromVariables = parseCacheResourceFromVariables(variables);
+    if (fromVariables)
+        return fromVariables;
+    const fromUri = parseCacheResourceFromUri(uri);
+    if (fromUri)
+        return fromUri;
+    throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource URI or template arguments');
+}
+function readCacheResource(uri, variables) {
+    const parts = resolveCacheResourceParts(uri, variables);
+    const cacheKey = `${parts.namespace}:${parts.hash}`;
+    const entry = getCacheEntry(cacheKey);
+    if (!entry) {
+        throw new McpError(RESOURCE_NOT_FOUND_ERROR_CODE, 'Resource not found', {
+            uri: uri.href,
+        });
+    }
+    const payload = parseCachedPayload(entry.content);
+    const markdown = payload ? resolveCachedPayloadContent(payload) : null;
+    const text = markdown ?? entry.content;
+    return {
+        contents: [
+            {
+                uri: uri.href,
+                mimeType: 'text/markdown',
+                text,
+            },
+        ],
+    };
+}
+export function registerInstructionResource(server, instructions, iconInfo) {
+    server.registerResource('fetch-url-mcp-instructions', 'internal://instructions', {
+        title: 'Server Instructions',
+        description: 'Guidance for using the Fetch URL MCP server.',
+        mimeType: 'text/markdown',
+        annotations: {
+            audience: ['assistant'],
+            priority: 0.9,
+        },
+        ...(iconInfo
+            ? {
+                icons: [
+                    {
+                        src: iconInfo.src,
+                        mimeType: iconInfo.mimeType,
+                    },
+                ],
+            }
+            : {}),
+    }, (uri) => ({
+        contents: [
+            {
+                uri: uri.href,
+                mimeType: 'text/markdown',
+                text: instructions,
+            },
+        ],
+    }));
+}
+export function registerCacheResourceTemplate(server, iconInfo) {
+    const template = new ResourceTemplate(CACHE_RESOURCE_TEMPLATE_URI, {
+        list: () => listCacheResources(),
+        complete: {
+            namespace: (value) => completeCacheNamespaces(value),
+            hash: (value, context) => completeCacheHashes(value, context),
+        },
+    });
+    server.registerResource('fetch-url-mcp-cache-entry', template, {
+        title: 'Cached Fetch Output',
+        description: 'Read cached markdown generated by previous fetch-url calls.',
+        mimeType: 'text/markdown',
+        annotations: {
+            audience: ['assistant'],
+            priority: 0.6,
+        },
+        ...(iconInfo
+            ? {
+                icons: [
+                    {
+                        src: iconInfo.src,
+                        mimeType: iconInfo.mimeType,
+                    },
+                ],
+            }
+            : {}),
+    }, (uri, variables) => readCacheResource(uri, variables));
+}

package/dist/server-tuning.d.ts

@@ -0,0 +1,13 @@
+export interface HttpServerTuningTarget {
+    headersTimeout?: number;
+    requestTimeout?: number;
+    keepAliveTimeout?: number;
+    keepAliveTimeoutBuffer?: number;
+    maxHeadersCount?: number | null;
+    maxConnections?: number;
+    on?: (event: string, listener: (...args: unknown[]) => void) => void;
+    closeIdleConnections?: () => void;
+    closeAllConnections?: () => void;
+}
+export declare function applyHttpServerTuning(server: HttpServerTuningTarget): void;
+export declare function drainConnectionsOnShutdown(server: HttpServerTuningTarget): void;

package/dist/server-tuning.js

@@ -0,0 +1,47 @@
+import { config } from './config.js';
+import { logDebug, logWarn } from './observability.js';
+const DROP_LOG_INTERVAL_MS = 10_000;
+export function applyHttpServerTuning(server) {
+    const { headersTimeoutMs, requestTimeoutMs, keepAliveTimeoutMs, keepAliveTimeoutBufferMs, maxHeadersCount, maxConnections, } = config.server.http;
+    if (headersTimeoutMs !== undefined) {
+        server.headersTimeout = headersTimeoutMs;
+    }
+    if (requestTimeoutMs !== undefined) {
+        server.requestTimeout = requestTimeoutMs;
+    }
+    if (keepAliveTimeoutMs !== undefined) {
+        server.keepAliveTimeout = keepAliveTimeoutMs;
+    }
+    if (keepAliveTimeoutBufferMs !== undefined) {
+        server.keepAliveTimeoutBuffer = keepAliveTimeoutBufferMs;
+    }
+    if (maxHeadersCount !== undefined) {
+        server.maxHeadersCount = maxHeadersCount;
+    }
+    if (typeof maxConnections === 'number' && maxConnections > 0) {
+        server.maxConnections = maxConnections;
+        if (typeof server.on === 'function') {
+            let lastLoggedAt = 0;
+            let droppedSinceLastLog = 0;
+            server.on('drop', (data) => {
+                droppedSinceLastLog += 1;
+                const now = Date.now();
+                if (now - lastLoggedAt < DROP_LOG_INTERVAL_MS)
+                    return;
+                logWarn('Incoming connection dropped (maxConnections reached)', {
+                    maxConnections,
+                    dropped: droppedSinceLastLog,
+                    data,
+                });
+                lastLoggedAt = now;
+                droppedSinceLastLog = 0;
+            });
+        }
+    }
+}
+export function drainConnectionsOnShutdown(server) {
+    if (typeof server.closeIdleConnections === 'function') {
+        server.closeIdleConnections();
+        logDebug('Closed idle HTTP connections during shutdown');
+    }
+}

package/dist/server.d.ts

@@ -0,0 +1,4 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+export declare function createMcpServer(): Promise<McpServer>;
+export declare function createMcpServerForHttpSession(): Promise<McpServer>;
+export declare function startStdioServer(): Promise<void>;

package/dist/server.js

@@ -0,0 +1,174 @@
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+import { InMemoryTaskMessageQueue, InMemoryTaskStore, } from '@modelcontextprotocol/sdk/experimental/tasks/stores/in-memory.js';
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { config } from './config.js';
+import { getErrorMessage } from './errors.js';
+import { abortAllTaskExecutions, registerTaskHandlers } from './mcp.js';
+import { logError, logInfo, setMcpServer } from './observability.js';
+import { registerGetHelpPrompt } from './prompts.js';
+import { registerCacheResourceTemplate, registerInstructionResource, } from './resources.js';
+import { registerTools } from './tools.js';
+import { shutdownTransformWorkerPool } from './transform.js';
+async function getLocalIconInfo() {
+    const name = 'logo.svg';
+    const mime = 'image/svg+xml';
+    try {
+        const iconPath = new URL(`../assets/${name}`, import.meta.url);
+        const buffer = await fs.readFile(iconPath);
+        return {
+            src: `data:${mime};base64,${buffer.toString('base64')}`,
+            mimeType: mime,
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+const currentDir = path.dirname(fileURLToPath(import.meta.url));
+let serverInstructions = `
+Fetch URL MCP Instructions
+(Detailed instructions failed to load - check logs)
+`;
+try {
+    serverInstructions = await fs.readFile(path.join(currentDir, 'instructions.md'), 'utf-8');
+}
+catch (error) {
+    console.error('[WARNING] Failed to load instructions.md:', getErrorMessage(error));
+}
+function createServerCapabilities() {
+    return {
+        logging: {},
+        resources: {},
+        tools: {},
+        prompts: {},
+        completions: {},
+        tasks: {
+            list: {},
+            cancel: {},
+            requests: {
+                tools: {
+                    call: {},
+                },
+            },
+        },
+    };
+}
+function createServerInfo(icons) {
+    return {
+        name: config.server.name,
+        title: 'Fetch URL',
+        description: 'Fetch web pages and convert them into clean, AI-readable Markdown.',
+        version: config.server.version,
+        websiteUrl: 'https://github.com/j0hanz/fetch-url-mcp',
+        ...(icons ? { icons } : {}),
+    };
+}
+/* -------------------------------------------------------------------------------------------------
+ * Server lifecycle
+ * ------------------------------------------------------------------------------------------------- */
+export async function createMcpServer() {
+    return createMcpServerWithOptions({ registerObservabilityServer: true });
+}
+async function createMcpServerWithOptions(options) {
+    const localIcon = await getLocalIconInfo();
+    const taskStore = new InMemoryTaskStore();
+    const taskMessageQueue = new InMemoryTaskMessageQueue();
+    const serverConfig = {
+        capabilities: createServerCapabilities(),
+        taskStore,
+        taskMessageQueue,
+    };
+    if (serverInstructions) {
+        serverConfig.instructions = serverInstructions;
+    }
+    const serverInfo = createServerInfo(localIcon
+        ? [
+            {
+                src: localIcon.src,
+                mimeType: localIcon.mimeType,
+            },
+        ]
+        : undefined);
+    const server = new McpServer(serverInfo, serverConfig);
+    if (options?.registerObservabilityServer ?? true) {
+        setMcpServer(server);
+    }
+    registerTools(server);
+    registerGetHelpPrompt(server, serverInstructions, localIcon);
+    registerInstructionResource(server, serverInstructions, localIcon);
+    registerCacheResourceTemplate(server, localIcon);
+    registerTaskHandlers(server);
+    return server;
+}
+export async function createMcpServerForHttpSession() {
+    return createMcpServerWithOptions({ registerObservabilityServer: false });
+}
+function attachServerErrorHandler(server) {
+    server.server.onerror = (error) => {
+        logError('[MCP Error]', error instanceof Error ? error : { error });
+    };
+}
+async function shutdownServer(server, signal) {
+    process.stderr.write(`\n${signal} received, shutting down Fetch URL MCP server...\n`);
+    // Ensure any in-flight tool executions are aborted promptly.
+    abortAllTaskExecutions();
+    await shutdownTransformWorkerPool();
+    await server.close();
+}
+function createShutdownHandler(server) {
+    let shuttingDown = false;
+    let initialSignal = null;
+    return (signal) => {
+        if (shuttingDown) {
+            logInfo('Shutdown already in progress; ignoring signal', {
+                signal,
+                initialSignal,
+            });
+            return;
+        }
+        shuttingDown = true;
+        initialSignal = signal;
+        Promise.resolve()
+            .then(() => shutdownServer(server, signal))
+            .catch((err) => {
+            const error = err instanceof Error ? err : new Error(String(err));
+            logError('Error during shutdown', error);
+            process.exitCode = 1;
+        })
+            .finally(() => {
+            if (process.exitCode === undefined)
+                process.exitCode = 0;
+        });
+    };
+}
+function registerSignalHandlers(handler) {
+    process.once('SIGINT', () => {
+        handler('SIGINT');
+    });
+    process.once('SIGTERM', () => {
+        handler('SIGTERM');
+    });
+}
+async function connectStdioServer(server, transport) {
+    try {
+        await server.connect(transport);
+        logInfo('Fetch URL MCP server running on stdio');
+    }
+    catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        throw new Error(`Failed to start stdio server: ${err.message}`, {
+            cause: err,
+        });
+    }
+}
+export async function startStdioServer() {
+    const server = await createMcpServer();
+    const transport = new StdioServerTransport();
+    attachServerErrorHandler(server);
+    registerSignalHandlers(createShutdownHandler(server));
+    await connectStdioServer(server, transport);
+}

package/dist/session.d.ts

@@ -0,0 +1,39 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+export interface SessionEntry {
+    readonly server: McpServer;
+    readonly transport: StreamableHTTPServerTransport;
+    createdAt: number;
+    lastSeen: number;
+    protocolInitialized: boolean;
+    authFingerprint: string;
+}
+export interface SessionStore {
+    get: (sessionId: string) => SessionEntry | undefined;
+    touch: (sessionId: string) => void;
+    set: (sessionId: string, entry: SessionEntry) => void;
+    remove: (sessionId: string) => SessionEntry | undefined;
+    size: () => number;
+    inFlight: () => number;
+    incrementInFlight: () => void;
+    decrementInFlight: () => void;
+    clear: () => SessionEntry[];
+    evictExpired: () => SessionEntry[];
+    evictOldest: () => SessionEntry | undefined;
+}
+export interface SlotTracker {
+    readonly releaseSlot: () => void;
+    readonly markInitialized: () => void;
+    readonly isInitialized: () => boolean;
+}
+export type CloseHandler = (() => void) | undefined;
+export declare function composeCloseHandlers(first: CloseHandler, second: CloseHandler): CloseHandler;
+export declare function startSessionCleanupLoop(store: SessionStore, sessionTtlMs: number): AbortController;
+export declare function createSessionStore(sessionTtlMs: number): SessionStore;
+export declare function createSlotTracker(store: SessionStore): SlotTracker;
+export declare function reserveSessionSlot(store: SessionStore, maxSessions: number): boolean;
+export declare function ensureSessionCapacity({ store, maxSessions, evictOldest, }: {
+    store: SessionStore;
+    maxSessions: number;
+    evictOldest: (store: SessionStore) => boolean;
+}): boolean;