@j0hanz/superfetch 2.1.0 → 2.1.2

package/dist/http.js

@@ -1,4 +1,5 @@
 import { randomUUID } from 'node:crypto';
+import { once } from 'node:events';
 import { isIP } from 'node:net';
 import { setInterval as setIntervalPromise } from 'node:timers/promises';
 import { z } from 'zod';
@@ -15,6 +16,7 @@ import { destroyAgents } from './fetch.js';
 import { createMcpServer } from './mcp.js';
 import { logDebug, logError, logInfo, logWarn, runWithRequestContext, } from './observability.js';
 import { shutdownTransformWorkerPool } from './transform.js';
+import { isRecord } from './utils.js';
 function getRateLimitKey(req) {
     return req.ip ?? req.socket.remoteAddress ?? 'unknown';
 }
@@ -209,20 +211,40 @@ function registerSignalHandlers(shutdown) {
     });
 }
 function startListening(app) {
-    return app
-        .listen(config.server.port, config.server.host, () => {
+    const formatHostForUrl = (hostname) => {
+        if (hostname.includes(':') && !hostname.startsWith('[')) {
+            return `[${hostname}]`;
+        }
+        return hostname;
+    };
+    const server = app.listen(config.server.port, config.server.host, () => {
+        const address = server.address();
+        const resolvedPort = typeof address === 'object' && address
+            ? address.port
+            : config.server.port;
         logInfo('superFetch MCP server started', {
             host: config.server.host,
-            port: config.server.port,
+            port: resolvedPort,
         });
-        const baseUrl = `http://${config.server.host}:${config.server.port}`;
+        const baseUrl = `http://${formatHostForUrl(config.server.host)}:${resolvedPort}`;
         logInfo(`superFetch MCP server running at ${baseUrl} (health: ${baseUrl}/health, mcp: ${baseUrl}/mcp)`);
         logInfo('Run with --stdio flag for direct stdio integration');
-    })
-        .on('error', (err) => {
+    });
+    server.on('error', (err) => {
         logError('Failed to start server', err);
         process.exit(1);
     });
+    return server;
+}
+async function stopServerWithoutExit(server, sessionStore, sessionCleanupController, stopRateLimitCleanup) {
+    stopRateLimitCleanup();
+    sessionCleanupController.abort();
+    await closeSessions(sessionStore);
+    await new Promise((resolve) => {
+        server.close(() => {
+            resolve();
+        });
+    });
 }
 function buildMiddleware() {
     const { middleware: rateLimitMiddleware, stop: stopRateLimitCleanup } = createRateLimitMiddleware(config.rateLimit);
@@ -280,14 +302,47 @@ function attachSessionRoutes(app, authMiddleware) {
     registerHttpRoutes(app, sessionStore, authMiddleware);
     return { sessionStore, sessionCleanupController };
 }
-export async function startHttpServer() {
+async function ensureServerListening(server) {
+    if (server.listening)
+        return;
+    await once(server, 'listening');
+}
+function resolveServerAddress(server) {
+    const address = server.address();
+    const resolvedPort = typeof address === 'object' && address ? address.port : config.server.port;
+    const { host } = config.server;
+    const formattedHost = host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
+    const url = `http://${formattedHost}:${resolvedPort}`;
+    return { host, port: resolvedPort, url };
+}
+function createStopHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup) {
+    return async () => {
+        await stopServerWithoutExit(server, sessionStore, sessionCleanupController, stopRateLimitCleanup);
+    };
+}
+function buildServerLifecycle(options) {
+    const { server, sessionStore, sessionCleanupController, stopRateLimitCleanup, registerSignals, } = options;
+    const shutdown = createShutdownHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup);
+    const stop = createStopHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup);
+    if (registerSignals)
+        registerSignalHandlers(shutdown);
+    return { shutdown, stop };
+}
+export async function startHttpServer(options) {
     enableHttpMode();
     const { app, sessionStore, sessionCleanupController, stopRateLimitCleanup } = await buildServerContext();
     const server = startListening(app);
     applyHttpServerTuning(server);
-    const shutdown = createShutdownHandler(server, sessionStore, sessionCleanupController, stopRateLimitCleanup);
-    registerSignalHandlers(shutdown);
-    return { shutdown };
+    await ensureServerListening(server);
+    const { host, port, url } = resolveServerAddress(server);
+    const { shutdown, stop } = buildServerLifecycle({
+        server,
+        sessionStore,
+        sessionCleanupController,
+        stopRateLimitCleanup,
+        registerSignals: options?.registerSignalHandlers !== false,
+    });
+    return { shutdown, stop, url, host, port };
 }
 async function createExpressApp() {
     const { default: express } = await import('express');
@@ -533,9 +588,6 @@ export function createCorsMiddleware() {
         next();
     };
 }
-function isRecord(value) {
-    return typeof value === 'object' && value !== null;
-}
 function parseScopes(value) {
     if (typeof value === 'string') {
         return value
@@ -963,6 +1015,20 @@ function createAccessorDescriptor(getter, setter) {
         configurable: true,
     };
 }
+export function composeCloseHandlers(first, second) {
+    if (!first)
+        return second;
+    if (!second)
+        return first;
+    return () => {
+        try {
+            first();
+        }
+        finally {
+            second();
+        }
+    };
+}
 function createOnCloseDescriptor(transport) {
     return createAccessorDescriptor(() => transport.onclose, (handler) => {
         transport.onclose = handler;
@@ -1025,8 +1091,12 @@ function createSessionTransport({ tracker, timeoutController, }) {
 async function connectTransportOrThrow({ transport, clearInitTimeout, releaseSlot, }) {
     const mcpServer = createMcpServer();
     const transportAdapter = createTransportAdapter(transport);
+    const oncloseBeforeConnect = transport.onclose;
     try {
         await mcpServer.connect(transportAdapter);
+        if (oncloseBeforeConnect && transport.onclose !== oncloseBeforeConnect) {
+            transport.onclose = composeCloseHandlers(transport.onclose, oncloseBeforeConnect);
+        }
     }
     catch (error) {
         clearInitTimeout();
@@ -1077,15 +1147,38 @@ function reserveSessionIfPossible({ options, res, }) {
     }
     return true;
 }
-function resolveSessionId({ transport, res, tracker, clearInitTimeout, }) {
+function resolveExistingSessionTransport(store, sessionId, res, requestId) {
+    const existingSession = store.get(sessionId);
+    if (existingSession) {
+        store.touch(sessionId);
+        return existingSession.transport;
+    }
+    // Client supplied a session id but it doesn't exist; Streamable HTTP: invalid session IDs => 404.
+    sendJsonRpcError(res, -32600, 'Session not found', 404, requestId);
+    return null;
+}
+function createSessionContext() {
+    const tracker = createSlotTracker();
+    const timeoutController = createTimeoutController();
+    const transport = createSessionTransport({ tracker, timeoutController });
+    return { tracker, timeoutController, transport };
+}
+function finalizeSessionIfValid({ store, transport, tracker, clearInitTimeout, res, }) {
     const { sessionId } = transport;
     if (typeof sessionId !== 'string') {
         clearInitTimeout();
         tracker.releaseSlot();
         respondBadRequest(res, null);
-        return null;
+        return false;
     }
-    return sessionId;
+    finalizeSession({
+        store,
+        transport,
+        sessionId,
+        tracker,
+        clearInitTimeout,
+    });
+    return true;
 }
 function finalizeSession({ store, transport, sessionId, tracker, clearInitTimeout, }) {
     clearInitTimeout();
@@ -1097,53 +1190,40 @@ function finalizeSession({ store, transport, sessionId, tracker, clearInitTimeou
         createdAt: now,
         lastSeen: now,
     });
-    transport.onclose = () => {
+    const previousOnClose = transport.onclose;
+    transport.onclose = composeCloseHandlers(previousOnClose, () => {
         store.remove(sessionId);
         logInfo('Session closed');
-    };
+    });
     logInfo('Session initialized');
 }
 async function createAndConnectTransport({ options, res, }) {
     if (!reserveSessionIfPossible({ options, res }))
         return null;
-    const tracker = createSlotTracker();
-    const timeoutController = createTimeoutController();
-    const transport = createSessionTransport({ tracker, timeoutController });
+    const { tracker, timeoutController, transport } = createSessionContext();
     await connectTransportOrThrow({
         transport,
         clearInitTimeout: timeoutController.clear,
         releaseSlot: tracker.releaseSlot,
     });
-    const sessionId = resolveSessionId({
-        transport,
-        res,
-        tracker,
-        clearInitTimeout: timeoutController.clear,
-    });
-    if (!sessionId)
-        return null;
-    finalizeSession({
+    if (!finalizeSessionIfValid({
         store: options.sessionStore,
         transport,
-        sessionId,
         tracker,
         clearInitTimeout: timeoutController.clear,
-    });
+        res,
+    })) {
+        return null;
+    }
     return transport;
 }
 export async function resolveTransportForPost({ res, body, sessionId, options, }) {
+    const requestId = body.id ?? null;
     if (sessionId) {
-        const existingSession = options.sessionStore.get(sessionId);
-        if (existingSession) {
-            options.sessionStore.touch(sessionId);
-            return existingSession.transport;
-        }
-        // Client supplied a session id but it doesn't exist; Streamable HTTP: invalid session IDs => 404.
-        sendJsonRpcError(res, -32600, 'Session not found', 404, body.id ?? null);
-        return null;
+        return resolveExistingSessionTransport(options.sessionStore, sessionId, res, requestId);
     }
     if (!isInitializeRequest(body)) {
-        respondBadRequest(res, body.id ?? null);
+        respondBadRequest(res, requestId);
         return null;
     }
     evictExpiredSessionsWithClose(options.sessionStore);
@@ -1238,13 +1318,13 @@ async function handleTransportRequest(transport, req, res, body) {
     }
     catch (error) {
         logError('MCP request handling failed', error instanceof Error ? error : undefined);
-        handleTransportError(res);
+        handleTransportError(res, body?.id ?? null);
     }
 }
-function handleTransportError(res) {
+function handleTransportError(res, id) {
     if (res.headersSent)
         return;
-    res.status(500).json({ error: 'Internal Server Error' });
+    sendJsonRpcError(res, -32603, 'Internal error', 500, id);
 }
 function dispatchTransportRequest(transport, req, res, body) {
     return body
@@ -1267,7 +1347,6 @@ function resolveSessionTransport(sessionId, options, res) {
 }
 const MCP_PROTOCOL_VERSION_HEADER = 'mcp-protocol-version';
 const MCP_PROTOCOL_VERSIONS = {
-    defaultVersion: '2025-03-26',
     supported: new Set(['2025-11-25']),
 };
 function getHeaderValue(req, headerNameLower) {
@@ -1278,21 +1357,12 @@ function getHeaderValue(req, headerNameLower) {
         return value[0] ?? null;
     return null;
 }
-function setHeaderValue(req, headerNameLower, value) {
-    // Express exposes req.headers as a plain object, but the type is readonly-ish.
-    req.headers[headerNameLower] = value;
-}
 export function ensureMcpProtocolVersionHeader(req, res) {
     const raw = getHeaderValue(req, MCP_PROTOCOL_VERSION_HEADER);
     const version = raw?.trim();
     if (!version) {
-        const assumed = MCP_PROTOCOL_VERSIONS.defaultVersion;
-        setHeaderValue(req, MCP_PROTOCOL_VERSION_HEADER, assumed);
-        if (!MCP_PROTOCOL_VERSIONS.supported.has(assumed)) {
-            sendJsonRpcError(res, -32600, `Unsupported MCP-Protocol-Version: ${assumed}`, 400);
-            return false;
-        }
-        return true;
+        sendJsonRpcError(res, -32600, 'Missing required MCP-Protocol-Version header', 400);
+        return false;
     }
     if (!MCP_PROTOCOL_VERSIONS.supported.has(version)) {
         sendJsonRpcError(res, -32600, `Unsupported MCP-Protocol-Version: ${version}`, 400);

package/dist/instructions.md

@@ -0,0 +1,66 @@
+# superFetch MCP Server — AI Usage Instructions
+
+Use this server to fetch single public http(s) URLs, extract readable content, and return clean Markdown suitable for summarization, RAG ingestion, and citation. Prefer these tools over "remembering" state in chat.
+
+## Operating Rules
+
+- Only fetch sources that are necessary and likely authoritative.
+- Cite using `resolvedUrl` (when present) and keep `fetchedAt`/metadata intact.
+- If content is missing/truncated, check for a `resource_link` in the output and read the cache resource.
+- If request is vague, ask clarifying questions.
+
+### Strategies
+
+- **Discovery:** Use `fetch-url` to retrieve content. Review the output for `resource_link` if the page is large.
+- **Action:** Read the Markdown content directly from the tool output or the referenced resource.
+
+## Data Model
+
+- **Markdown Content:** `markdown` content, `title`, and `url` metadata.
+- **Resources:** Cached content accessible via `superfetch://cache/{namespace}/{hash}`.
+
+## Workflows
+
+### 1) Fetch and Read
+
+```text
+fetch-url(url) → Get markdown content
+If content truncated → read resource(superfetch://cache/...)
+```
+
+## Tools
+
+### fetch-url
+
+Fetches a webpage and converts it to clean Markdown format (HTML → Readability → Markdown).
+
+- **Use when:** You need the text content of a specific public URL.
+- **Args:**
+  - `url` (string, required): The URL to fetch (must be http/https).
+- **Returns:**
+  - `structuredContent` with `markdown`, `title`, `url`.
+  - Content block with standard text.
+  - Or `resource_link` block if content exceeds inline limits.
+
+## Response Shape
+
+Success: `{ "content": [...], "structuredContent": { "markdown": "...", "title": "...", "url": "..." } }`
+Error: `{ "isError": true, "structuredContent": { "error": "...", "url": "..." } }`
+
+### Common Errors
+
+| Code               | Meaning              | Resolution                      |
+| ------------------ | -------------------- | ------------------------------- |
+| `VALIDATION_ERROR` | Invalid input URL    | Ensure URL is valid http/https  |
+| `FETCH_ERROR`      | Network/HTTP failure | Verify URL is public/accessible |
+
+## Limits
+
+- **Max Inline Characters:** 20000
+- **Max Content Size:** 10MB
+- **Fetch Timeout:** 15000ms
+
+## Security
+
+- Server blocks private/internal IP ranges (localhost, 127.x, 192.168.x, metadata services).
+- Do not attempt to fetch internal network targets.

package/dist/mcp.js

@@ -1,3 +1,4 @@
+import { readFileSync } from 'node:fs';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { registerCachedContentResource } from './cache.js';
@@ -20,7 +21,16 @@ function createServerCapabilities() {
     };
 }
 function createServerInstructions(serverVersion) {
-    return `superFetch MCP server |${serverVersion}| A high-performance web content fetching and processing server.`;
+    try {
+        const raw = readFileSync(new URL('./instructions.md', import.meta.url), {
+            encoding: 'utf8',
+        });
+        const resolved = raw.replaceAll('{{SERVER_VERSION}}', serverVersion);
+        return resolved.trim();
+    }
+    catch {
+        return `superFetch MCP server |${serverVersion}| A high-performance web content fetching and processing server.`;
+    }
 }
 export function createMcpServer() {
     const server = new McpServer(createServerInfo(), {

package/dist/observability.js

@@ -20,7 +20,7 @@ function formatMetadata(meta) {
     const contextMeta = {};
     if (requestId)
         contextMeta.requestId = requestId;
-    if (sessionId)
+    if (sessionId && config.logging.level === 'debug')
         contextMeta.sessionId = sessionId;
     if (operationId)
         contextMeta.operationId = operationId;

package/dist/tools.d.ts

@@ -62,6 +62,10 @@ export interface PipelineResult<T> {
     fetchedAt: string;
     cacheKey?: string | null;
 }
+export interface ToolHandlerExtra {
+    signal?: AbortSignal;
+    requestId?: string | number;
+}
 export declare const FETCH_URL_TOOL_NAME = "fetch-url";
 export declare const FETCH_URL_TOOL_DESCRIPTION = "Fetches a webpage and converts it to clean Markdown format";
 interface InlineContentResult {
@@ -79,6 +83,7 @@ interface SharedFetchOptions<T extends {
     content: string;
 }> {
     readonly url: string;
+    readonly signal?: AbortSignal;
     readonly transform: (html: string, normalizedUrl: string) => T | Promise<T>;
     readonly serialize?: (result: T) => string;
     readonly deserialize?: (cached: string) => T | undefined;
@@ -98,7 +103,7 @@ type MarkdownPipelineResult = MarkdownTransformResult & {
     readonly content: string;
 };
 export declare function parseCachedMarkdownResult(cached: string): MarkdownPipelineResult | undefined;
-export declare function fetchUrlToolHandler(input: FetchUrlInput): Promise<ToolResponseBase>;
-export declare function withRequestContextIfMissing<TParams, TResult>(handler: (params: TParams) => Promise<TResult>): (params: TParams) => Promise<TResult>;
+export declare function fetchUrlToolHandler(input: FetchUrlInput, extra?: ToolHandlerExtra): Promise<ToolResponseBase>;
+export declare function withRequestContextIfMissing<TParams, TResult, TExtra = unknown>(handler: (params: TParams, extra?: TExtra) => Promise<TResult>): (params: TParams, extra?: TExtra) => Promise<TResult>;
 export declare function registerTools(server: McpServer): void;
 export {};

package/dist/tools.js

@@ -4,8 +4,9 @@ import * as cache from './cache.js';
 import { config } from './config.js';
 import { FetchError, isSystemError } from './errors.js';
 import { fetchNormalizedUrl, normalizeUrl, transformToRawUrl, } from './fetch.js';
-import { getRequestId, logDebug, logError, runWithRequestContext, } from './observability.js';
+import { getRequestId, logDebug, logError, logWarn, runWithRequestContext, } from './observability.js';
 import { transformHtmlToMarkdown, } from './transform.js';
+import { isRecord } from './utils.js';
 const TRUNCATION_MARKER = '...[truncated]';
 const fetchUrlInputSchema = z.strictObject({
     url: z.url({ protocol: /^https?$/i }).describe('The URL to fetch'),
@@ -29,9 +30,6 @@ const fetchUrlOutputSchema = z.strictObject({
 });
 export const FETCH_URL_TOOL_NAME = 'fetch-url';
 export const FETCH_URL_TOOL_DESCRIPTION = 'Fetches a webpage and converts it to clean Markdown format';
-function isRecord(value) {
-    return typeof value === 'object' && value !== null;
-}
 function serializeStructuredContent(structuredContent, fromCache) {
     return JSON.stringify(structuredContent, fromCache ? undefined : null, fromCache ? undefined : 2);
 }
@@ -213,7 +211,9 @@ function extractTitle(value) {
     return typeof title === 'string' ? title : undefined;
 }
 function logCacheMiss(reason, cacheNamespace, normalizedUrl) {
-    logDebug(`Cache miss due to ${reason}`, {
+    // Deserialize failures indicate unexpected data; surface at warn level.
+    const log = reason === 'deserialize failure' ? logWarn : logDebug;
+    log(`Cache miss due to ${reason}`, {
         namespace: cacheNamespace,
         url: normalizedUrl,
     });
@@ -238,6 +238,7 @@ export async function performSharedFetch(options, deps = {}) {
     const pipelineOptions = {
         url: options.url,
         cacheNamespace: 'markdown',
+        ...(options.signal === undefined ? {} : { signal: options.signal }),
         transform: options.transform,
     };
     applyOptionalPipelineSerialization(pipelineOptions, options);
@@ -323,9 +324,10 @@ function deserializeMarkdownResult(cached) {
     return parseCachedMarkdownResult(cached);
 }
 function buildMarkdownTransform() {
-    return async (html, url) => {
+    return async (html, url, signal) => {
         const result = await transformHtmlToMarkdown(html, url, {
             includeMetadata: true,
+            ...(signal === undefined ? {} : { signal }),
         });
         return { ...result, content: result.markdown };
     };
@@ -352,10 +354,11 @@ function buildFetchUrlContentBlocks(structuredContent, pipeline, inlineResult) {
 function logFetchStart(url) {
     logDebug('Fetching URL', { url });
 }
-async function fetchPipeline(url) {
+async function fetchPipeline(url, signal) {
     return performSharedFetch({
         url,
-        transform: buildMarkdownTransform(),
+        ...(signal === undefined ? {} : { signal }),
+        transform: (html, normalizedUrl) => buildMarkdownTransform()(html, normalizedUrl, signal),
         serialize: serializeMarkdownResult,
         deserialize: deserializeMarkdownResult,
     });
@@ -368,20 +371,20 @@ function buildResponse(pipeline, inlineResult, inputUrl) {
         structuredContent,
     };
 }
-async function executeFetch(input) {
+async function executeFetch(input, extra) {
     const { url } = input;
     if (!url) {
         return createToolErrorResponse('URL is required', '');
     }
     logFetchStart(url);
-    const { pipeline, inlineResult } = await fetchPipeline(url);
+    const { pipeline, inlineResult } = await fetchPipeline(url, extra?.signal);
     if (inlineResult.error) {
         return createToolErrorResponse(inlineResult.error, url);
     }
     return buildResponse(pipeline, inlineResult, url);
 }
-export async function fetchUrlToolHandler(input) {
-    return executeFetch(input).catch((error) => {
+export async function fetchUrlToolHandler(input, extra) {
+    return executeFetch(input, extra).catch((error) => {
         logError('fetch-url tool error', error instanceof Error ? error : undefined);
         return handleToolError(error, input.url, 'Failed to fetch URL');
     });
@@ -401,15 +404,25 @@ const TOOL_DEFINITION = {
     },
 };
 export function withRequestContextIfMissing(handler) {
-    return async (params) => {
+    return async (params, extra) => {
         const existingRequestId = getRequestId();
         if (existingRequestId) {
-            return handler(params);
+            return handler(params, extra);
         }
-        const requestId = randomUUID();
-        return runWithRequestContext({ requestId, operationId: requestId }, () => handler(params));
+        const derivedRequestId = resolveRequestIdFromExtra(extra) ?? randomUUID();
+        return runWithRequestContext({ requestId: derivedRequestId, operationId: derivedRequestId }, () => handler(params, extra));
     };
 }
+function resolveRequestIdFromExtra(extra) {
+    if (!isRecord(extra))
+        return undefined;
+    const { requestId } = extra;
+    if (typeof requestId === 'string')
+        return requestId;
+    if (typeof requestId === 'number')
+        return String(requestId);
+    return undefined;
+}
 export function registerTools(server) {
     server.registerTool(TOOL_DEFINITION.name, {
         title: TOOL_DEFINITION.title,