@j0hanz/superfetch 2.0.0 → 2.1.0

package/dist/index.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { parseArgs } from 'node:util';
-import { logError } from './services/logger.js';
-import { startHttpServer } from './http/server.js';
-import { startStdioServer } from './server.js';
+import { startHttpServer } from './http.js';
+import { startStdioServer } from './mcp.js';
+import { logError } from './observability.js';
 const { values } = parseArgs({
     options: {
         stdio: { type: 'boolean', default: false },

package/dist/mcp.d.ts

@@ -0,0 +1,3 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+export declare function createMcpServer(): McpServer;
+export declare function startStdioServer(): Promise<void>;

package/dist/mcp.js

@@ -0,0 +1,94 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { registerCachedContentResource } from './cache.js';
+import { config } from './config.js';
+import { destroyAgents } from './fetch.js';
+import { logError, logInfo } from './observability.js';
+import { registerTools } from './tools.js';
+import { shutdownTransformWorkerPool } from './transform.js';
+function createServerInfo() {
+    return {
+        name: config.server.name,
+        version: config.server.version,
+    };
+}
+function createServerCapabilities() {
+    return {
+        tools: { listChanged: false },
+        resources: { listChanged: true, subscribe: true },
+        logging: {},
+    };
+}
+function createServerInstructions(serverVersion) {
+    return `superFetch MCP server |${serverVersion}| A high-performance web content fetching and processing server.`;
+}
+export function createMcpServer() {
+    const server = new McpServer(createServerInfo(), {
+        capabilities: createServerCapabilities(),
+        instructions: createServerInstructions(config.server.version),
+    });
+    registerTools(server);
+    registerCachedContentResource(server);
+    return server;
+}
+function attachServerErrorHandler(server) {
+    server.server.onerror = (error) => {
+        logError('[MCP Error]', error instanceof Error ? error : { error });
+    };
+}
+function handleShutdownSignal(server, signal) {
+    process.stderr.write(`\n${signal} received, shutting down superFetch MCP server...\n`);
+    Promise.resolve()
+        .then(async () => {
+        destroyAgents();
+        await shutdownTransformWorkerPool();
+        await server.close();
+    })
+        .catch((err) => {
+        logError('Error during shutdown', err instanceof Error ? err : undefined);
+    })
+        .finally(() => {
+        process.exit(0);
+    });
+}
+function createShutdownHandler(server) {
+    let shuttingDown = false;
+    let initialSignal = null;
+    return (signal) => {
+        if (shuttingDown) {
+            logInfo('Shutdown already in progress; ignoring signal', {
+                signal,
+                initialSignal,
+            });
+            return;
+        }
+        shuttingDown = true;
+        initialSignal = signal;
+        handleShutdownSignal(server, signal);
+    };
+}
+function registerSignalHandlers(handler) {
+    process.once('SIGINT', () => {
+        handler('SIGINT');
+    });
+    process.once('SIGTERM', () => {
+        handler('SIGTERM');
+    });
+}
+async function connectStdioServer(server, transport) {
+    try {
+        await server.connect(transport);
+        logInfo('superFetch MCP server running on stdio');
+    }
+    catch (error) {
+        logError('Failed to start stdio server', error instanceof Error ? error : undefined);
+        process.exit(1);
+    }
+}
+export async function startStdioServer() {
+    const server = createMcpServer();
+    const transport = new StdioServerTransport();
+    attachServerErrorHandler(server);
+    registerSignalHandlers(createShutdownHandler(server));
+    await connectStdioServer(server, transport);
+}

package/dist/middleware/error-handler.d.ts

@@ -1,2 +1,2 @@
 import type { NextFunction, Request, Response } from 'express';
-export declare function errorHandler(err: Error, req: Request, res: Response, _next: NextFunction): void;
+export declare function errorHandler(err: Error, req: Request, res: Response, next: NextFunction): void;

package/dist/middleware/error-handler.js

@@ -1,55 +1,56 @@
 import { FetchError } from '../errors/app-error.js';
 import { logError } from '../services/logger.js';
-function getStatusCode(err) {
-    return err instanceof FetchError ? err.statusCode : 500;
+function getStatusCode(fetchError) {
+    return fetchError ? fetchError.statusCode : 500;
 }
-function getErrorCode(err) {
-    return err instanceof FetchError ? err.code : 'INTERNAL_ERROR';
+function getErrorCode(fetchError) {
+    return fetchError ? fetchError.code : 'INTERNAL_ERROR';
 }
-function getErrorMessage(err) {
-    return err instanceof FetchError ? err.message : 'Internal Server Error';
+function getErrorMessage(fetchError) {
+    return fetchError ? fetchError.message : 'Internal Server Error';
 }
-function getErrorDetails(err) {
-    if (err instanceof FetchError && Object.keys(err.details).length > 0) {
-        return err.details;
+function getErrorDetails(fetchError) {
+    if (fetchError && Object.keys(fetchError.details).length > 0) {
+        return fetchError.details;
     }
     return undefined;
 }
-function setRetryAfterHeader(res, err) {
-    const retryAfter = resolveRetryAfter(err);
-    if (!retryAfter)
+function setRetryAfterHeader(res, fetchError) {
+    const retryAfter = resolveRetryAfter(fetchError);
+    if (retryAfter === undefined)
         return;
     res.set('Retry-After', retryAfter);
 }
-function buildErrorResponse(err) {
-    const details = getErrorDetails(err);
+function buildErrorResponse(fetchError) {
+    const details = getErrorDetails(fetchError);
     const response = {
         error: {
-            message: getErrorMessage(err),
-            code: getErrorCode(err),
-            statusCode: getStatusCode(err),
+            message: getErrorMessage(fetchError),
+            code: getErrorCode(fetchError),
+            statusCode: getStatusCode(fetchError),
             ...(details && { details }),
         },
     };
     // Never expose stack traces in production
     return response;
 }
-function resolveRetryAfter(err) {
-    if (!(err instanceof FetchError))
-        return null;
-    if (err.statusCode !== 429)
-        return null;
-    const { retryAfter } = err.details;
-    if (!isRetryAfterValue(retryAfter))
-        return null;
-    return String(retryAfter);
+function resolveRetryAfter(fetchError) {
+    if (fetchError?.statusCode !== 429)
+        return undefined;
+    const { retryAfter } = fetchError.details;
+    return isRetryAfterValue(retryAfter) ? String(retryAfter) : undefined;
 }
 function isRetryAfterValue(value) {
     return typeof value === 'number' || typeof value === 'string';
 }
-export function errorHandler(err, req, res, _next) {
-    const statusCode = getStatusCode(err);
+export function errorHandler(err, req, res, next) {
+    if (res.headersSent) {
+        next(err);
+        return;
+    }
+    const fetchError = err instanceof FetchError ? err : null;
+    const statusCode = getStatusCode(fetchError);
     logError(`HTTP ${statusCode}: ${err.message} - ${req.method} ${req.path}`, err);
-    setRetryAfterHeader(res, err);
-    res.status(statusCode).json(buildErrorResponse(err));
+    setRetryAfterHeader(res, fetchError);
+    res.status(statusCode).json(buildErrorResponse(fetchError));
 }

package/dist/observability.d.ts

@@ -0,0 +1,16 @@
+export type LogMetadata = Record<string, unknown>;
+interface RequestContext {
+    readonly requestId: string;
+    readonly sessionId?: string;
+    readonly operationId?: string;
+}
+export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
+export declare function getRequestId(): string | undefined;
+export declare function getSessionId(): string | undefined;
+export declare function getOperationId(): string | undefined;
+export declare function logInfo(message: string, meta?: LogMetadata): void;
+export declare function logDebug(message: string, meta?: LogMetadata): void;
+export declare function logWarn(message: string, meta?: LogMetadata): void;
+export declare function logError(message: string, error?: Error | LogMetadata): void;
+export declare function redactUrl(rawUrl: string): string;
+export {};

package/dist/observability.js

@@ -0,0 +1,78 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { config } from './config.js';
+const requestContext = new AsyncLocalStorage();
+export function runWithRequestContext(context, fn) {
+    return requestContext.run(context, fn);
+}
+export function getRequestId() {
+    return requestContext.getStore()?.requestId;
+}
+export function getSessionId() {
+    return requestContext.getStore()?.sessionId;
+}
+export function getOperationId() {
+    return requestContext.getStore()?.operationId;
+}
+function formatMetadata(meta) {
+    const requestId = getRequestId();
+    const sessionId = getSessionId();
+    const operationId = getOperationId();
+    const contextMeta = {};
+    if (requestId)
+        contextMeta.requestId = requestId;
+    if (sessionId)
+        contextMeta.sessionId = sessionId;
+    if (operationId)
+        contextMeta.operationId = operationId;
+    const merged = { ...contextMeta, ...meta };
+    return Object.keys(merged).length > 0 ? ` ${JSON.stringify(merged)}` : '';
+}
+function createTimestamp() {
+    return new Date().toISOString();
+}
+function formatLogEntry(level, message, meta) {
+    return `[${createTimestamp()}] ${level.toUpperCase()}: ${message}${formatMetadata(meta)}`;
+}
+function shouldLog(level) {
+    // Debug logs only when LOG_LEVEL=debug
+    if (level === 'debug')
+        return config.logging.level === 'debug';
+    // All other levels always log
+    return true;
+}
+export function logInfo(message, meta) {
+    if (shouldLog('info')) {
+        process.stderr.write(`${formatLogEntry('info', message, meta)}\n`);
+    }
+}
+export function logDebug(message, meta) {
+    if (shouldLog('debug')) {
+        process.stderr.write(`${formatLogEntry('debug', message, meta)}\n`);
+    }
+}
+export function logWarn(message, meta) {
+    if (shouldLog('warn')) {
+        process.stderr.write(`${formatLogEntry('warn', message, meta)}\n`);
+    }
+}
+export function logError(message, error) {
+    if (!shouldLog('error'))
+        return;
+    const errorMeta = error instanceof Error
+        ? { error: error.message, stack: error.stack }
+        : (error ?? {});
+    process.stderr.write(`${formatLogEntry('error', message, errorMeta)}\n`);
+}
+export function redactUrl(rawUrl) {
+    try {
+        const url = new URL(rawUrl);
+        url.username = '';
+        url.password = '';
+        url.hash = '';
+        url.search = '';
+        return url.toString();
+    }
+    catch {
+        return rawUrl;
+    }
+}

package/dist/resources/cached-content-params.d.ts

@@ -0,0 +1,5 @@
+export declare const CACHE_NAMESPACE = "markdown";
+export declare function resolveCacheParams(params: unknown): {
+    namespace: string;
+    urlHash: string;
+};

package/dist/resources/cached-content-params.js

@@ -0,0 +1,36 @@
+import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
+import { isRecord } from '../utils/guards.js';
+export const CACHE_NAMESPACE = 'markdown';
+const HASH_PATTERN = /^[a-f0-9.]+$/i;
+export function resolveCacheParams(params) {
+    const parsed = requireRecordParams(params);
+    const namespace = requireParamString(parsed, 'namespace');
+    const urlHash = requireParamString(parsed, 'urlHash');
+    if (!isValidNamespace(namespace) || !isValidHash(urlHash)) {
+        throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
+    }
+    return { namespace, urlHash };
+}
+function requireRecordParams(value) {
+    if (!isRecord(value)) {
+        throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
+    }
+    return value;
+}
+function requireParamString(params, key) {
+    const raw = params[key];
+    const resolved = resolveStringParam(raw);
+    if (!resolved) {
+        throw new McpError(ErrorCode.InvalidParams, 'Both namespace and urlHash parameters are required');
+    }
+    return resolved;
+}
+function isValidNamespace(namespace) {
+    return namespace === CACHE_NAMESPACE;
+}
+function isValidHash(hash) {
+    return HASH_PATTERN.test(hash) && hash.length >= 8 && hash.length <= 64;
+}
+function resolveStringParam(value) {
+    return typeof value === 'string' ? value : null;
+}

package/dist/resources/cached-content.js

@@ -4,10 +4,42 @@ import * as cache from '../services/cache.js';
 import { parseCacheKey, toResourceUri } from '../services/cache-keys.js';
 import { logWarn } from '../services/logger.js';
 import { parseCachedPayload, resolveCachedPayloadContent, } from '../utils/cached-payload.js';
-import { getErrorMessage } from '../utils/error-utils.js';
+import { getErrorMessage } from '../utils/error-details.js';
 import { isRecord } from '../utils/guards.js';
 const CACHE_NAMESPACE = 'markdown';
 const HASH_PATTERN = /^[a-f0-9.]+$/i;
+function resolveCacheParams(params) {
+    const parsed = requireRecordParams(params);
+    const namespace = requireParamString(parsed, 'namespace');
+    const urlHash = requireParamString(parsed, 'urlHash');
+    if (!isValidNamespace(namespace) || !isValidHash(urlHash)) {
+        throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
+    }
+    return { namespace, urlHash };
+}
+function requireRecordParams(value) {
+    if (!isRecord(value)) {
+        throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
+    }
+    return value;
+}
+function requireParamString(params, key) {
+    const raw = params[key];
+    const resolved = resolveStringParam(raw);
+    if (!resolved) {
+        throw new McpError(ErrorCode.InvalidParams, 'Both namespace and urlHash parameters are required');
+    }
+    return resolved;
+}
+function isValidNamespace(namespace) {
+    return namespace === CACHE_NAMESPACE;
+}
+function isValidHash(hash) {
+    return HASH_PATTERN.test(hash) && hash.length >= 8 && hash.length <= 64;
+}
+function resolveStringParam(value) {
+    return typeof value === 'string' ? value : null;
+}
 function buildResourceEntry(namespace, urlHash) {
     return {
         name: `${namespace}:${urlHash}`,
@@ -42,29 +74,6 @@ export function registerCachedContentResource(server) {
     registerCacheContentResource(server);
     registerCacheUpdateSubscription(server);
 }
-function resolveCacheParams(params) {
-    const parsed = requireRecordParams(params);
-    const namespace = requireParamString(parsed, 'namespace');
-    const urlHash = requireParamString(parsed, 'urlHash');
-    if (!isValidNamespace(namespace) || !isValidHash(urlHash)) {
-        throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
-    }
-    return { namespace, urlHash };
-}
-function requireRecordParams(value) {
-    if (!isRecord(value)) {
-        throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
-    }
-    return value;
-}
-function requireParamString(params, key) {
-    const raw = params[key];
-    const resolved = resolveStringParam(raw);
-    if (!resolved) {
-        throw new McpError(ErrorCode.InvalidParams, 'Both namespace and urlHash parameters are required');
-    }
-    return resolved;
-}
 function buildCachedContentResponse(uri, cacheKey) {
     const cached = requireCacheEntry(cacheKey);
     return buildMarkdownContentResponse(uri, cached.content);
@@ -98,15 +107,6 @@ function registerCacheUpdateSubscription(server) {
         unsubscribe();
     };
 }
-function isValidNamespace(namespace) {
-    return namespace === CACHE_NAMESPACE;
-}
-function isValidHash(hash) {
-    return HASH_PATTERN.test(hash) && hash.length >= 8 && hash.length <= 64;
-}
-function resolveStringParam(value) {
-    return typeof value === 'string' ? value : null;
-}
 function requireCacheEntry(cacheKey) {
     const cached = cache.get(cacheKey);
     if (!cached) {

package/dist/server.js

@@ -1,8 +1,9 @@
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { config } from './config/index.js';
-import { destroyAgents } from './services/fetcher/agents.js';
+import { destroyAgents } from './services/fetcher.js';
 import { logError, logInfo } from './services/logger.js';
+import { shutdownTransformWorkerPool } from './services/transform-worker-pool.js';
 import { registerTools } from './tools/index.js';
 import { registerCachedContentResource } from './resources/cached-content.js';
 function createServerInfo() {
@@ -37,9 +38,12 @@ function attachServerErrorHandler(server) {
 }
 function handleShutdownSignal(server, signal) {
     process.stderr.write(`\n${signal} received, shutting down superFetch MCP server...\n`);
-    destroyAgents();
-    server
-        .close()
+    Promise.resolve()
+        .then(async () => {
+        destroyAgents();
+        await shutdownTransformWorkerPool();
+        await server.close();
+    })
         .catch((err) => {
         logError('Error during shutdown', err instanceof Error ? err : undefined);
     })
@@ -48,15 +52,26 @@ function handleShutdownSignal(server, signal) {
     });
 }
 function createShutdownHandler(server) {
+    let shuttingDown = false;
+    let initialSignal = null;
     return (signal) => {
+        if (shuttingDown) {
+            logInfo('Shutdown already in progress; ignoring signal', {
+                signal,
+                initialSignal,
+            });
+            return;
+        }
+        shuttingDown = true;
+        initialSignal = signal;
         handleShutdownSignal(server, signal);
     };
 }
 function registerSignalHandlers(handler) {
-    process.on('SIGINT', () => {
+    process.once('SIGINT', () => {
         handler('SIGINT');
     });
-    process.on('SIGTERM', () => {
+    process.once('SIGTERM', () => {
         handler('SIGTERM');
     });
 }

package/dist/services/cache-events.d.ts

@@ -0,0 +1,8 @@
+import type { CacheKeyParts } from './cache-keys.js';
+export interface CacheUpdateEvent extends CacheKeyParts {
+    cacheKey: string;
+}
+type CacheUpdateListener = (event: CacheUpdateEvent) => void;
+export declare function onCacheUpdate(listener: CacheUpdateListener): () => void;
+export declare function notifyCacheUpdate(cacheKey: string): void;
+export {};

package/dist/services/cache-events.js

@@ -0,0 +1,19 @@
+import { parseCacheKey } from './cache-keys.js';
+const updateListeners = new Set();
+export function onCacheUpdate(listener) {
+    updateListeners.add(listener);
+    return () => {
+        updateListeners.delete(listener);
+    };
+}
+export function notifyCacheUpdate(cacheKey) {
+    if (updateListeners.size === 0)
+        return;
+    const parts = parseCacheKey(cacheKey);
+    if (!parts)
+        return;
+    const event = { cacheKey, ...parts };
+    for (const listener of updateListeners) {
+        listener(event);
+    }
+}

package/dist/services/cache.d.ts

@@ -1,14 +1,15 @@
 import type { CacheEntry } from '../config/types/content.js';
-import type { CacheKeyParts } from './cache-keys.js';
-interface CacheUpdateEvent extends CacheKeyParts {
+interface CacheUpdateEvent {
     cacheKey: string;
+    namespace: string;
+    urlHash: string;
 }
+type CacheUpdateListener = (event: CacheUpdateEvent) => void;
+export declare function onCacheUpdate(listener: CacheUpdateListener): () => void;
 interface CacheEntryMetadata {
     url: string;
     title?: string;
 }
-type CacheUpdateListener = (event: CacheUpdateEvent) => void;
-export declare function onCacheUpdate(listener: CacheUpdateListener): () => void;
 export declare function get(cacheKey: string | null): CacheEntry | undefined;
 export declare function set(cacheKey: string | null, content: string, metadata: CacheEntryMetadata): void;
 export declare function keys(): readonly string[];