@j0hanz/superfetch 2.2.1 → 2.3.0

package/dist/cache.d.ts

@@ -1,4 +1,4 @@
-import type { Express } from 'express';
+import type { ServerResponse } from 'node:http';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 export interface CacheEntry {
     url: string;
@@ -38,6 +38,5 @@ export declare function keys(): readonly string[];
 export declare function isEnabled(): boolean;
 export declare function registerCachedContentResource(server: McpServer): void;
 export declare function generateSafeFilename(url: string, title?: string, hashFallback?: string, extension?: string): string;
-export declare function registerDownloadRoutes(app: Express): void;
+export declare function handleDownload(res: ServerResponse, namespace: string, hash: string): void;
 export {};
-//# sourceMappingURL=cache.d.ts.map

package/dist/cache.js

@@ -1,11 +1,12 @@
-import { setInterval as setIntervalPromise } from 'node:timers/promises';
+import { LRUCache } from 'lru-cache';
 import { ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { ErrorCode, McpError, SubscribeRequestSchema, UnsubscribeRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
 import { config } from './config.js';
 import { sha256Hex } from './crypto.js';
 import { getErrorMessage } from './errors.js';
+import { stableStringify as stableJsonStringify } from './json.js';
 import { logDebug, logWarn } from './observability.js';
-import { isRecord } from './type-guards.js';
+import { isObject } from './type-guards.js';
 export function parseCachedPayload(raw) {
     try {
         const parsed = JSON.parse(raw);
@@ -31,7 +32,7 @@ function hasOptionalStringProperty(value, key) {
     return typeof prop === 'string';
 }
 function isCachedPayload(value) {
-    if (!isRecord(value))
+    if (!isObject(value))
         return false;
     if (!hasOptionalStringProperty(value, 'content'))
         return false;
@@ -42,124 +43,16 @@ function isCachedPayload(value) {
     return true;
 }
 const CACHE_HASH = {
-    URL_HASH_LENGTH: 16,
-    VARY_HASH_LENGTH: 12,
+    URL_HASH_LENGTH: 32,
+    VARY_HASH_LENGTH: 16,
 };
-const CACHE_VARY_LIMITS = {
-    MAX_STRING_LENGTH: 4096,
-    MAX_KEYS: 64,
-    MAX_ARRAY_LENGTH: 64,
-    MAX_DEPTH: 6,
-    MAX_NODES: 512,
-};
-function bumpStableStringifyNodeCount(state) {
-    state.nodes += 1;
-    return state.nodes <= CACHE_VARY_LIMITS.MAX_NODES;
-}
-function stableStringifyPrimitive(value) {
-    if (value === null || value === undefined) {
-        return '';
-    }
-    const json = JSON.stringify(value);
-    return typeof json === 'string' ? json : '';
-}
-function stableStringifyArray(value, state) {
-    if (value.length > CACHE_VARY_LIMITS.MAX_ARRAY_LENGTH) {
-        return null;
-    }
-    const parts = ['['];
-    let length = 1;
-    for (let index = 0; index < value.length; index += 1) {
-        if (index > 0) {
-            parts.push(',');
-            length += 1;
-            if (length > CACHE_VARY_LIMITS.MAX_STRING_LENGTH)
-                return null;
-        }
-        const entry = stableStringifyInner(value[index], state);
-        if (entry === null)
-            return null;
-        parts.push(entry);
-        length += entry.length;
-        if (length > CACHE_VARY_LIMITS.MAX_STRING_LENGTH)
-            return null;
-    }
-    parts.push(']');
-    length += 1;
-    return length > CACHE_VARY_LIMITS.MAX_STRING_LENGTH ? null : parts.join('');
-}
-function stableStringifyRecord(value, state) {
-    const keys = Object.keys(value);
-    if (keys.length > CACHE_VARY_LIMITS.MAX_KEYS) {
-        return null;
-    }
-    keys.sort((a, b) => a.localeCompare(b));
-    const parts = ['{'];
-    let length = 1;
-    let isFirst = true;
-    for (const key of keys) {
-        const entryValue = value[key];
-        if (entryValue === undefined)
-            continue;
-        const encodedValue = stableStringifyInner(entryValue, state);
-        if (encodedValue === null)
-            return null;
-        const entry = `${JSON.stringify(key)}:${encodedValue}`;
-        if (!isFirst) {
-            parts.push(',');
-            length += 1;
-            if (length > CACHE_VARY_LIMITS.MAX_STRING_LENGTH)
-                return null;
-        }
-        parts.push(entry);
-        length += entry.length;
-        if (length > CACHE_VARY_LIMITS.MAX_STRING_LENGTH)
-            return null;
-        isFirst = false;
-    }
-    parts.push('}');
-    length += 1;
-    return length > CACHE_VARY_LIMITS.MAX_STRING_LENGTH ? null : parts.join('');
-}
-function stableStringifyObject(value, state) {
-    if (state.stack.has(value)) {
-        return null;
-    }
-    if (state.depth >= CACHE_VARY_LIMITS.MAX_DEPTH) {
-        return null;
-    }
-    state.stack.add(value);
-    state.depth += 1;
+function stableStringify(value) {
     try {
-        if (Array.isArray(value)) {
-            return stableStringifyArray(value, state);
-        }
-        return isRecord(value) ? stableStringifyRecord(value, state) : null;
+        return stableJsonStringify(value);
     }
-    finally {
-        state.depth -= 1;
-        state.stack.delete(value);
-    }
-}
-function stableStringifyInner(value, state) {
-    if (!bumpStableStringifyNodeCount(state)) {
+    catch {
         return null;
     }
-    if (value === null || value === undefined) {
-        return 'null';
-    }
-    if (typeof value !== 'object') {
-        return stableStringifyPrimitive(value);
-    }
-    return stableStringifyObject(value, state);
-}
-function stableStringify(value) {
-    const state = {
-        depth: 0,
-        nodes: 0,
-        stack: new WeakSet(),
-    };
-    return stableStringifyInner(value, state);
 }
 function createHashFragment(input, length) {
     return sha256Hex(input).substring(0, length);
@@ -174,8 +67,7 @@ function getVaryHash(vary) {
         return undefined;
     let varyString;
     if (typeof vary === 'string') {
-        varyString =
-            vary.length > CACHE_VARY_LIMITS.MAX_STRING_LENGTH ? null : vary;
+        varyString = vary;
     }
     else {
         varyString = stableStringify(vary);
@@ -204,15 +96,20 @@ export function parseCacheKey(cacheKey) {
         return null;
     return { namespace, urlHash };
 }
+function buildCacheResourceUri(namespace, urlHash) {
+    return `superfetch://cache/${namespace}/${urlHash}`;
+}
 export function toResourceUri(cacheKey) {
     const parts = parseCacheKey(cacheKey);
     if (!parts)
         return null;
-    return `superfetch://cache/${parts.namespace}/${parts.urlHash}`;
+    return buildCacheResourceUri(parts.namespace, parts.urlHash);
 }
-const contentCache = new Map();
-let cleanupController = null;
-let nextExpiryAt = null;
+const contentCache = new LRUCache({
+    max: config.cache.maxKeys,
+    ttl: config.cache.ttl * 1000,
+    updateAgeOnGet: false,
+});
 const updateListeners = new Set();
 export function onCacheUpdate(listener) {
     updateListeners.add(listener);
@@ -231,47 +128,10 @@ function notifyCacheUpdate(cacheKey) {
         listener(event);
     }
 }
-function startCleanupLoop() {
-    if (cleanupController)
-        return;
-    cleanupController = new AbortController();
-    void runCleanupLoop(cleanupController.signal).catch((error) => {
-        if (error instanceof Error && error.name !== 'AbortError') {
-            logWarn('Cache cleanup loop failed', { error: getErrorMessage(error) });
-        }
-    });
-}
-async function runCleanupLoop(signal) {
-    const intervalMs = Math.floor(config.cache.ttl * 1000);
-    for await (const getNow of setIntervalPromise(intervalMs, Date.now, {
-        signal,
-        ref: false,
-    })) {
-        enforceCacheLimits(getNow());
-    }
-}
-function enforceCacheLimits(now) {
-    if (nextExpiryAt !== null && now < nextExpiryAt) {
-        trimCacheToMaxKeys();
-        return;
-    }
-    let nextExpiry = null;
-    for (const [key, item] of contentCache.entries()) {
-        if (now > item.expiresAt) {
-            contentCache.delete(key);
-            continue;
-        }
-        if (nextExpiry === null || item.expiresAt < nextExpiry) {
-            nextExpiry = item.expiresAt;
-        }
-    }
-    nextExpiryAt = nextExpiry;
-    trimCacheToMaxKeys();
-}
 export function get(cacheKey) {
     if (!isCacheReadable(cacheKey))
         return undefined;
-    return runCacheOperation(cacheKey, 'Cache get error', () => readCacheEntry(cacheKey));
+    return runCacheOperation(cacheKey, 'Cache get error', () => contentCache.get(cacheKey));
 }
 function isCacheReadable(cacheKey) {
     return config.cache.enabled && Boolean(cacheKey);
@@ -288,28 +148,10 @@ function runCacheOperation(cacheKey, message, operation) {
         return undefined;
     }
 }
-function readCacheEntry(cacheKey) {
-    const now = Date.now();
-    return readCacheItem(cacheKey, now)?.entry;
-}
-function isExpired(item, now) {
-    return now > item.expiresAt;
-}
-function readCacheItem(cacheKey, now) {
-    const item = contentCache.get(cacheKey);
-    if (!item)
-        return undefined;
-    if (isExpired(item, now)) {
-        contentCache.delete(cacheKey);
-        return undefined;
-    }
-    return item;
-}
 export function set(cacheKey, content, metadata) {
     if (!isCacheWritable(cacheKey, content))
         return;
     runCacheOperation(cacheKey, 'Cache set error', () => {
-        startCleanupLoop();
         const now = Date.now();
         const expiresAtMs = now + config.cache.ttl * 1000;
         const entry = buildCacheEntry({
@@ -318,11 +160,11 @@ export function set(cacheKey, content, metadata) {
             fetchedAtMs: now,
             expiresAtMs,
         });
-        persistCacheEntry(cacheKey, entry, expiresAtMs);
+        persistCacheEntry(cacheKey, entry);
     });
 }
 export function keys() {
-    return Array.from(contentCache.keys());
+    return [...contentCache.keys()];
 }
 export function isEnabled() {
     return config.cache.enabled;
@@ -336,33 +178,10 @@ function buildCacheEntry({ content, metadata, fetchedAtMs, expiresAtMs, }) {
         ...(metadata.title === undefined ? {} : { title: metadata.title }),
     };
 }
-function persistCacheEntry(cacheKey, entry, expiresAtMs) {
-    contentCache.set(cacheKey, { entry, expiresAt: expiresAtMs });
-    if (nextExpiryAt === null || expiresAtMs < nextExpiryAt) {
-        nextExpiryAt = expiresAtMs;
-    }
-    trimCacheToMaxKeys();
+function persistCacheEntry(cacheKey, entry) {
+    contentCache.set(cacheKey, entry);
     notifyCacheUpdate(cacheKey);
 }
-function trimCacheToMaxKeys() {
-    if (contentCache.size <= config.cache.maxKeys)
-        return;
-    removeOldestEntries(contentCache.size - config.cache.maxKeys);
-}
-function removeOldestEntries(count) {
-    const iterator = contentCache.keys();
-    for (let removed = 0; removed < count; removed += 1) {
-        const next = iterator.next();
-        if (next.done)
-            break;
-        const removedKey = next.value;
-        const removedItem = contentCache.get(removedKey);
-        contentCache.delete(removedKey);
-        if (nextExpiryAt !== null && removedItem?.expiresAt === nextExpiryAt) {
-            nextExpiryAt = null;
-        }
-    }
-}
 function logCacheError(message, cacheKey, error) {
     logWarn(message, {
         key: cacheKey.length > 100 ? cacheKey.slice(0, 100) : cacheKey,
@@ -371,18 +190,22 @@ function logCacheError(message, cacheKey, error) {
 }
 const CACHE_NAMESPACE = 'markdown';
 const HASH_PATTERN = /^[a-f0-9.]+$/i;
+const INVALID_CACHE_PARAMS_MESSAGE = 'Invalid cache resource parameters';
+function throwInvalidCacheParams() {
+    throw new McpError(ErrorCode.InvalidParams, INVALID_CACHE_PARAMS_MESSAGE);
+}
 function resolveCacheParams(params) {
     const parsed = requireRecordParams(params);
     const namespace = requireParamString(parsed, 'namespace');
     const urlHash = requireParamString(parsed, 'urlHash');
     if (!isValidNamespace(namespace) || !isValidHash(urlHash)) {
-        throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
+        throwInvalidCacheParams();
     }
     return { namespace, urlHash };
 }
 function requireRecordParams(value) {
-    if (!isRecord(value)) {
-        throw new McpError(ErrorCode.InvalidParams, 'Invalid cache resource parameters');
+    if (!isObject(value)) {
+        throwInvalidCacheParams();
     }
     return value;
 }
@@ -428,7 +251,7 @@ function resolveStringParam(value) {
 function buildResourceEntry(namespace, urlHash) {
     return {
         name: `${namespace}:${urlHash}`,
-        uri: `superfetch://cache/${namespace}/${urlHash}`,
+        uri: buildCacheResourceUri(namespace, urlHash),
         description: `Cached content entry for ${namespace}`,
         mimeType: 'text/markdown',
     };
@@ -462,11 +285,11 @@ function attachInitializedGate(server) {
 }
 function getClientResourceCapabilities(server) {
     const caps = server.server.getClientCapabilities();
-    if (!caps || !isRecord(caps)) {
+    if (!caps || !isObject(caps)) {
         return { listChanged: false, subscribe: false };
     }
     const { resources } = caps;
-    if (!isRecord(resources)) {
+    if (!isObject(resources)) {
         return { listChanged: false, subscribe: false };
     }
     const { listChanged, subscribe } = resources;
@@ -573,41 +396,32 @@ function buildMarkdownContentResponse(uri, content) {
         ],
     };
 }
-function isSingleParam(value) {
-    return typeof value === 'string';
-}
-function parseDownloadParams(req) {
-    const { namespace, hash } = req.params;
-    if (!isSingleParam(namespace) || !isSingleParam(hash))
+function parseDownloadParams(namespace, hash) {
+    const resolvedNamespace = resolveStringParam(namespace);
+    const resolvedHash = resolveStringParam(hash);
+    if (!resolvedNamespace || !resolvedHash)
         return null;
-    if (!namespace || !hash)
+    if (!isValidNamespace(resolvedNamespace))
         return null;
-    if (!isValidNamespace(namespace))
+    if (!isValidHash(resolvedHash))
         return null;
-    if (!isValidHash(hash))
-        return null;
-    return { namespace, hash };
+    return { namespace: resolvedNamespace, hash: resolvedHash };
 }
 function buildCacheKeyFromParams(params) {
     return `${params.namespace}:${params.hash}`;
 }
+function sendJsonError(res, status, error, code) {
+    res.writeHead(status, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify({ error, code }));
+}
 function respondBadRequest(res, message) {
-    res.status(400).json({
-        error: message,
-        code: 'BAD_REQUEST',
-    });
+    sendJsonError(res, 400, message, 'BAD_REQUEST');
 }
 function respondNotFound(res) {
-    res.status(404).json({
-        error: 'Content not found or expired',
-        code: 'NOT_FOUND',
-    });
+    sendJsonError(res, 404, 'Content not found or expired', 'NOT_FOUND');
 }
 function respondServiceUnavailable(res) {
-    res.status(503).json({
-        error: 'Download service is disabled',
-        code: 'SERVICE_UNAVAILABLE',
-    });
+    sendJsonError(res, 503, 'Download service is disabled', 'SERVICE_UNAVAILABLE');
 }
 export function generateSafeFilename(url, title, hashFallback, extension = '.md') {
     const fromUrl = extractFilenameFromUrl(url);
@@ -714,14 +528,14 @@ function sendDownloadPayload(res, payload) {
     res.setHeader('Content-Disposition', disposition);
     res.setHeader('Cache-Control', `private, max-age=${config.cache.ttl}`);
     res.setHeader('X-Content-Type-Options', 'nosniff');
-    res.send(payload.content);
+    res.end(payload.content);
 }
-function handleDownload(req, res) {
+export function handleDownload(res, namespace, hash) {
     if (!config.cache.enabled) {
         respondServiceUnavailable(res);
         return;
     }
-    const params = parseDownloadParams(req);
+    const params = parseDownloadParams(namespace, hash);
     if (!params) {
         respondBadRequest(res, 'Invalid namespace or hash format');
         return;
@@ -742,7 +556,3 @@ function handleDownload(req, res) {
     logDebug('Serving download', { cacheKey, fileName: payload.fileName });
     sendDownloadPayload(res, payload);
 }
-export function registerDownloadRoutes(app) {
-    app.get('/mcp/downloads/:namespace/:hash', handleDownload);
-}
-//# sourceMappingURL=cache.js.map

package/dist/config.d.ts

@@ -57,6 +57,10 @@ export declare const config: {
         maxBlockLength: number;
         minParagraphLength: number;
     };
+    noiseRemoval: {
+        extraTokens: string[];
+        extraSelectors: string[];
+    };
     logging: {
         level: LogLevel;
     };
@@ -68,6 +72,8 @@ export declare const config: {
     security: {
         blockedHosts: Set<string>;
         blockedIpPatterns: readonly [RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp, RegExp];
+        blockedIpPattern: RegExp;
+        blockedIpv4MappedPattern: RegExp;
         allowedHosts: Set<string>;
         apiKey: string | undefined;
         allowRemote: boolean;
@@ -83,4 +89,3 @@ export declare const config: {
 };
 export declare function enableHttpMode(): void;
 export {};
-//# sourceMappingURL=config.d.ts.map

package/dist/config.js

@@ -33,39 +33,24 @@ const ALLOWED_LOG_LEVELS = new Set([
 function isLogLevel(value) {
     return ALLOWED_LOG_LEVELS.has(value);
 }
-function isBelowMin(value, min) {
-    if (min === undefined)
-        return false;
-    return value < min;
+function isOutsideRange(value, min, max) {
+    return ((min !== undefined && value < min) || (max !== undefined && value > max));
 }
-function isAboveMax(value, max) {
-    if (max === undefined)
-        return false;
-    return value > max;
-}
-function parseInteger(envValue, defaultValue, min, max) {
+function parseIntegerValue(envValue, min, max) {
     if (!envValue)
-        return defaultValue;
+        return null;
     const parsed = parseInt(envValue, 10);
     if (Number.isNaN(parsed))
-        return defaultValue;
-    if (isBelowMin(parsed, min))
-        return defaultValue;
-    if (isAboveMax(parsed, max))
-        return defaultValue;
+        return null;
+    if (isOutsideRange(parsed, min, max))
+        return null;
     return parsed;
 }
+function parseInteger(envValue, defaultValue, min, max) {
+    return parseIntegerValue(envValue, min, max) ?? defaultValue;
+}
 function parseOptionalInteger(envValue, min, max) {
-    if (!envValue)
-        return undefined;
-    const parsed = parseInt(envValue, 10);
-    if (Number.isNaN(parsed))
-        return undefined;
-    if (isBelowMin(parsed, min))
-        return undefined;
-    if (isAboveMax(parsed, max))
-        return undefined;
-    return parsed;
+    return parseIntegerValue(envValue, min, max) ?? undefined;
 }
 function parseBoolean(envValue, defaultValue) {
     if (!envValue)
@@ -88,6 +73,9 @@ function parseUrlEnv(value, name) {
     }
     return new URL(value);
 }
+function readUrlEnv(name) {
+    return parseUrlEnv(process.env[name], name);
+}
 function parseAllowedHosts(envValue) {
     const hosts = new Set();
     for (const entry of parseList(envValue)) {
@@ -123,16 +111,16 @@ const DEFAULT_TOOL_TIMEOUT_MS = TIMEOUT.DEFAULT_FETCH_TIMEOUT_MS +
     5000;
 function readCoreOAuthUrls() {
     return {
-        issuerUrl: parseUrlEnv(process.env.OAUTH_ISSUER_URL, 'OAUTH_ISSUER_URL'),
-        authorizationUrl: parseUrlEnv(process.env.OAUTH_AUTHORIZATION_URL, 'OAUTH_AUTHORIZATION_URL'),
-        tokenUrl: parseUrlEnv(process.env.OAUTH_TOKEN_URL, 'OAUTH_TOKEN_URL'),
+        issuerUrl: readUrlEnv('OAUTH_ISSUER_URL'),
+        authorizationUrl: readUrlEnv('OAUTH_AUTHORIZATION_URL'),
+        tokenUrl: readUrlEnv('OAUTH_TOKEN_URL'),
     };
 }
 function readOptionalOAuthUrls(baseUrl) {
     return {
-        revocationUrl: parseUrlEnv(process.env.OAUTH_REVOCATION_URL, 'OAUTH_REVOCATION_URL'),
-        registrationUrl: parseUrlEnv(process.env.OAUTH_REGISTRATION_URL, 'OAUTH_REGISTRATION_URL'),
-        introspectionUrl: parseUrlEnv(process.env.OAUTH_INTROSPECTION_URL, 'OAUTH_INTROSPECTION_URL'),
+        revocationUrl: readUrlEnv('OAUTH_REVOCATION_URL'),
+        registrationUrl: readUrlEnv('OAUTH_REGISTRATION_URL'),
+        introspectionUrl: readUrlEnv('OAUTH_INTROSPECTION_URL'),
         resourceUrl: parseUrlEnv(process.env.OAUTH_RESOURCE_URL, 'OAUTH_RESOURCE_URL') ??
             new URL('/mcp', baseUrl),
     };
@@ -158,7 +146,7 @@ function collectStaticTokens() {
     if (process.env.API_KEY) {
         staticTokens.add(process.env.API_KEY);
     }
-    return Array.from(staticTokens);
+    return [...staticTokens];
 }
 function buildAuthConfig(baseUrl) {
     const urls = readOAuthUrls(baseUrl);
@@ -225,6 +213,10 @@ export const config = {
         maxBlockLength: 5000,
         minParagraphLength: 10,
     },
+    noiseRemoval: {
+        extraTokens: parseList(process.env.SUPERFETCH_EXTRA_NOISE_TOKENS),
+        extraSelectors: parseList(process.env.SUPERFETCH_EXTRA_NOISE_SELECTORS),
+    },
     logging: {
         level: parseLogLevel(process.env.LOG_LEVEL),
     },
@@ -262,6 +254,10 @@ export const config = {
             /^::ffff:192\.168\./,
             /^::ffff:169\.254\./,
         ],
+        // Combined regex patterns for fast IP blocking (used in fetch.ts)
+        // Split into two patterns to reduce complexity while maintaining performance
+        blockedIpPattern: /^(?:10\.|172\.(?:1[6-9]|2\d|3[01])\.|192\.168\.|127\.|0\.|169\.254\.|100\.64\.|fc00:|fd00:|fe80:)/i,
+        blockedIpv4MappedPattern: /^::ffff:(?:127\.|10\.|172\.(?:1[6-9]|2\d|3[01])\.|192\.168\.|169\.254\.)/i,
         allowedHosts: parseAllowedHosts(process.env.ALLOWED_HOSTS),
         apiKey: process.env.API_KEY,
         allowRemote,
@@ -278,4 +274,3 @@ export const config = {
 export function enableHttpMode() {
     runtimeState.httpMode = true;
 }
-//# sourceMappingURL=config.js.map

package/dist/crypto.d.ts

@@ -1,3 +1,2 @@
 export declare function timingSafeEqualUtf8(a: string, b: string): boolean;
 export declare function sha256Hex(input: string | Uint8Array): string;
-//# sourceMappingURL=crypto.d.ts.map

package/dist/crypto.js

@@ -30,4 +30,3 @@ function hashHex(algorithm, input) {
 export function sha256Hex(input) {
     return hashHex('sha256', input);
 }
-//# sourceMappingURL=crypto.js.map

package/dist/dom-noise-removal.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Remove noise elements from HTML and resolve relative URLs.
+ * Used as a preprocessing step before markdown conversion.
+ */
+export declare function removeNoiseFromHtml(html: string, document?: Document, baseUrl?: string): string;