@j0hanz/fetch-url-mcp 1.5.0 → 1.5.1

package/dist/lib/core.d.ts

@@ -131,6 +131,7 @@ export declare const config: {
         allowedHosts: Set<string>;
         apiKey: string | undefined;
         allowRemote: boolean;
+        allowLocalFetch: boolean;
     };
     auth: AuthConfig;
     rateLimit: {

package/dist/lib/core.js

@@ -458,6 +458,7 @@ export const config = {
         allowedHosts: parseAllowedHosts(env['ALLOWED_HOSTS']),
         apiKey: env['API_KEY'],
         allowRemote,
+        allowLocalFetch: parseBoolean(env['ALLOW_LOCAL_FETCH'], false),
     },
     auth: buildAuthConfig(baseUrl),
     rateLimit: {

package/dist/lib/http.js

@@ -15,7 +15,7 @@ import { Agent } from 'undici';
 import { z } from 'zod';
 import { get as cacheGet, config, getOperationId, getRequestId, logDebug, logError, logWarn, parseCachedPayload, redactUrl, resolveCachedPayloadContent, } from './core.js';
 import { BLOCKED_HOST_SUFFIXES, createDnsPreflight, IpBlocker, RawUrlTransformer, SafeDnsResolver, UrlNormalizer, VALIDATION_ERROR_CODE, } from './url.js';
-import { createErrorWithCode, FetchError, isError, isObject, isSystemError, toError, } from './utils.js';
+import { createErrorWithCode, FetchError, isAbortError, isError, isObject, isSystemError, toError, } from './utils.js';
 const FILENAME_RULES = {
     MAX_LEN: 200,
     UNSAFE_CHARS: /[<>:"/\\|?*\p{C}]/gu,
@@ -315,10 +315,6 @@ function createFetchError(input, url) {
             return new FetchError(input.message ?? 'Unexpected error', url);
     }
 }
-function isAbortError(error) {
-    return (isError(error) &&
-        (error.name === 'AbortError' || error.name === 'TimeoutError'));
-}
 function isTimeoutError(error) {
     return isError(error) && error.name === 'TimeoutError';
 }
@@ -334,7 +330,7 @@ function mapFetchError(error, fallbackUrl, timeoutMs) {
     if (error instanceof FetchError)
         return error;
     const url = resolveErrorUrl(error, fallbackUrl);
-    if (isAbortError(error)) {
+    if (isAbortError(error) || isTimeoutError(error)) {
         return isTimeoutError(error)
             ? createFetchError({ kind: 'timeout', timeout: timeoutMs }, url)
             : createFetchError({ kind: 'canceled' }, url);
@@ -386,7 +382,12 @@ class RedirectFollower {
     async fetchWithRedirects(url, init, maxRedirects) {
         let currentUrl = url;
         const redirectLimit = Math.max(0, maxRedirects);
+        const visited = new Set();
         for (let redirectCount = 0; redirectCount <= redirectLimit; redirectCount += 1) {
+            if (visited.has(currentUrl)) {
+                throw createFetchError({ kind: 'too-many-redirects' }, currentUrl);
+            }
+            visited.add(currentUrl);
             const { response, nextUrl } = await this.withRedirectErrorContext(currentUrl, async () => {
                 let ipAddress;
                 if (this.preflight) {
@@ -405,9 +406,10 @@ class RedirectFollower {
             ...init,
             redirect: 'manual',
         };
+        let agent;
         if (ipAddress) {
             const ca = tls.rootCertificates.length > 0 ? tls.rootCertificates : undefined;
-            const agent = new Agent({
+            agent = new Agent({
                 connect: {
                     lookup: (hostname, options, callback) => {
                         const family = isIP(ipAddress) === 6 ? 6 : 4;
@@ -428,25 +430,30 @@ class RedirectFollower {
             });
             fetchInit.dispatcher = agent;
         }
-        const response = await this.fetchFn(currentUrl, fetchInit);
-        if (!isRedirectStatus(response.status))
-            return { response };
-        if (redirectCount >= redirectLimit) {
+        try {
+            const response = await this.fetchFn(currentUrl, fetchInit);
+            if (!isRedirectStatus(response.status))
+                return { response };
+            if (redirectCount >= redirectLimit) {
+                cancelResponseBody(response);
+                throw createFetchError({ kind: 'too-many-redirects' }, currentUrl);
+            }
+            const location = this.getRedirectLocation(response, currentUrl);
             cancelResponseBody(response);
-            throw createFetchError({ kind: 'too-many-redirects' }, currentUrl);
+            const nextUrl = this.resolveRedirectTarget(currentUrl, location);
+            const parsedNextUrl = new URL(nextUrl);
+            if (parsedNextUrl.protocol !== 'http:' &&
+                parsedNextUrl.protocol !== 'https:') {
+                throw createErrorWithCode(`Unsupported redirect protocol: ${parsedNextUrl.protocol}`, 'EUNSUPPORTEDPROTOCOL');
+            }
+            return {
+                response,
+                nextUrl,
+            };
         }
-        const location = this.getRedirectLocation(response, currentUrl);
-        cancelResponseBody(response);
-        const nextUrl = this.resolveRedirectTarget(currentUrl, location);
-        const parsedNextUrl = new URL(nextUrl);
-        if (parsedNextUrl.protocol !== 'http:' &&
-            parsedNextUrl.protocol !== 'https:') {
-            throw createErrorWithCode(`Unsupported redirect protocol: ${parsedNextUrl.protocol}`, 'EUNSUPPORTEDPROTOCOL');
+        finally {
+            await agent?.close();
         }
-        return {
-            response,
-            nextUrl,
-        };
     }
     getRedirectLocation(response, currentUrl) {
         const location = response.headers.get('location');
@@ -694,16 +701,8 @@ async function decodeResponseIfNeeded(response, url, signal) {
         clearAbortListener();
         abortDecodePipeline();
         void decodedReader.cancel(error).catch(() => undefined);
-        logDebug('Content-Encoding decode failed; using passthrough body', {
-            url: redactUrl(url),
-            encoding: encodingHeader ?? encodings.join(','),
-            error: isError(error) ? error.message : String(error),
-        });
-        return new Response(passthroughBranch, {
-            status: response.status,
-            statusText: response.statusText,
-            headers,
-        });
+        void passthroughBranch.cancel().catch(() => undefined);
+        throw new FetchError(`Content-Encoding decode failed for ${redactUrl(url)}: ${isError(error) ? error.message : String(error)}`, url);
     }
 }
 class ResponseTextReader {
@@ -851,17 +850,16 @@ async function readAndRecordDecodedResponse(response, finalUrl, ctx, telemetry,
         cancelResponseBody(response);
         throw responseError;
     }
-    const decodedResponse = await decodeResponseIfNeeded(response, finalUrl, signal);
-    const contentType = decodedResponse.headers.get('content-type');
+    const contentType = response.headers.get('content-type');
     assertSupportedContentType(contentType, finalUrl);
     const declaredEncoding = getCharsetFromContentType(contentType ?? null);
     if (mode === 'text') {
-        const { text, size, truncated } = await reader.read(decodedResponse, finalUrl, maxBytes, signal, declaredEncoding);
-        telemetry.recordResponse(ctx, decodedResponse, size);
+        const { text, size, truncated } = await reader.read(response, finalUrl, maxBytes, signal, declaredEncoding);
+        telemetry.recordResponse(ctx, response, size);
         return { kind: 'text', text, size, truncated };
     }
-    const { buffer, encoding, size, truncated } = await reader.readBuffer(decodedResponse, finalUrl, maxBytes, signal, declaredEncoding);
-    telemetry.recordResponse(ctx, decodedResponse, size);
+    const { buffer, encoding, size, truncated } = await reader.readBuffer(response, finalUrl, maxBytes, signal, declaredEncoding);
+    telemetry.recordResponse(ctx, response, size);
     return { kind: 'buffer', buffer, encoding, size, truncated };
 }
 function isReadableStreamLike(value) {
@@ -1087,8 +1085,9 @@ const DEFAULT_HEADERS = {
     'User-Agent': config.fetcher.userAgent,
     Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
     'Accept-Language': 'en-US,en;q=0.5',
-    'Accept-Encoding': 'gzip, deflate, br',
-    Connection: 'keep-alive',
+    // Accept-Encoding and Connection are forbidden Fetch API headers.
+    // The undici-based globalThis.fetch manages content negotiation and
+    // decompression transparently per the Fetch spec.
 };
 function buildHeaders() {
     return DEFAULT_HEADERS;

package/dist/lib/url.d.ts

@@ -1,5 +1,5 @@
 import { BlockList } from 'node:net';
-import { type config } from './core.js';
+import { config } from './core.js';
 export declare class SafeDnsResolver {
     private readonly ipBlocker;
     private readonly security;

package/dist/lib/url.js

@@ -1,7 +1,7 @@
 import dns from 'node:dns';
 import { BlockList, isIP, SocketAddress } from 'node:net';
 import { domainToASCII } from 'node:url';
-import { logDebug } from './core.js';
+import { config, logDebug } from './core.js';
 import { createErrorWithCode, isError, isSystemError } from './utils.js';
 const DNS_LOOKUP_TIMEOUT_MS = 5000;
 const CNAME_LOOKUP_MAX_DEPTH = 5;
@@ -76,7 +76,7 @@ export class SafeDnsResolver {
             if (isCloudMetadataHost(normalizedHostname)) {
                 throw createErrorWithCode(`Blocked IP range: ${normalizedHostname}. Cloud metadata endpoints are not allowed`, 'EBLOCKED');
             }
-            if (process.env['ALLOW_LOCAL_FETCH'] !== 'true' &&
+            if (!isLocalFetchAllowed() &&
                 this.ipBlocker.isBlockedIp(normalizedHostname)) {
                 throw createErrorWithCode(`Blocked IP range: ${normalizedHostname}. Private IPs are not allowed`, 'EBLOCKED');
             }
@@ -590,7 +590,7 @@ function isCloudMetadataHost(hostname) {
     return normalized !== null && CLOUD_METADATA_HOSTS.has(normalized.ip);
 }
 function isLocalFetchAllowed() {
-    return process.env['ALLOW_LOCAL_FETCH'] === 'true';
+    return config.security.allowLocalFetch;
 }
 export class IpBlocker {
     security;

package/dist/transform/transform.js

@@ -9,7 +9,7 @@ import { config } from '../lib/core.js';
 import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
 import { isRawTextContentUrl } from '../lib/http.js';
 import { createAbortError, throwIfAborted } from '../lib/utils.js';
-import { FetchError, getErrorMessage } from '../lib/utils.js';
+import { FetchError, getErrorMessage, toError } from '../lib/utils.js';
 import { isObject } from '../lib/utils.js';
 import { translateHtmlFragmentToMarkdown } from './html-translators.js';
 import { extractMetadata, extractMetadataFromHead, mergeMetadata, } from './metadata.js';
@@ -178,13 +178,22 @@ function trimUtf8Buffer(buffer, maxBytes) {
     return buffer.subarray(0, end);
 }
 function trimDanglingTagFragment(content) {
-    const lastOpen = content.lastIndexOf('<');
-    const lastClose = content.lastIndexOf('>');
+    let result = content;
+    // Trim dangling HTML entity (e.g. "&amp" cut before ";")
+    const lastAmp = result.lastIndexOf('&');
+    if (lastAmp !== -1 && lastAmp > result.length - 10) {
+        const tail = result.slice(lastAmp + 1);
+        if (!tail.includes(';') && /^[#a-zA-Z][a-zA-Z0-9]*$/.test(tail)) {
+            result = result.substring(0, lastAmp);
+        }
+    }
+    const lastOpen = result.lastIndexOf('<');
+    const lastClose = result.lastIndexOf('>');
     if (lastOpen > lastClose) {
-        if (lastOpen === content.length - 1) {
-            return content.substring(0, lastOpen);
+        if (lastOpen === result.length - 1) {
+            return result.substring(0, lastOpen);
         }
-        const code = content.codePointAt(lastOpen + 1);
+        const code = result.codePointAt(lastOpen + 1);
         if (code !== undefined &&
             (code === 47 || // '/'
                 code === 33 || // '!'
@@ -192,10 +201,10 @@ function trimDanglingTagFragment(content) {
                 (code >= 65 && code <= 90) || // A-Z
                 (code >= 97 && code <= 122)) // a-z
         ) {
-            return content.substring(0, lastOpen);
+            return result.substring(0, lastOpen);
         }
     }
-    return content;
+    return result;
 }
 function truncateHtml(html, inputTruncated = false) {
     const maxSize = config.constants.maxHtmlSize;
@@ -1047,6 +1056,8 @@ function resolveWorkerFallback(error, htmlOrBuffer, url, options) {
     abortPolicy.throwIfAborted(options.signal, url, 'transform:worker-fallback');
     if (error instanceof FetchError)
         throw error;
+    if (!(error instanceof Error))
+        throw toError(error);
     const message = getErrorMessage(error);
     logWarn('Transform worker failed; falling back to in-process', {
         url: redactUrl(url),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@j0hanz/fetch-url-mcp",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "mcpName": "io.github.j0hanz/fetch-url-mcp",
   "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable Markdown",
   "type": "module",