@lmcc-dev/mult-fetch-mcp-server 1.3.2 → 1.3.3

package/README.md

@@ -1,3 +1,5 @@
+[![MseeP.ai Security Assessment Badge](https://mseep.net/mseep-audited.png)](https://mseep.ai/app/lmcc-dev-mult-fetch-mcp-server)
+
 # mult-fetch-mcp-server
 
 [![npm version](https://img.shields.io/npm/v/@lmcc-dev/mult-fetch-mcp-server.svg)](https://www.npmjs.com/package/@lmcc-dev/mult-fetch-mcp-server)

package/dist/src/lib/fetchers/browser/BrowserFetcher.js

@@ -11,6 +11,7 @@ import { getRandomUserAgent, getSystemProxy } from '../common/utils.js';
 import { ContentSizeManager } from '../../utils/ContentSizeManager.js';
 import { BaseFetcher } from '../common/BaseFetcher.js';
 import { ContentProcessor } from '../../utils/ContentProcessor.js';
+import { validateFetchUrl } from '../../utils/UrlValidator.js';
 /**
  * 浏览器模式获取器类 (Browser mode fetcher class)
  * 使用Puppeteer实现浏览器模式的网页获取 (Implements webpage fetching in browser mode using Puppeteer)
@@ -75,6 +76,7 @@ export class BrowserFetcher extends BaseFetcher {
             if (effectiveProxy) {
                 log('browser.usingProxy', debug, { proxy: effectiveProxy }, COMPONENTS.BROWSER_FETCH);
             }
+            await validateFetchUrl(url);
             // 导航到URL
             log('browser.navigating', debug, { url }, COMPONENTS.BROWSER_FETCH);
             await page.goto(url, {

package/dist/src/lib/fetchers/browser/BrowserInstance.js

@@ -94,32 +94,35 @@ export class BrowserInstance {
                 try {
                     log('browser.startingBrowser', debug, {}, COMPONENTS.BROWSER_FETCH);
                     // 准备浏览器启动参数 (Prepare browser startup parameters)
+                    const browserArgs = [
+                        '--no-sandbox',
+                        '--disable-setuid-sandbox',
+                        '--disable-dev-shm-usage',
+                        '--disable-accelerated-2d-canvas',
+                        '--no-first-run',
+                        '--no-zygote',
+                        '--disable-gpu',
+                        '--disable-infobars',
+                        '--window-position=0,0',
+                        '--ignore-certificate-errors',
+                        '--ignore-certificate-errors-spki-list',
+                        '--disable-extensions',
+                        '--disable-default-apps',
+                        '--enable-features=NetworkService',
+                        '--disable-features=IsolateOrigins,site-per-process',
+                        '--disable-web-security',
+                        '--disable-site-isolation-trials',
+                        '--disable-features=IsolateOrigins',
+                        '--disable-features=site-per-process',
+                        '--disable-blink-features=AutomationControlled',
+                        '--user-agent=' + getRandomUserAgent()
+                    ];
+                    if (process.env.ALLOW_PRIVATE_NETWORK_REQUESTS === 'true') {
+                        browserArgs.push('--disable-features=BlockInsecurePrivateNetworkRequests');
+                    }
                     const launchOptions = {
                         headless: 'new',
-                        args: [
-                            '--no-sandbox',
-                            '--disable-setuid-sandbox',
-                            '--disable-dev-shm-usage',
-                            '--disable-accelerated-2d-canvas',
-                            '--no-first-run',
-                            '--no-zygote',
-                            '--disable-gpu',
-                            '--disable-infobars',
-                            '--window-position=0,0',
-                            '--ignore-certificate-errors',
-                            '--ignore-certificate-errors-spki-list',
-                            '--disable-extensions',
-                            '--disable-default-apps',
-                            '--enable-features=NetworkService',
-                            '--disable-features=IsolateOrigins,site-per-process',
-                            '--disable-web-security',
-                            '--disable-site-isolation-trials',
-                            '--disable-features=BlockInsecurePrivateNetworkRequests',
-                            '--disable-features=IsolateOrigins',
-                            '--disable-features=site-per-process',
-                            '--disable-blink-features=AutomationControlled',
-                            '--user-agent=' + getRandomUserAgent()
-                        ],
+                        args: browserArgs,
                         ignoreHTTPSErrors: true,
                         defaultViewport: {
                             width: 1920,

package/dist/src/lib/fetchers/node/HttpClient.js

@@ -8,6 +8,7 @@ import { HttpProxyAgent } from "http-proxy-agent";
 import { HttpsProxyAgent } from "https-proxy-agent";
 import { log, COMPONENTS } from '../../logger.js';
 import { getRandomUserAgent, randomDelay, getSystemProxy } from '../common/utils.js';
+import { validateFetchUrl, UrlValidationError } from '../../utils/UrlValidator.js';
 /**
  * HTTP客户端类 (HTTP client class)
  * 处理HTTP请求、重定向和错误 (Handle HTTP requests, redirects and errors)
@@ -164,6 +165,7 @@ export class HttpClient {
         // 初始化重定向计数器 (Initialize redirect counter)
         let redirectCount = 0;
         let currentUrl = url;
+        await validateFetchUrl(currentUrl);
         // 记录请求开始时间 (Record request start time)
         const fetchStart = Date.now();
         // 创建AbortController用于超时控制 (Create AbortController for timeout control)
@@ -214,6 +216,7 @@ export class HttpClient {
                     // 获取重定向URL (Get redirect URL)
                     const location = response.headers.get('location');
                     currentUrl = this.buildRedirectUrl(location, currentUrl, debug);
+                    await validateFetchUrl(currentUrl);
                     // 增加重定向计数 (Increase redirect count)
                     redirectCount++;
                     // 记录重定向 (Log redirect)
@@ -236,6 +239,9 @@ export class HttpClient {
         }
         catch (error) {
             clearTimeout(timeoutId);
+            if (error instanceof UrlValidationError) {
+                throw error;
+            }
             return this.handleRequestError(error, fetchStart, timeout, redirectCount, maxRedirects, debug);
         }
     }

package/dist/src/lib/utils/UrlValidator.js

@@ -0,0 +1,205 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: SSRF protection for outbound fetch requests.
+ */
+import { isIP } from 'net';
+import { lookup } from 'dns/promises';
+const ALLOWED_SCHEMES = new Set(['http:', 'https:']);
+const BLOCKED_HOSTNAMES = new Set([
+    'localhost',
+    'metadata',
+    'metadata.google.internal',
+    'metadata.goog',
+]);
+export class UrlValidationError extends Error {
+    code = 'EBLOCKEDURL';
+    constructor(message) {
+        super(message);
+        this.name = 'UrlValidationError';
+    }
+}
+function ipv4ToInt(ip) {
+    const parts = ip.split('.');
+    if (parts.length !== 4) {
+        return -1;
+    }
+    let value = 0;
+    for (const part of parts) {
+        const octet = Number(part);
+        if (!Number.isInteger(octet) || octet < 0 || octet > 255) {
+            return -1;
+        }
+        value = (value << 8) + octet;
+    }
+    return value >>> 0;
+}
+function isBlockedIPv4(ip) {
+    const value = ipv4ToInt(ip);
+    if (value < 0) {
+        return true;
+    }
+    const firstOctet = value >>> 24;
+    const secondOctet = (value >>> 16) & 0xff;
+    if (firstOctet === 0)
+        return true; // 0.0.0.0/8
+    if (firstOctet === 10)
+        return true; // 10.0.0.0/8
+    if (firstOctet === 127)
+        return true; // 127.0.0.0/8
+    if (firstOctet === 100 && (value & 0xc0000000) === 0x64400000)
+        return true; // 100.64.0.0/10
+    if (firstOctet === 169 && secondOctet === 254)
+        return true; // 169.254.0.0/16
+    if (firstOctet === 172 && secondOctet >= 16 && secondOctet <= 31)
+        return true; // 172.16.0.0/12
+    if (firstOctet === 192 && secondOctet === 168)
+        return true; // 192.168.0.0/16
+    if (firstOctet === 192 && secondOctet === 0)
+        return true; // 192.0.0.0/24
+    if (firstOctet === 198 && (secondOctet === 18 || secondOctet === 19))
+        return true; // 198.18.0.0/15
+    if (firstOctet >= 224)
+        return true; // multicast and reserved ranges
+    return false;
+}
+function expandIPv6(ip) {
+    const lower = ip.toLowerCase();
+    const [head = '', tail = ''] = lower.split('::');
+    const headParts = head ? head.split(':') : [];
+    const tailParts = tail ? tail.split(':') : [];
+    if (headParts.length + tailParts.length > 7) {
+        return null;
+    }
+    const missing = 8 - headParts.length - tailParts.length;
+    const parts = [
+        ...headParts,
+        ...Array.from({ length: missing }, () => '0'),
+        ...tailParts,
+    ];
+    if (parts.length !== 8) {
+        return null;
+    }
+    const groups = [];
+    for (const part of parts) {
+        if (!part) {
+            return null;
+        }
+        const value = Number.parseInt(part, 16);
+        if (!Number.isFinite(value) || value < 0 || value > 0xffff) {
+            return null;
+        }
+        groups.push(value);
+    }
+    return groups;
+}
+function isBlockedIPv6(ip) {
+    const normalized = ip.toLowerCase();
+    if (normalized === '::' || normalized === '::1') {
+        return true;
+    }
+    const ipv4MappedMatch = normalized.match(/^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/);
+    if (ipv4MappedMatch) {
+        return isBlockedIPv4(ipv4MappedMatch[1]);
+    }
+    const groups = expandIPv6(normalized);
+    if (!groups) {
+        return true;
+    }
+    if (groups[0] === 0 &&
+        groups[1] === 0 &&
+        groups[2] === 0 &&
+        groups[3] === 0 &&
+        groups[4] === 0 &&
+        groups[5] === 0xffff) {
+        const mappedValue = (groups[6] << 16) | groups[7];
+        const mappedIpv4 = [
+            (mappedValue >>> 24) & 0xff,
+            (mappedValue >>> 16) & 0xff,
+            (mappedValue >>> 8) & 0xff,
+            mappedValue & 0xff,
+        ].join('.');
+        return isBlockedIPv4(mappedIpv4);
+    }
+    const first = groups[0];
+    if ((first & 0xffc0) === 0xfe80)
+        return true; // fe80::/10
+    if ((first & 0xfe00) === 0xfc00)
+        return true; // fc00::/7
+    if (first === 0x2001 && groups[1] === 0xdb8)
+        return true; // documentation range 2001:db8::/32
+    return false;
+}
+function isBlockedIpAddress(ip) {
+    const version = isIP(ip);
+    if (version === 4) {
+        return isBlockedIPv4(ip);
+    }
+    if (version === 6) {
+        return isBlockedIPv6(ip);
+    }
+    return true;
+}
+function isBlockedHostname(hostname) {
+    const normalized = hostname.toLowerCase().replace(/\.$/, '');
+    if (BLOCKED_HOSTNAMES.has(normalized)) {
+        return true;
+    }
+    return normalized.endsWith('.localhost');
+}
+function parseFetchUrl(urlString) {
+    try {
+        return new URL(urlString);
+    }
+    catch {
+        throw new UrlValidationError(`Invalid URL: ${urlString}`);
+    }
+}
+function validateScheme(url) {
+    if (!ALLOWED_SCHEMES.has(url.protocol)) {
+        throw new UrlValidationError(`Blocked URL scheme: ${url.protocol}`);
+    }
+}
+function normalizeHostname(hostname) {
+    if (hostname.startsWith('[') && hostname.endsWith(']')) {
+        return hostname.slice(1, -1);
+    }
+    return hostname;
+}
+function validateHostnameOrIp(hostname) {
+    const normalizedHostname = normalizeHostname(hostname);
+    const ipVersion = isIP(normalizedHostname);
+    if (ipVersion) {
+        if (isBlockedIpAddress(normalizedHostname)) {
+            throw new UrlValidationError(`Blocked IP address: ${normalizedHostname}`);
+        }
+        return;
+    }
+    if (isBlockedHostname(normalizedHostname)) {
+        throw new UrlValidationError(`Blocked hostname: ${normalizedHostname}`);
+    }
+}
+async function validateResolvedAddresses(hostname) {
+    const normalizedHostname = normalizeHostname(hostname);
+    const addresses = await lookup(normalizedHostname, { all: true, verbatim: true });
+    if (addresses.length === 0) {
+        throw new UrlValidationError(`Unable to resolve hostname: ${normalizedHostname}`);
+    }
+    for (const address of addresses) {
+        if (isBlockedIpAddress(address.address)) {
+            throw new UrlValidationError(`Blocked IP address resolved for ${normalizedHostname}: ${address.address}`);
+        }
+    }
+}
+/**
+ * Validate a URL before making an outbound fetch request.
+ * Blocks private, loopback, link-local, and metadata targets.
+ */
+export async function validateFetchUrl(urlString) {
+    const url = parseFetchUrl(urlString);
+    validateScheme(url);
+    validateHostnameOrIp(url.hostname);
+    if (!isIP(normalizeHostname(url.hostname))) {
+        await validateResolvedAddresses(url.hostname);
+    }
+}

package/dist/tests/fetchers/node/HttpClient.test.js

@@ -0,0 +1,50 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: Tests for HttpClient SSRF protections.
+ */
+import { describe, expect, test, vi, beforeEach } from 'vitest';
+import fetch from 'node-fetch';
+import { lookup } from 'dns/promises';
+import { HttpClient } from '../../../src/lib/fetchers/node/HttpClient.js';
+import { UrlValidationError } from '../../../src/lib/utils/UrlValidator.js';
+vi.mock('node-fetch', () => ({
+    default: vi.fn(),
+}));
+vi.mock('dns/promises', () => ({
+    lookup: vi.fn(),
+}));
+vi.mock('../../../src/lib/fetchers/common/utils.js', () => ({
+    getRandomUserAgent: vi.fn(() => 'test-agent'),
+    randomDelay: vi.fn(async () => undefined),
+    getSystemProxy: vi.fn(() => undefined),
+}));
+describe('HttpClient SSRF protection', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+        vi.mocked(lookup).mockResolvedValue([{ address: '93.184.216.34', family: 4 }]);
+    });
+    test('blocks direct loopback requests before fetch is called', async () => {
+        await expect(HttpClient.fetchWithRedirects({
+            url: 'http://127.0.0.1:8080/latest/meta-data/',
+            useSystemProxy: false,
+        })).rejects.toThrow(UrlValidationError);
+        expect(fetch).not.toHaveBeenCalled();
+    });
+    test('blocks redirect targets that point to private addresses', async () => {
+        vi.mocked(fetch).mockResolvedValueOnce({
+            status: 302,
+            statusText: 'Found',
+            headers: {
+                has: (name) => name.toLowerCase() === 'location',
+                get: () => 'http://127.0.0.1:8080/internal',
+            },
+            text: async () => '',
+        });
+        await expect(HttpClient.fetchWithRedirects({
+            url: 'https://example.com/redirect',
+            useSystemProxy: false,
+        })).rejects.toThrow(/Blocked IP address/);
+        expect(fetch).toHaveBeenCalledTimes(1);
+    });
+});

package/dist/tests/utils/UrlValidator.test.js

@@ -0,0 +1,58 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: Tests for SSRF URL validation.
+ */
+import { describe, expect, test, vi, beforeEach } from 'vitest';
+import { lookup } from 'dns/promises';
+import { validateFetchUrl, UrlValidationError } from '../../src/lib/utils/UrlValidator.js';
+vi.mock('dns/promises', () => ({
+    lookup: vi.fn(),
+}));
+describe('UrlValidator', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+        vi.mocked(lookup).mockResolvedValue([{ address: '93.184.216.34', family: 4 }]);
+    });
+    test('allows public https URLs after DNS resolution', async () => {
+        await expect(validateFetchUrl('https://example.com/path')).resolves.toBeUndefined();
+        expect(lookup).toHaveBeenCalledWith('example.com', { all: true, verbatim: true });
+    });
+    test('allows public http URLs', async () => {
+        await expect(validateFetchUrl('http://example.org')).resolves.toBeUndefined();
+    });
+    test('blocks non-http schemes', async () => {
+        await expect(validateFetchUrl('file:///etc/passwd')).rejects.toThrow(UrlValidationError);
+        await expect(validateFetchUrl('ftp://example.com')).rejects.toThrow(UrlValidationError);
+    });
+    test('blocks loopback IPv4 addresses', async () => {
+        await expect(validateFetchUrl('http://127.0.0.1/')).rejects.toThrow(/Blocked IP address/);
+        await expect(validateFetchUrl('http://127.0.0.1:8080/latest/meta-data/')).rejects.toThrow(UrlValidationError);
+    });
+    test('blocks cloud metadata IP addresses', async () => {
+        await expect(validateFetchUrl('http://169.254.169.254/latest/meta-data/')).rejects.toThrow(/Blocked IP address/);
+    });
+    test('blocks private IPv4 ranges', async () => {
+        await expect(validateFetchUrl('http://10.0.0.1/')).rejects.toThrow(UrlValidationError);
+        await expect(validateFetchUrl('http://192.168.1.1/')).rejects.toThrow(UrlValidationError);
+        await expect(validateFetchUrl('http://172.16.0.1/')).rejects.toThrow(UrlValidationError);
+    });
+    test('blocks localhost hostnames without DNS lookup', async () => {
+        await expect(validateFetchUrl('http://localhost/')).rejects.toThrow(/Blocked hostname/);
+        expect(lookup).not.toHaveBeenCalled();
+    });
+    test('blocks hostnames that resolve to private addresses', async () => {
+        vi.mocked(lookup).mockResolvedValue([{ address: '127.0.0.1', family: 4 }]);
+        await expect(validateFetchUrl('http://rebind.example/')).rejects.toThrow(/Blocked IP address resolved/);
+    });
+    test('blocks IPv6 loopback and link-local addresses', async () => {
+        await expect(validateFetchUrl('http://[::1]/')).rejects.toThrow(UrlValidationError);
+        await expect(validateFetchUrl('http://[fe80::1]/')).rejects.toThrow(UrlValidationError);
+    });
+    test('blocks IPv4-mapped loopback addresses', async () => {
+        await expect(validateFetchUrl('http://[::ffff:127.0.0.1]/')).rejects.toThrow(UrlValidationError);
+    });
+    test('rejects invalid URLs', async () => {
+        await expect(validateFetchUrl('not-a-url')).rejects.toThrow(/Invalid URL/);
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lmcc-dev/mult-fetch-mcp-server",
-  "version": "1.3.2",
+  "version": "1.3.3",
   "description": "An MCP protocol-based web content fetching tool that supports multiple modes and formats, can be integrated with AI assistants like Claude",
   "type": "module",
   "main": "dist/index.js",
@@ -20,32 +20,6 @@
   "engines": {
     "node": ">=18"
   },
-  "scripts": {
-    "build": "tsc && shx chmod +x dist/src/*.js dist/tests/*.js || true && shx chmod +x dist/*.js || true",
-    "prepare": "pnpm run build",
-    "dev": "tsc --watch",
-    "start": "node dist/index.js",
-    "test:mcp": "node dist/tests/test-mcp.js",
-    "test:mini4k": "node dist/tests/test-mini4k.js",
-    "test:direct": "node dist/tests/test-direct-client.js",
-    "test:methods": "node dist/tests/test-mcp-methods.js",
-    "test:i18n": "pnpm run build && node dist/tests/test-i18n.js",
-    "test:i18n:unused": "pnpm run build && node dist/tests/i18n-unused-keys.js",
-    "test:i18n:missing": "pnpm run build && node --loader ts-node/esm tests/i18n-missing-keys.ts",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "test:coverage": "vitest run --coverage",
-    "client": "node dist/src/client.js",
-    "server": "node dist/index.js",
-    "prepublishOnly": "pnpm run build",
-    "release": "pnpm publish --access public",
-    "codecov": "vitest run --coverage && codecov",
-    "test:ci": "vitest run --coverage && codecov",
-    "lint": "eslint . --ext .ts",
-    "lint:fix": "eslint . --ext .ts --fix --max-warnings=0",
-    "lint:src": "eslint src --ext .ts",
-    "lint:src:fix": "eslint src --ext .ts --fix"
-  },
   "keywords": [
     "mcp",
     "model-context-protocol",
@@ -104,5 +78,29 @@
     "README.zh.md",
     "LICENSE",
     "images"
-  ]
+  ],
+  "scripts": {
+    "build": "tsc && shx chmod +x dist/src/*.js dist/tests/*.js || true && shx chmod +x dist/*.js || true",
+    "dev": "tsc --watch",
+    "start": "node dist/index.js",
+    "test:mcp": "node dist/tests/test-mcp.js",
+    "test:mini4k": "node dist/tests/test-mini4k.js",
+    "test:direct": "node dist/tests/test-direct-client.js",
+    "test:methods": "node dist/tests/test-mcp-methods.js",
+    "test:i18n": "pnpm run build && node dist/tests/test-i18n.js",
+    "test:i18n:unused": "pnpm run build && node dist/tests/i18n-unused-keys.js",
+    "test:i18n:missing": "pnpm run build && node --loader ts-node/esm tests/i18n-missing-keys.ts",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "client": "node dist/src/client.js",
+    "server": "node dist/index.js",
+    "release": "pnpm publish --access public",
+    "codecov": "vitest run --coverage && codecov",
+    "test:ci": "vitest run --coverage && codecov",
+    "lint": "eslint . --ext .ts",
+    "lint:fix": "eslint . --ext .ts --fix --max-warnings=0",
+    "lint:src": "eslint src --ext .ts",
+    "lint:src:fix": "eslint src --ext .ts --fix"
+  }
 }