@lmcc-dev/mult-fetch-mcp-server 1.3.1 → 1.3.3

package/dist/src/lib/utils/ContentExtractor.js

@@ -0,0 +1,141 @@
+/*
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { Readability } from '@mozilla/readability';
+import { JSDOM } from 'jsdom';
+import { log, COMPONENTS } from '../logger.js';
+/**
+ * 内容提取器类 (Content extractor class)
+ * 提供智能内容提取功能，基于Mozilla的Readability库 (Provides intelligent content extraction based on Mozilla's Readability library)
+ */
+export class ContentExtractor {
+    /**
+     * 提取HTML中的主要内容 (Extract main content from HTML)
+     * @param html HTML内容 (HTML content)
+     * @param url 页面URL，用于处理相对路径 (Page URL, used for handling relative paths)
+     * @param debug 是否开启调试 (Whether to enable debugging)
+     * @returns 提取的内容对象，包括标题、内容、文本等 (Extracted content object, including title, content, text, etc.)
+     */
+    static extractContent(html, url, debug = false) {
+        log('extractor.creating_jsdom', debug, { url }, COMPONENTS.EXTRACTOR);
+        try {
+            // 创建JSDOM文档 (Create JSDOM document)
+            const doc = new JSDOM(html, { url });
+            // 检查页面是否适合进行可读性提取 (Check if the page is suitable for readability extraction)
+            const isReaderable = this.isProbablyReaderable(doc.window.document, debug);
+            // 如果页面不适合提取，返回空结果 (If the page is not suitable for extraction, return empty result)
+            if (!isReaderable && debug) {
+                log('extractor.page_not_readerable', true, { url }, COMPONENTS.EXTRACTOR);
+            }
+            // 即使页面可能不适合提取，我们也尝试进行提取 (Even if the page may not be suitable for extraction, we try to extract anyway)
+            // 创建Readability解析器 (Create Readability parser)
+            log('extractor.creating_reader', debug, {}, COMPONENTS.EXTRACTOR);
+            const reader = new Readability(doc.window.document);
+            // 解析内容 (Parse content)
+            log('extractor.parsing_content', debug, {}, COMPONENTS.EXTRACTOR);
+            const article = reader.parse();
+            // 如果解析失败，返回空结果 (If parsing failed, return empty result)
+            if (!article) {
+                log('extractor.parsing_failed', debug, { url }, COMPONENTS.EXTRACTOR);
+                return {
+                    title: null,
+                    content: null,
+                    textContent: null,
+                    excerpt: null,
+                    byline: null,
+                    siteName: null,
+                    length: 0,
+                    isReaderable
+                };
+            }
+            log('extractor.parsing_success', debug, {
+                title: article.title,
+                contentLength: article.content?.length || 0,
+                textLength: article.textContent?.length || 0
+            }, COMPONENTS.EXTRACTOR);
+            // 返回解析结果 (Return parsing result)
+            return {
+                title: article.title,
+                content: article.content,
+                textContent: article.textContent,
+                excerpt: article.excerpt,
+                byline: article.byline,
+                siteName: article.siteName,
+                length: article.length,
+                isReaderable
+            };
+        }
+        catch (error) {
+            // 处理错误 (Handle error)
+            log('extractor.extraction_error', true, {
+                error: error instanceof Error ? error.message : String(error),
+                url
+            }, COMPONENTS.EXTRACTOR);
+            return {
+                title: null,
+                content: null,
+                textContent: null,
+                excerpt: null,
+                byline: null,
+                siteName: null,
+                length: 0,
+                isReaderable: false
+            };
+        }
+    }
+    /**
+     * 检查页面是否适合进行可读性提取 (Check if the page is suitable for readability extraction)
+     * @param document DOM文档 (DOM document)
+     * @param debug 是否开启调试 (Whether to enable debugging)
+     * @returns 是否适合提取 (Whether it is suitable for extraction)
+     */
+    static isProbablyReaderable(document, debug = false) {
+        // 这些是常见的无法提取内容的页面类型 (These are common page types that cannot extract content)
+        const unlikelyPageTypes = [
+            /login/i, /signup/i, /register/i, /404/i, /403/i, /error/i, /captcha/i,
+            /password/i, /forgot/i, /reset/i, /signin/i, /signout/i, /logout/i,
+            /search/i, /contact/i, /about/i, /faq/i, /help/i, /support/i,
+            /dashboard/i, /admin/i, /profile/i, /account/i, /settings/i,
+            /cart/i, /checkout/i, /basket/i, /purchase/i, /payment/i,
+            /calculator/i, /converter/i, /translator/i
+        ];
+        // 检查URL和标题是否包含不太可能包含文章的关键词 (Check if URL and title contain keywords that are unlikely to contain articles)
+        const url = document.location?.href || '';
+        const title = document.title || '';
+        for (const pattern of unlikelyPageTypes) {
+            if (pattern.test(url) || pattern.test(title)) {
+                if (debug) {
+                    log('extractor.unlikely_page_type', debug, { pattern: pattern.toString(), url, title }, COMPONENTS.EXTRACTOR);
+                }
+                return false;
+            }
+        }
+        // 检查是否有文章相关元素 (Check if there are article-related elements)
+        const hasArticleElements = !!document.querySelector('article') ||
+            !!document.querySelector('[role="article"]') ||
+            !!document.querySelector('[itemprop="articleBody"]') ||
+            !!document.querySelector('.post-content') ||
+            !!document.querySelector('.article-content') ||
+            !!document.querySelector('.entry-content');
+        if (hasArticleElements) {
+            if (debug) {
+                log('extractor.has_article_elements', debug, {}, COMPONENTS.EXTRACTOR);
+            }
+            return true;
+        }
+        // 检查是否有足够的段落 (Check if there are enough paragraphs)
+        const paragraphs = document.querySelectorAll('p');
+        if (paragraphs.length >= 5) {
+            if (debug) {
+                log('extractor.has_enough_paragraphs', debug, { count: paragraphs.length }, COMPONENTS.EXTRACTOR);
+            }
+            return true;
+        }
+        if (debug) {
+            log('extractor.not_enough_content', debug, { paragraphs: paragraphs.length }, COMPONENTS.EXTRACTOR);
+        }
+        return false;
+    }
+}

package/dist/src/lib/utils/ContentProcessor.js

@@ -16,10 +16,9 @@ export class ContentProcessor {
      * 将HTML转换为Markdown (Convert HTML to Markdown)
      * @param html HTML内容 (HTML content)
      * @param debug 是否开启调试 (Whether to enable debugging)
-     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
      * @returns Markdown内容 (Markdown content)
      */
-    static htmlToMarkdown(html, debug, component) {
+    static htmlToMarkdown(html, debug) {
         // 始终使用COMPONENTS.PROCESSOR作为组件标识符 (Always use COMPONENTS.PROCESSOR as component identifier)
         log('processor.creatingTurndown', debug, {}, COMPONENTS.PROCESSOR);
         const turndownService = new TurndownService({
@@ -45,10 +44,9 @@ export class ContentProcessor {
      * 将HTML转换为纯文本 (Convert HTML to plain text)
      * @param html HTML内容 (HTML content)
      * @param debug 是否开启调试 (Whether to enable debugging)
-     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
      * @returns 纯文本内容 (Plain text content)
      */
-    static htmlToText(html, debug, component) {
+    static htmlToText(html, debug) {
         // 始终使用COMPONENTS.PROCESSOR作为组件标识符 (Always use COMPONENTS.PROCESSOR as component identifier)
         log('processor.creatingHtmlToText', debug, {}, COMPONENTS.PROCESSOR);
         // 配置html-to-text选项 (Configure html-to-text options)
@@ -70,10 +68,9 @@ export class ContentProcessor {
      * 解析JSON字符串 (Parse JSON string)
      * @param text JSON字符串 (JSON string)
      * @param debug 是否开启调试 (Whether to enable debugging)
-     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
      * @returns 解析结果 (Parse result - success or error with message)
      */
-    static parseJson(text, debug, component) {
+    static parseJson(text, debug) {
         log('processor.parsingJson', debug, {}, COMPONENTS.PROCESSOR);
         try {
             const parsed = JSON.parse(text);
@@ -95,10 +92,9 @@ export class ContentProcessor {
      * 处理文本内容，确保是UTF-8编码 (Process text content, ensure it's UTF-8 encoded)
      * @param text 文本内容 (Text content)
      * @param debug 是否开启调试 (Whether to enable debugging)
-     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
      * @returns 处理后的文本 (Processed text)
      */
-    static processTextContent(text, debug, component) {
+    static processTextContent(text, debug) {
         log('processor.processingText', debug, { length: text.length }, COMPONENTS.PROCESSOR);
         // 这里可以添加文本处理逻辑，如编码转换、去除特殊字符等
         // (Add text processing logic here, such as encoding conversion, removing special characters, etc.)
@@ -109,10 +105,8 @@ export class ContentProcessor {
      * @param content 内容 (Content)
      * @param contentSizeLimit 内容大小限制 (Content size limit)
      * @param debug 是否开启调试 (Whether to enable debugging)
-     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
-     * @throws {ToolError} 如果内容太大且不允许分割 (If content is too large and splitting is not allowed)
      */
-    static validateContentSize(content, contentSizeLimit, debug, component) {
+    static validateContentSize(content, contentSizeLimit, debug) {
         const contentSize = content.length;
         if (contentSize > contentSizeLimit) {
             log('processor.contentTooLarge', debug, {

package/dist/src/lib/utils/ContentSizeManager.js

@@ -89,10 +89,10 @@ export class ContentSizeManager {
      * @param content 原始内容 (Original content)
      * @param sizeLimit 每个片段的大小限制，单位为字节 (Size limit for each chunk in bytes)
      * @param debug 是否启用调试模式 (Whether debug mode is enabled)
-     * @param offset 当前偏移量，用于确定是首次请求还是后续请求 (Current offset, used to determine if it's initial or subsequent request)
+     * @param _offset 当前偏移量，用于确定是首次请求还是后续请求 (Current offset, used to determine if it's initial or subsequent request)
      * @returns 内容片段数组 (Array of content chunks)
      */
-    static splitContentIntoChunks(content, sizeLimit = this.DEFAULT_SIZE_LIMIT, debug = false, offset = 0) {
+    static splitContentIntoChunks(content, sizeLimit = this.DEFAULT_SIZE_LIMIT, debug = false, _offset = 0) {
         // 计算内容总字节大小 (Calculate total size of content in bytes)
         const totalBytes = Buffer.byteLength(content, 'utf8');
         // 使用TemplateUtils中的常量和方法生成示例模板以计算大小

package/dist/src/lib/utils/ErrorHandler.js

@@ -5,6 +5,7 @@
  */
 import { isAccessDeniedError, isNetworkError } from './errorDetection.js';
 import { log, COMPONENTS } from '../logger.js';
+// import { ERROR_KEYS } from '../i18n/keys/errors.js';
 /**
  * 错误类型枚举
  * (Error type enumeration)

package/dist/src/lib/utils/TemplateUtils.js

@@ -44,6 +44,11 @@ export class TemplateUtils {
      * @returns 格式化的提示文本 (Formatted prompt text)
      */
     static generateSizeBasedChunkPrompt(fetchedBytes, totalBytes, chunkId, remainingBytes, estimatedRequests, currentSizeLimit, isFirstRequest = true) {
+        // 如果剩余字节数为0，使用完成提示而不是继续获取的提示
+        // (If remaining bytes is 0, use completion prompt instead of continuation prompt)
+        if (remainingBytes <= 0) {
+            return this.generateSizeBasedLastChunkPrompt(fetchedBytes, totalBytes, isFirstRequest);
+        }
         const prefix = isFirstRequest ? 'Content is too long and has been split. ' : '';
         const fetchedPercent = Math.round((fetchedBytes / totalBytes) * 100);
         return `\n\n${TemplateUtils.SYSTEM_NOTE.START}\n${prefix}You've retrieved ${fetchedBytes.toLocaleString()} bytes (${fetchedPercent}% of total ${totalBytes.toLocaleString()} bytes). ${remainingBytes.toLocaleString()} bytes remaining. With current contentSizeLimit=${currentSizeLimit.toLocaleString()}, approximately ${estimatedRequests} more requests needed to retrieve all content. To continue, use the same tool function with parameters chunkId="${chunkId}" and startCursor=${fetchedBytes}.\n${TemplateUtils.SYSTEM_NOTE.END}`;
@@ -57,8 +62,7 @@ export class TemplateUtils {
      */
     static generateSizeBasedLastChunkPrompt(fetchedBytes, totalBytes, isFirstRequest = true) {
         const prefix = isFirstRequest ? 'Content is too long and has been split. ' : '';
-        const fetchedPercent = Math.round((fetchedBytes / totalBytes) * 100);
-        return `\n\n${TemplateUtils.SYSTEM_NOTE.START}\n${prefix}You've retrieved ${fetchedBytes.toLocaleString()} bytes (100% of total ${totalBytes.toLocaleString()} bytes).\nThis is the last part of the content.\n${TemplateUtils.SYSTEM_NOTE.END}`;
+        return `\n\n${TemplateUtils.SYSTEM_NOTE.START}\n${prefix}You've retrieved ${fetchedBytes.toLocaleString()} bytes (100% of total ${totalBytes.toLocaleString()} bytes).\nThis is the last part of the content. No further requests needed.\n${TemplateUtils.SYSTEM_NOTE.END}`;
     }
     /**
      * 检查内容是否已包含系统提示 (Check if content already contains system prompt)

package/dist/src/lib/utils/UrlValidator.js

@@ -0,0 +1,205 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: SSRF protection for outbound fetch requests.
+ */
+import { isIP } from 'net';
+import { lookup } from 'dns/promises';
+const ALLOWED_SCHEMES = new Set(['http:', 'https:']);
+const BLOCKED_HOSTNAMES = new Set([
+    'localhost',
+    'metadata',
+    'metadata.google.internal',
+    'metadata.goog',
+]);
+export class UrlValidationError extends Error {
+    code = 'EBLOCKEDURL';
+    constructor(message) {
+        super(message);
+        this.name = 'UrlValidationError';
+    }
+}
+function ipv4ToInt(ip) {
+    const parts = ip.split('.');
+    if (parts.length !== 4) {
+        return -1;
+    }
+    let value = 0;
+    for (const part of parts) {
+        const octet = Number(part);
+        if (!Number.isInteger(octet) || octet < 0 || octet > 255) {
+            return -1;
+        }
+        value = (value << 8) + octet;
+    }
+    return value >>> 0;
+}
+function isBlockedIPv4(ip) {
+    const value = ipv4ToInt(ip);
+    if (value < 0) {
+        return true;
+    }
+    const firstOctet = value >>> 24;
+    const secondOctet = (value >>> 16) & 0xff;
+    if (firstOctet === 0)
+        return true; // 0.0.0.0/8
+    if (firstOctet === 10)
+        return true; // 10.0.0.0/8
+    if (firstOctet === 127)
+        return true; // 127.0.0.0/8
+    if (firstOctet === 100 && (value & 0xc0000000) === 0x64400000)
+        return true; // 100.64.0.0/10
+    if (firstOctet === 169 && secondOctet === 254)
+        return true; // 169.254.0.0/16
+    if (firstOctet === 172 && secondOctet >= 16 && secondOctet <= 31)
+        return true; // 172.16.0.0/12
+    if (firstOctet === 192 && secondOctet === 168)
+        return true; // 192.168.0.0/16
+    if (firstOctet === 192 && secondOctet === 0)
+        return true; // 192.0.0.0/24
+    if (firstOctet === 198 && (secondOctet === 18 || secondOctet === 19))
+        return true; // 198.18.0.0/15
+    if (firstOctet >= 224)
+        return true; // multicast and reserved ranges
+    return false;
+}
+function expandIPv6(ip) {
+    const lower = ip.toLowerCase();
+    const [head = '', tail = ''] = lower.split('::');
+    const headParts = head ? head.split(':') : [];
+    const tailParts = tail ? tail.split(':') : [];
+    if (headParts.length + tailParts.length > 7) {
+        return null;
+    }
+    const missing = 8 - headParts.length - tailParts.length;
+    const parts = [
+        ...headParts,
+        ...Array.from({ length: missing }, () => '0'),
+        ...tailParts,
+    ];
+    if (parts.length !== 8) {
+        return null;
+    }
+    const groups = [];
+    for (const part of parts) {
+        if (!part) {
+            return null;
+        }
+        const value = Number.parseInt(part, 16);
+        if (!Number.isFinite(value) || value < 0 || value > 0xffff) {
+            return null;
+        }
+        groups.push(value);
+    }
+    return groups;
+}
+function isBlockedIPv6(ip) {
+    const normalized = ip.toLowerCase();
+    if (normalized === '::' || normalized === '::1') {
+        return true;
+    }
+    const ipv4MappedMatch = normalized.match(/^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/);
+    if (ipv4MappedMatch) {
+        return isBlockedIPv4(ipv4MappedMatch[1]);
+    }
+    const groups = expandIPv6(normalized);
+    if (!groups) {
+        return true;
+    }
+    if (groups[0] === 0 &&
+        groups[1] === 0 &&
+        groups[2] === 0 &&
+        groups[3] === 0 &&
+        groups[4] === 0 &&
+        groups[5] === 0xffff) {
+        const mappedValue = (groups[6] << 16) | groups[7];
+        const mappedIpv4 = [
+            (mappedValue >>> 24) & 0xff,
+            (mappedValue >>> 16) & 0xff,
+            (mappedValue >>> 8) & 0xff,
+            mappedValue & 0xff,
+        ].join('.');
+        return isBlockedIPv4(mappedIpv4);
+    }
+    const first = groups[0];
+    if ((first & 0xffc0) === 0xfe80)
+        return true; // fe80::/10
+    if ((first & 0xfe00) === 0xfc00)
+        return true; // fc00::/7
+    if (first === 0x2001 && groups[1] === 0xdb8)
+        return true; // documentation range 2001:db8::/32
+    return false;
+}
+function isBlockedIpAddress(ip) {
+    const version = isIP(ip);
+    if (version === 4) {
+        return isBlockedIPv4(ip);
+    }
+    if (version === 6) {
+        return isBlockedIPv6(ip);
+    }
+    return true;
+}
+function isBlockedHostname(hostname) {
+    const normalized = hostname.toLowerCase().replace(/\.$/, '');
+    if (BLOCKED_HOSTNAMES.has(normalized)) {
+        return true;
+    }
+    return normalized.endsWith('.localhost');
+}
+function parseFetchUrl(urlString) {
+    try {
+        return new URL(urlString);
+    }
+    catch {
+        throw new UrlValidationError(`Invalid URL: ${urlString}`);
+    }
+}
+function validateScheme(url) {
+    if (!ALLOWED_SCHEMES.has(url.protocol)) {
+        throw new UrlValidationError(`Blocked URL scheme: ${url.protocol}`);
+    }
+}
+function normalizeHostname(hostname) {
+    if (hostname.startsWith('[') && hostname.endsWith(']')) {
+        return hostname.slice(1, -1);
+    }
+    return hostname;
+}
+function validateHostnameOrIp(hostname) {
+    const normalizedHostname = normalizeHostname(hostname);
+    const ipVersion = isIP(normalizedHostname);
+    if (ipVersion) {
+        if (isBlockedIpAddress(normalizedHostname)) {
+            throw new UrlValidationError(`Blocked IP address: ${normalizedHostname}`);
+        }
+        return;
+    }
+    if (isBlockedHostname(normalizedHostname)) {
+        throw new UrlValidationError(`Blocked hostname: ${normalizedHostname}`);
+    }
+}
+async function validateResolvedAddresses(hostname) {
+    const normalizedHostname = normalizeHostname(hostname);
+    const addresses = await lookup(normalizedHostname, { all: true, verbatim: true });
+    if (addresses.length === 0) {
+        throw new UrlValidationError(`Unable to resolve hostname: ${normalizedHostname}`);
+    }
+    for (const address of addresses) {
+        if (isBlockedIpAddress(address.address)) {
+            throw new UrlValidationError(`Blocked IP address resolved for ${normalizedHostname}: ${address.address}`);
+        }
+    }
+}
+/**
+ * Validate a URL before making an outbound fetch request.
+ * Blocks private, loopback, link-local, and metadata targets.
+ */
+export async function validateFetchUrl(urlString) {
+    const url = parseFetchUrl(urlString);
+    validateScheme(url);
+    validateHostnameOrIp(url.hostname);
+    if (!isIP(normalizeHostname(url.hostname))) {
+        await validateResolvedAddresses(url.hostname);
+    }
+}

package/dist/tests/client.test.js

@@ -94,7 +94,7 @@ const mockClientModule = {
             return;
         }
         const method = process.argv[2];
-        let paramsJson = process.argv[3];
+        const paramsJson = process.argv[3];
         try {
             JSON.parse(paramsJson);
         }

package/dist/tests/fetchers/node/HttpClient.test.js

@@ -0,0 +1,50 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: Tests for HttpClient SSRF protections.
+ */
+import { describe, expect, test, vi, beforeEach } from 'vitest';
+import fetch from 'node-fetch';
+import { lookup } from 'dns/promises';
+import { HttpClient } from '../../../src/lib/fetchers/node/HttpClient.js';
+import { UrlValidationError } from '../../../src/lib/utils/UrlValidator.js';
+vi.mock('node-fetch', () => ({
+    default: vi.fn(),
+}));
+vi.mock('dns/promises', () => ({
+    lookup: vi.fn(),
+}));
+vi.mock('../../../src/lib/fetchers/common/utils.js', () => ({
+    getRandomUserAgent: vi.fn(() => 'test-agent'),
+    randomDelay: vi.fn(async () => undefined),
+    getSystemProxy: vi.fn(() => undefined),
+}));
+describe('HttpClient SSRF protection', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+        vi.mocked(lookup).mockResolvedValue([{ address: '93.184.216.34', family: 4 }]);
+    });
+    test('blocks direct loopback requests before fetch is called', async () => {
+        await expect(HttpClient.fetchWithRedirects({
+            url: 'http://127.0.0.1:8080/latest/meta-data/',
+            useSystemProxy: false,
+        })).rejects.toThrow(UrlValidationError);
+        expect(fetch).not.toHaveBeenCalled();
+    });
+    test('blocks redirect targets that point to private addresses', async () => {
+        vi.mocked(fetch).mockResolvedValueOnce({
+            status: 302,
+            statusText: 'Found',
+            headers: {
+                has: (name) => name.toLowerCase() === 'location',
+                get: () => 'http://127.0.0.1:8080/internal',
+            },
+            text: async () => '',
+        });
+        await expect(HttpClient.fetchWithRedirects({
+            url: 'https://example.com/redirect',
+            useSystemProxy: false,
+        })).rejects.toThrow(/Blocked IP address/);
+        expect(fetch).toHaveBeenCalledTimes(1);
+    });
+});