@lmcc-dev/mult-fetch-mcp-server 1.3.1 → 1.3.2

package/dist/src/lib/server/tools.js

@@ -9,6 +9,128 @@ import { fetchWithAutoDetect } from './fetcher.js';
 import { closeBrowserInstance } from './browser.js';
 import { BaseFetcher } from '../fetchers/common/BaseFetcher.js';
 import { TemplateUtils } from '../utils/TemplateUtils.js';
+/**
+ * 基本请求参数定义 (Basic request parameters definition)
+ * 所有工具共享的基本参数 (Basic parameters shared by all tools)
+ */
+const baseRequestParams = {
+    url: {
+        type: "string",
+        description: "URL of the website to fetch",
+    },
+    startCursor: {
+        type: "number",
+        description: "Starting cursor position in bytes. Set to 0 for initial requests, and use the value from previous responses for subsequent requests to resume content retrieval.",
+    },
+    headers: {
+        type: "object",
+        description: "Optional headers to include in the request",
+    },
+    proxy: {
+        type: "string",
+        description: "Optional proxy server to use (format: http://host:port or https://host:port)",
+    },
+    timeout: {
+        type: "number",
+        description: "Optional timeout in milliseconds (default: 30000)",
+    },
+    maxRedirects: {
+        type: "number",
+        description: "Optional maximum number of redirects to follow (default: 10)",
+    },
+    useSystemProxy: {
+        type: "boolean",
+        description: "Optional flag to use system proxy environment variables (default: true)",
+    },
+    debug: {
+        type: "boolean",
+        description: "Optional flag to enable detailed debug logging (default: false)",
+    },
+    noDelay: {
+        type: "boolean",
+        description: "Optional flag to disable random delay between requests (default: false)",
+    }
+};
+/**
+ * 浏览器相关参数定义 (Browser-related parameters definition)
+ * 与浏览器模式相关的参数 (Parameters related to browser mode)
+ */
+const browserParams = {
+    useBrowser: {
+        type: "boolean",
+        description: "Optional flag to use headless browser for fetching (default: false)",
+    },
+    waitForSelector: {
+        type: "string",
+        description: "Optional CSS selector to wait for when using browser mode",
+    },
+    waitForTimeout: {
+        type: "number",
+        description: "Optional timeout to wait after page load in browser mode (default: 5000)",
+    },
+    scrollToBottom: {
+        type: "boolean",
+        description: "Optional flag to scroll to bottom of page in browser mode (default: false)",
+    },
+    closeBrowser: {
+        type: "boolean",
+        description: "Optional flag to close the browser after fetching (default: false)",
+    },
+    saveCookies: {
+        type: "boolean",
+        description: "Optional flag to save cookies for future requests to the same domain (default: true)",
+    },
+    autoDetectMode: {
+        type: "boolean",
+        description: "Optional flag to automatically switch to browser mode if standard fetch fails (default: true). Set to false to strictly use the specified mode without automatic switching.",
+    }
+};
+/**
+ * 内容管理参数定义 (Content management parameters definition)
+ * 与内容大小和分块相关的参数 (Parameters related to content size and chunking)
+ */
+const contentManagementParams = {
+    contentSizeLimit: {
+        type: "number",
+        description: "Optional maximum content size in bytes before splitting into chunks (default: 50KB). Set between 20KB-50KB for optimal results. For large content, prefer smaller values (20KB-30KB) to avoid truncation.",
+    },
+    enableContentSplitting: {
+        type: "boolean",
+        description: "Optional flag to enable content splitting for large responses (default: true)",
+    },
+    chunkId: {
+        type: "string",
+        description: `Optional chunk ID for retrieving a specific chunk of content from a previous request. The system adds prompts in the format ${TemplateUtils.SYSTEM_NOTE.START} ... ${TemplateUtils.SYSTEM_NOTE.END} which AI models should ignore when processing the content.`,
+    }
+};
+/**
+ * 内容提取参数定义 (Content extraction parameters definition)
+ * 与内容提取相关的参数 (Parameters related to content extraction)
+ */
+const contentExtractionParams = {
+    extractContent: {
+        type: "boolean",
+        description: "Optional flag to enable intelligent content extraction using Readability algorithm (default: false). Extracts main article content from web pages."
+    },
+    includeMetadata: {
+        type: "boolean",
+        description: "Optional flag to include metadata (title, author, etc.) in the extracted content (default: false). Only works when extractContent is true."
+    },
+    fallbackToOriginal: {
+        type: "boolean",
+        description: "Optional flag to fall back to the original content when extraction fails (default: true). Only works when extractContent is true."
+    }
+};
+/**
+ * 合并参数对象 (Merge parameter objects)
+ * @param objects 要合并的参数对象数组 (Array of parameter objects to merge)
+ * @returns 合并后的参数对象 (Merged parameter object)
+ */
+function mergeParams(...objects) {
+    return objects.reduce((result, current) => {
+        return { ...result, ...current };
+    }, {});
+}
 /**
  * 获取HTML获取工具定义 (Get HTML fetch tool definition)
  * @returns HTML获取工具定义 (HTML fetch tool definition)
@@ -19,84 +141,7 @@ function getHtmlFetchToolDefinition() {
         description: "Fetch a website and return the content as HTML. Best practices: 1) Always set startCursor=0 for initial requests, and use the fetchedBytes value from previous response for subsequent requests to ensure content continuity. 2) Set contentSizeLimit between 20000-50000 for large pages. 3) When handling large content, use the chunking system by following the startCursor instructions in the system notes rather than increasing contentSizeLimit. 4) If content retrieval fails, you can retry using the same chunkId and startCursor, or adjust startCursor as needed but you must handle any resulting data duplication or gaps yourself. 5) Always explain to users when content is chunked and ask if they want to continue retrieving subsequent parts.",
         inputSchema: {
             type: "object",
-            properties: {
-                url: {
-                    type: "string",
-                    description: "URL of the website to fetch",
-                },
-                startCursor: {
-                    type: "number",
-                    description: "Starting cursor position in bytes. Set to 0 for initial requests, and use the value from previous responses for subsequent requests to resume content retrieval.",
-                },
-                headers: {
-                    type: "object",
-                    description: "Optional headers to include in the request",
-                },
-                proxy: {
-                    type: "string",
-                    description: "Optional proxy server to use (format: http://host:port or https://host:port)",
-                },
-                noDelay: {
-                    type: "boolean",
-                    description: "Optional flag to disable random delay between requests (default: false)",
-                },
-                timeout: {
-                    type: "number",
-                    description: "Optional timeout in milliseconds (default: 30000)",
-                },
-                maxRedirects: {
-                    type: "number",
-                    description: "Optional maximum number of redirects to follow (default: 10)",
-                },
-                useSystemProxy: {
-                    type: "boolean",
-                    description: "Optional flag to use system proxy environment variables (default: true)",
-                },
-                debug: {
-                    type: "boolean",
-                    description: "Optional flag to enable detailed debug logging (default: false)",
-                },
-                useBrowser: {
-                    type: "boolean",
-                    description: "Optional flag to use headless browser for fetching (default: false)",
-                },
-                autoDetectMode: {
-                    type: "boolean",
-                    description: "Optional flag to automatically switch to browser mode if standard fetch fails (default: true). Set to false to strictly use the specified mode without automatic switching.",
-                },
-                waitForSelector: {
-                    type: "string",
-                    description: "Optional CSS selector to wait for when using browser mode",
-                },
-                waitForTimeout: {
-                    type: "number",
-                    description: "Optional timeout to wait after page load in browser mode (default: 5000)",
-                },
-                scrollToBottom: {
-                    type: "boolean",
-                    description: "Optional flag to scroll to bottom of page in browser mode (default: false)",
-                },
-                saveCookies: {
-                    type: "boolean",
-                    description: "Optional flag to save cookies for future requests to the same domain (default: true)",
-                },
-                closeBrowser: {
-                    type: "boolean",
-                    description: "Optional flag to close the browser after fetching (default: false)",
-                },
-                contentSizeLimit: {
-                    type: "number",
-                    description: "Optional maximum content size in bytes before splitting into chunks (default: 50KB). Set between 20KB-50KB for optimal results. For large content, prefer smaller values (20KB-30KB) to avoid truncation.",
-                },
-                enableContentSplitting: {
-                    type: "boolean",
-                    description: "Optional flag to enable content splitting for large responses (default: true)",
-                },
-                chunkId: {
-                    type: "string",
-                    description: `Optional chunk ID for retrieving a specific chunk of content from a previous request. The system adds prompts in the format ${TemplateUtils.SYSTEM_NOTE.START} ... ${TemplateUtils.SYSTEM_NOTE.END} which AI models should ignore when processing the content.`,
-                },
-            },
+            properties: mergeParams(baseRequestParams, browserParams, contentManagementParams, contentExtractionParams),
             required: ["url", "startCursor"],
         },
     };
@@ -111,72 +156,7 @@ function getJsonFetchToolDefinition() {
         description: "Fetch a JSON file from a URL. Best practices: 1) Always set startCursor=0 for initial requests, and use the fetchedBytes value from previous response for subsequent requests to ensure content continuity. 2) Set contentSizeLimit between 20000-50000 for large files. 3) When handling large content, use the chunking system by following the startCursor instructions in the system notes rather than increasing contentSizeLimit. 4) If content retrieval fails, you can retry using the same chunkId and startCursor, or adjust startCursor as needed but you must handle any resulting data duplication or gaps yourself. 5) Always explain to users when content is chunked and ask if they want to continue retrieving subsequent parts.",
         inputSchema: {
             type: "object",
-            properties: {
-                url: {
-                    type: "string",
-                    description: "URL of the JSON to fetch",
-                },
-                startCursor: {
-                    type: "number",
-                    description: "Starting cursor position in bytes. Set to 0 for initial requests, and use the value from previous responses for subsequent requests to resume content retrieval.",
-                },
-                headers: {
-                    type: "object",
-                    description: "Optional headers to include in the request",
-                },
-                proxy: {
-                    type: "string",
-                    description: "Optional proxy server to use (format: http://host:port or https://host:port)",
-                },
-                timeout: {
-                    type: "number",
-                    description: "Optional timeout in milliseconds (default: 30000)",
-                },
-                maxRedirects: {
-                    type: "number",
-                    description: "Optional maximum number of redirects to follow (default: 10)",
-                },
-                useSystemProxy: {
-                    type: "boolean",
-                    description: "Optional flag to use system proxy environment variables (default: true)",
-                },
-                debug: {
-                    type: "boolean",
-                    description: "Optional flag to enable detailed debug logging (default: false)",
-                },
-                noDelay: {
-                    type: "boolean",
-                    description: "Optional flag to disable random delay between requests (default: false)",
-                },
-                useBrowser: {
-                    type: "boolean",
-                    description: "Optional flag to use headless browser for fetching (default: false)",
-                },
-                waitForSelector: {
-                    type: "string",
-                    description: "Optional CSS selector to wait for when using browser mode",
-                },
-                waitForTimeout: {
-                    type: "number",
-                    description: "Optional timeout to wait after page load in browser mode (default: 5000)",
-                },
-                closeBrowser: {
-                    type: "boolean",
-                    description: "Optional flag to close the browser after fetching (default: false)",
-                },
-                contentSizeLimit: {
-                    type: "number",
-                    description: "Optional maximum content size in bytes before splitting into chunks (default: 50KB). Set between 20KB-50KB for optimal results. For large content, prefer smaller values (20KB-30KB) to avoid truncation.",
-                },
-                enableContentSplitting: {
-                    type: "boolean",
-                    description: "Optional flag to enable content splitting for large responses (default: true)",
-                },
-                chunkId: {
-                    type: "string",
-                    description: `Optional chunk ID for retrieving a specific chunk of content from a previous request. The system adds prompts in the format ${TemplateUtils.SYSTEM_NOTE.START} ... ${TemplateUtils.SYSTEM_NOTE.END} which AI models should ignore when processing the content.`,
-                },
-            },
+            properties: mergeParams(baseRequestParams, browserParams, contentManagementParams),
             required: ["url", "startCursor"],
         },
     };
@@ -191,72 +171,7 @@ function getTextFetchToolDefinition() {
         description: "Fetch a website, return the content as plain text (no HTML). Best practices: 1) Always set startCursor=0 for initial requests, and use the fetchedBytes value from previous response for subsequent requests to ensure content continuity. 2) Set contentSizeLimit between 20000-50000 for large pages. 3) When handling large content, use the chunking system by following the startCursor instructions in the system notes rather than increasing contentSizeLimit. 4) If content retrieval fails, you can retry using the same chunkId and startCursor, or adjust startCursor as needed but you must handle any resulting data duplication or gaps yourself. 5) Always explain to users when content is chunked and ask if they want to continue retrieving subsequent parts.",
         inputSchema: {
             type: "object",
-            properties: {
-                url: {
-                    type: "string",
-                    description: "URL of the website to fetch",
-                },
-                startCursor: {
-                    type: "number",
-                    description: "Starting cursor position in bytes. Set to 0 for initial requests, and use the value from previous responses for subsequent requests to resume content retrieval.",
-                },
-                headers: {
-                    type: "object",
-                    description: "Optional headers to include in the request",
-                },
-                proxy: {
-                    type: "string",
-                    description: "Optional proxy server to use (format: http://host:port or https://host:port)",
-                },
-                timeout: {
-                    type: "number",
-                    description: "Optional timeout in milliseconds (default: 30000)",
-                },
-                maxRedirects: {
-                    type: "number",
-                    description: "Optional maximum number of redirects to follow (default: 10)",
-                },
-                useSystemProxy: {
-                    type: "boolean",
-                    description: "Optional flag to use system proxy environment variables (default: true)",
-                },
-                debug: {
-                    type: "boolean",
-                    description: "Optional flag to enable detailed debug logging (default: false)",
-                },
-                noDelay: {
-                    type: "boolean",
-                    description: "Optional flag to disable random delay between requests (default: false)",
-                },
-                useBrowser: {
-                    type: "boolean",
-                    description: "Optional flag to use headless browser for fetching (default: false)",
-                },
-                waitForSelector: {
-                    type: "string",
-                    description: "Optional CSS selector to wait for when using browser mode",
-                },
-                waitForTimeout: {
-                    type: "number",
-                    description: "Optional timeout to wait after page load in browser mode (default: 5000)",
-                },
-                scrollToBottom: {
-                    type: "boolean",
-                    description: "Optional flag to scroll to bottom of page in browser mode (default: false)",
-                },
-                contentSizeLimit: {
-                    type: "number",
-                    description: "Optional maximum content size in bytes before splitting into chunks (default: 50KB). Set between 20KB-50KB for optimal results. For large content, prefer smaller values (20KB-30KB) to avoid truncation.",
-                },
-                enableContentSplitting: {
-                    type: "boolean",
-                    description: "Optional flag to enable content splitting for large responses (default: true)",
-                },
-                chunkId: {
-                    type: "string",
-                    description: `Optional chunk ID for retrieving a specific chunk of content from a previous request. The system adds prompts in the format ${TemplateUtils.SYSTEM_NOTE.START} ... ${TemplateUtils.SYSTEM_NOTE.END} which AI models should ignore when processing the content.`,
-                },
-            },
+            properties: mergeParams(baseRequestParams, browserParams, contentManagementParams, contentExtractionParams),
             required: ["url", "startCursor"],
         },
     };
@@ -271,72 +186,7 @@ function getMarkdownFetchToolDefinition() {
         description: "Fetch a website and return the content as Markdown. Best practices: 1) Always set startCursor=0 for initial requests, and use the fetchedBytes value from previous response for subsequent requests to ensure content continuity. 2) Set contentSizeLimit between 20000-50000 for large pages. 3) When handling large content, use the chunking system by following the startCursor instructions in the system notes rather than increasing contentSizeLimit. 4) If content retrieval fails, you can retry using the same chunkId and startCursor, or adjust startCursor as needed but you must handle any resulting data duplication or gaps yourself. 5) Always explain to users when content is chunked and ask if they want to continue retrieving subsequent parts.",
         inputSchema: {
             type: "object",
-            properties: {
-                url: {
-                    type: "string",
-                    description: "URL of the website to fetch",
-                },
-                startCursor: {
-                    type: "number",
-                    description: "Starting cursor position in bytes. Set to 0 for initial requests, and use the value from previous responses for subsequent requests to resume content retrieval.",
-                },
-                headers: {
-                    type: "object",
-                    description: "Optional headers to include in the request",
-                },
-                proxy: {
-                    type: "string",
-                    description: "Optional proxy server to use (format: http://host:port or https://host:port)",
-                },
-                timeout: {
-                    type: "number",
-                    description: "Optional timeout in milliseconds (default: 30000)",
-                },
-                maxRedirects: {
-                    type: "number",
-                    description: "Optional maximum number of redirects to follow (default: 10)",
-                },
-                useSystemProxy: {
-                    type: "boolean",
-                    description: "Optional flag to use system proxy environment variables (default: true)",
-                },
-                debug: {
-                    type: "boolean",
-                    description: "Optional flag to enable detailed debug logging (default: false)",
-                },
-                noDelay: {
-                    type: "boolean",
-                    description: "Optional flag to disable random delay between requests (default: false)",
-                },
-                useBrowser: {
-                    type: "boolean",
-                    description: "Optional flag to use headless browser for fetching (default: false)",
-                },
-                waitForSelector: {
-                    type: "string",
-                    description: "Optional CSS selector to wait for when using browser mode",
-                },
-                waitForTimeout: {
-                    type: "number",
-                    description: "Optional timeout to wait after page load in browser mode (default: 5000)",
-                },
-                scrollToBottom: {
-                    type: "boolean",
-                    description: "Optional flag to scroll to bottom of page in browser mode (default: false)",
-                },
-                contentSizeLimit: {
-                    type: "number",
-                    description: "Optional maximum content size in bytes before splitting into chunks (default: 50KB). Set between 20KB-50KB for optimal results. For large content, prefer smaller values (20KB-30KB) to avoid truncation.",
-                },
-                enableContentSplitting: {
-                    type: "boolean",
-                    description: "Optional flag to enable content splitting for large responses (default: true)",
-                },
-                chunkId: {
-                    type: "string",
-                    description: `Optional chunk ID for retrieving a specific chunk of content from a previous request. The system adds prompts in the format ${TemplateUtils.SYSTEM_NOTE.START} ... ${TemplateUtils.SYSTEM_NOTE.END} which AI models should ignore when processing the content.`,
-                },
-            },
+            properties: mergeParams(baseRequestParams, browserParams, contentManagementParams, contentExtractionParams),
             required: ["url", "startCursor"],
         },
     };
@@ -351,84 +201,7 @@ function getPlainTextFetchToolDefinition() {
         description: "Fetch a website and return the content as plain text with HTML tags removed. Best practices: 1) Always set startCursor=0 for initial requests, and use the fetchedBytes value from previous response for subsequent requests to ensure content continuity. 2) Set contentSizeLimit between 20000-50000 for large pages. 3) When handling large content, use the chunking system by following the startCursor instructions in the system notes rather than increasing contentSizeLimit. 4) If content retrieval fails, you can retry using the same chunkId and startCursor, or adjust startCursor as needed but you must handle any resulting data duplication or gaps yourself. 5) Always explain to users when content is chunked and ask if they want to continue retrieving subsequent parts.",
         inputSchema: {
             type: "object",
-            properties: {
-                url: {
-                    type: "string",
-                    description: "URL of the website to fetch",
-                },
-                startCursor: {
-                    type: "number",
-                    description: "Starting cursor position in bytes. Set to 0 for initial requests, and use the value from previous responses for subsequent requests to resume content retrieval.",
-                },
-                headers: {
-                    type: "object",
-                    description: "Optional headers to include in the request",
-                },
-                proxy: {
-                    type: "string",
-                    description: "Optional proxy server to use (format: http://host:port or https://host:port)",
-                },
-                noDelay: {
-                    type: "boolean",
-                    description: "Optional flag to disable random delay between requests (default: false)",
-                },
-                timeout: {
-                    type: "number",
-                    description: "Optional timeout in milliseconds (default: 30000)",
-                },
-                maxRedirects: {
-                    type: "number",
-                    description: "Optional maximum number of redirects to follow (default: 10)",
-                },
-                useSystemProxy: {
-                    type: "boolean",
-                    description: "Optional flag to use system proxy environment variables (default: true)",
-                },
-                debug: {
-                    type: "boolean",
-                    description: "Optional flag to enable detailed debug logging (default: false)",
-                },
-                useBrowser: {
-                    type: "boolean",
-                    description: "Optional flag to use headless browser for fetching (default: false)",
-                },
-                autoDetectMode: {
-                    type: "boolean",
-                    description: "Optional flag to automatically switch to browser mode if standard fetch fails (default: true). Set to false to strictly use the specified mode without automatic switching.",
-                },
-                waitForSelector: {
-                    type: "string",
-                    description: "Optional CSS selector to wait for when using browser mode",
-                },
-                waitForTimeout: {
-                    type: "number",
-                    description: "Optional timeout to wait after page load in browser mode (default: 5000)",
-                },
-                scrollToBottom: {
-                    type: "boolean",
-                    description: "Optional flag to scroll to bottom of page in browser mode (default: false)",
-                },
-                saveCookies: {
-                    type: "boolean",
-                    description: "Optional flag to save cookies for future requests to the same domain (default: true)",
-                },
-                closeBrowser: {
-                    type: "boolean",
-                    description: "Optional flag to close the browser after fetching (default: false)",
-                },
-                contentSizeLimit: {
-                    type: "number",
-                    description: "Optional maximum content size in bytes before splitting into chunks (default: 50KB). Set between 20KB-50KB for optimal results. For large content, prefer smaller values (20KB-30KB) to avoid truncation.",
-                },
-                enableContentSplitting: {
-                    type: "boolean",
-                    description: "Optional flag to enable content splitting for large responses (default: true)",
-                },
-                chunkId: {
-                    type: "string",
-                    description: `Optional chunk ID for retrieving a specific chunk of content from a previous request. The system adds prompts in the format ${TemplateUtils.SYSTEM_NOTE.START} ... ${TemplateUtils.SYSTEM_NOTE.END} which AI models should ignore when processing the content.`,
-                },
-            },
+            properties: mergeParams(baseRequestParams, browserParams, contentManagementParams, contentExtractionParams),
             required: ["url", "startCursor"],
         },
     };

package/dist/src/lib/utils/ChunkManager.js

@@ -92,7 +92,7 @@ export class ChunkManager {
         // 计算当前位置和每个分块的起始位置 (Calculate current position and start position of each chunk)
         let currentPosition = 0;
         let chunkIndex = 0;
-        let fetchedSoFar = 0;
+        let _fetchedSoFar = 0;
         let resultContent = '';
         // 找到开始位置对应的分块 (Find the chunk corresponding to the start position)
         for (let i = 0; i < fetchedBytes.length; i++) {
@@ -100,7 +100,7 @@ export class ChunkManager {
                 chunkIndex = i;
                 break;
             }
-            fetchedSoFar += fetchedBytes[i];
+            _fetchedSoFar += fetchedBytes[i];
             currentPosition += fetchedBytes[i];
         }
         // 从找到的分块开始，读取指定大小的内容 (Start reading from the found chunk, up to the specified size)