@lmcc-dev/mult-fetch-mcp-server 1.0.1

package/dist/src/lib/server/tools.js

@@ -0,0 +1,346 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+import { log, COMPONENTS } from '../logger.js';
+import { fetchWithAutoDetect } from './fetcher.js';
+/**
+ * 注册工具到服务器 (Register tools to server)
+ * 设置工具列表和处理程序 (Set up tool list and handlers)
+ * @param server MCP服务器实例 (MCP server instance)
+ */
+export function registerTools(server) {
+    // 注册工具列表处理程序
+    server.setRequestHandler(ListToolsRequestSchema, async () => {
+        return {
+            tools: [
+                {
+                    name: "fetch_html",
+                    description: "Fetch a website and return the content as HTML",
+                    inputSchema: {
+                        type: "object",
+                        properties: {
+                            url: {
+                                type: "string",
+                                description: "URL of the website to fetch",
+                            },
+                            headers: {
+                                type: "object",
+                                description: "Optional headers to include in the request",
+                            },
+                            proxy: {
+                                type: "string",
+                                description: "Optional proxy server to use (format: http://host:port or https://host:port)",
+                            },
+                            noDelay: {
+                                type: "boolean",
+                                description: "Optional flag to disable random delay between requests (default: false)",
+                            },
+                            timeout: {
+                                type: "number",
+                                description: "Optional timeout in milliseconds (default: 30000)",
+                            },
+                            maxRedirects: {
+                                type: "number",
+                                description: "Optional maximum number of redirects to follow (default: 10)",
+                            },
+                            useSystemProxy: {
+                                type: "boolean",
+                                description: "Optional flag to use system proxy environment variables (default: true)",
+                            },
+                            debug: {
+                                type: "boolean",
+                                description: "Optional flag to enable detailed debug logging (default: false)",
+                            },
+                            useBrowser: {
+                                type: "boolean",
+                                description: "Optional flag to use headless browser for fetching (default: false)",
+                            },
+                            autoDetectMode: {
+                                type: "boolean",
+                                description: "Optional flag to automatically switch to browser mode if standard fetch fails (default: true)",
+                            },
+                            waitForSelector: {
+                                type: "string",
+                                description: "Optional CSS selector to wait for when using browser mode",
+                            },
+                            waitForTimeout: {
+                                type: "number",
+                                description: "Optional timeout to wait after page load in browser mode (default: 5000)",
+                            },
+                            scrollToBottom: {
+                                type: "boolean",
+                                description: "Optional flag to scroll to bottom of page in browser mode (default: false)",
+                            },
+                            saveCookies: {
+                                type: "boolean",
+                                description: "Optional flag to save cookies for future requests to the same domain (default: true)",
+                            },
+                            closeBrowser: {
+                                type: "boolean",
+                                description: "Optional flag to close the browser after fetching (default: false)",
+                            },
+                        },
+                        required: ["url"],
+                    },
+                },
+                {
+                    name: "fetch_json",
+                    description: "Fetch a JSON file from a URL",
+                    inputSchema: {
+                        type: "object",
+                        properties: {
+                            url: {
+                                type: "string",
+                                description: "URL of the JSON to fetch",
+                            },
+                            headers: {
+                                type: "object",
+                                description: "Optional headers to include in the request",
+                            },
+                            proxy: {
+                                type: "string",
+                                description: "Optional proxy server to use (format: http://host:port or https://host:port)",
+                            },
+                            noDelay: {
+                                type: "boolean",
+                                description: "Optional flag to disable random delay between requests (default: false)",
+                            },
+                            timeout: {
+                                type: "number",
+                                description: "Optional timeout in milliseconds (default: 30000)",
+                            },
+                            maxRedirects: {
+                                type: "number",
+                                description: "Optional maximum number of redirects to follow (default: 10)",
+                            },
+                            useSystemProxy: {
+                                type: "boolean",
+                                description: "Optional flag to use system proxy environment variables (default: true)",
+                            },
+                            debug: {
+                                type: "boolean",
+                                description: "Optional flag to enable detailed debug logging (default: false)",
+                            },
+                            useBrowser: {
+                                type: "boolean",
+                                description: "Optional flag to use headless browser for fetching (default: false)",
+                            },
+                            waitForSelector: {
+                                type: "string",
+                                description: "Optional CSS selector to wait for when using browser mode",
+                            },
+                            waitForTimeout: {
+                                type: "number",
+                                description: "Optional timeout to wait after page load in browser mode (default: 5000)",
+                            },
+                            closeBrowser: {
+                                type: "boolean",
+                                description: "Optional flag to close the browser after fetching (default: false)",
+                            },
+                        },
+                        required: ["url"],
+                    },
+                },
+                {
+                    name: "fetch_txt",
+                    description: "Fetch a website, return the content as plain text (no HTML)",
+                    inputSchema: {
+                        type: "object",
+                        properties: {
+                            url: {
+                                type: "string",
+                                description: "URL of the website to fetch",
+                            },
+                            headers: {
+                                type: "object",
+                                description: "Optional headers to include in the request",
+                            },
+                            proxy: {
+                                type: "string",
+                                description: "Optional proxy server to use (format: http://host:port or https://host:port)",
+                            },
+                            noDelay: {
+                                type: "boolean",
+                                description: "Optional flag to disable random delay between requests (default: false)",
+                            },
+                            timeout: {
+                                type: "number",
+                                description: "Optional timeout in milliseconds (default: 30000)",
+                            },
+                            maxRedirects: {
+                                type: "number",
+                                description: "Optional maximum number of redirects to follow (default: 10)",
+                            },
+                            useSystemProxy: {
+                                type: "boolean",
+                                description: "Optional flag to use system proxy environment variables (default: true)",
+                            },
+                            debug: {
+                                type: "boolean",
+                                description: "Optional flag to enable detailed debug logging (default: false)",
+                            },
+                            useBrowser: {
+                                type: "boolean",
+                                description: "Optional flag to use headless browser for fetching (default: false)",
+                            },
+                            waitForSelector: {
+                                type: "string",
+                                description: "Optional CSS selector to wait for when using browser mode",
+                            },
+                            waitForTimeout: {
+                                type: "number",
+                                description: "Optional timeout to wait after page load in browser mode (default: 5000)",
+                            },
+                            scrollToBottom: {
+                                type: "boolean",
+                                description: "Optional flag to scroll to bottom of page in browser mode (default: false)",
+                            },
+                            closeBrowser: {
+                                type: "boolean",
+                                description: "Optional flag to close the browser after fetching (default: false)",
+                            },
+                        },
+                        required: ["url"],
+                    },
+                },
+                {
+                    name: "fetch_markdown",
+                    description: "Fetch a website and return the content as Markdown",
+                    inputSchema: {
+                        type: "object",
+                        properties: {
+                            url: {
+                                type: "string",
+                                description: "URL of the website to fetch",
+                            },
+                            headers: {
+                                type: "object",
+                                description: "Optional headers to include in the request",
+                            },
+                            proxy: {
+                                type: "string",
+                                description: "Optional proxy server to use (format: http://host:port or https://host:port)",
+                            },
+                            noDelay: {
+                                type: "boolean",
+                                description: "Optional flag to disable random delay between requests (default: false)",
+                            },
+                            timeout: {
+                                type: "number",
+                                description: "Optional timeout in milliseconds (default: 30000)",
+                            },
+                            maxRedirects: {
+                                type: "number",
+                                description: "Optional maximum number of redirects to follow (default: 10)",
+                            },
+                            useSystemProxy: {
+                                type: "boolean",
+                                description: "Optional flag to use system proxy environment variables (default: true)",
+                            },
+                            debug: {
+                                type: "boolean",
+                                description: "Optional flag to enable detailed debug logging (default: false)",
+                            },
+                            useBrowser: {
+                                type: "boolean",
+                                description: "Optional flag to use headless browser for fetching (default: false)",
+                            },
+                            waitForSelector: {
+                                type: "string",
+                                description: "Optional CSS selector to wait for when using browser mode",
+                            },
+                            waitForTimeout: {
+                                type: "number",
+                                description: "Optional timeout to wait after page load in browser mode (default: 5000)",
+                            },
+                            scrollToBottom: {
+                                type: "boolean",
+                                description: "Optional flag to scroll to bottom of page in browser mode (default: false)",
+                            },
+                            closeBrowser: {
+                                type: "boolean",
+                                description: "Optional flag to close the browser after fetching (default: false)",
+                            },
+                        },
+                        required: ["url"],
+                    },
+                },
+            ],
+        };
+    });
+    // 注册工具调用处理程序
+    server.setRequestHandler(CallToolRequestSchema, async (request) => {
+        const name = request.params.name;
+        const parameters = request.params.arguments;
+        const params = parameters;
+        try {
+            // 记录工具请求
+            log('server.receivedToolRequest', params.debug === true, { name, url: params.url }, COMPONENTS.TOOLS);
+            if (name === "fetch_html") {
+                const result = await fetchWithAutoDetect(params, 'html');
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: result.content[0].text
+                        }
+                    ],
+                    isError: result.isError
+                };
+            }
+            else if (name === "fetch_json") {
+                const result = await fetchWithAutoDetect(params, 'json');
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: result.content[0].text
+                        }
+                    ],
+                    isError: result.isError
+                };
+            }
+            else if (name === "fetch_txt") {
+                const result = await fetchWithAutoDetect(params, 'txt');
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: result.content[0].text
+                        }
+                    ],
+                    isError: result.isError
+                };
+            }
+            else if (name === "fetch_markdown") {
+                const result = await fetchWithAutoDetect(params, 'markdown');
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: result.content[0].text
+                        }
+                    ],
+                    isError: result.isError
+                };
+            }
+            throw new Error(`Unknown tool name: ${name}`);
+        }
+        catch (error) {
+            // 记录错误
+            log('server.processingToolRequestError', params.debug === true, { name, error: error instanceof Error ? error.message : String(error) }, COMPONENTS.TOOLS);
+            return {
+                isError: true,
+                content: [
+                    {
+                        type: "text",
+                        text: `Error processing request: ${error instanceof Error ? error.message : String(error)}`
+                    }
+                ]
+            };
+        }
+    });
+}

package/dist/src/lib/server/types.js

@@ -0,0 +1,6 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+export {};

package/dist/src/lib/types.js

@@ -0,0 +1,35 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { z } from "zod";
+// 浏览器特定参数
+export const BrowserParamsSchema = z.object({
+    waitForSelector: z.string().optional(),
+    waitForTimeout: z.number().optional(),
+    scrollToBottom: z.boolean().optional(),
+    saveCookies: z.boolean().optional(),
+    useBrowser: z.boolean().optional(),
+    useNodeFetch: z.boolean().optional(),
+    autoDetectMode: z.boolean().optional(),
+    closeBrowser: z.boolean().optional(),
+});
+export const RequestPayloadSchema = z.object({
+    url: z.string(),
+    method: z.string().optional(),
+    headers: z.record(z.string()).optional(),
+    proxy: z.string().optional(),
+    noDelay: z.boolean().optional(),
+    timeout: z.number().optional(),
+    maxRedirects: z.number().optional(),
+    useSystemProxy: z.boolean().optional(),
+    debug: z.boolean().optional(),
+    useBrowser: z.boolean().optional(),
+    useNodeFetch: z.boolean().optional(),
+    autoDetectMode: z.boolean().optional(),
+    waitForSelector: z.string().optional(),
+    waitForTimeout: z.number().optional(),
+    scrollToBottom: z.boolean().optional(),
+    closeBrowser: z.boolean().optional(),
+}).merge(BrowserParamsSchema);

package/dist/src/mcp-server.js

@@ -0,0 +1,22 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { startServer } from './lib/server/index.js';
+import { log, COMPONENTS } from './lib/logger.js';
+/**
+ * 主函数 (Main function)
+ * 统一处理debug和MCP_LANG的获取方式 (Unified handling of debug and MCP_LANG acquisition)
+ */
+async function main() {
+    // MCP_LANG从环境变量获取，已在i18n/index.ts中处理
+    // debug将从调用参数获取，不需要在这里处理
+    // 启动服务器
+    await startServer();
+}
+// 启动服务器
+main().catch(async (error) => {
+    log('server.startupError', true, { error: error instanceof Error ? error.message : String(error) }, COMPONENTS.SERVER);
+    process.exit(1);
+});

package/dist/tests/test-direct-client.js

@@ -0,0 +1,129 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { spawn } from 'child_process';
+import path from 'path';
+import { fileURLToPath } from 'url';
+// 获取当前文件的目录路径 (Get the directory path of the current file)
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+// 获取项目根目录 (Get the project root directory)
+const rootDir = process.cwd();
+/**
+ * 直接调用客户端脚本进行测试 (Directly call client script for testing)
+ * 不通过MCP协议，直接测试客户端功能 (Test client functionality directly without going through MCP protocol)
+ */
+async function testDirectClient() {
+    console.log('开始直接调用客户端脚本进行测试...');
+    // 测试用例列表
+    const testCases = [
+        {
+            name: 'HTML 获取测试',
+            method: 'fetch_html',
+            params: {
+                url: 'https://example.com',
+                debug: true
+            }
+        },
+        {
+            name: 'JSON 获取测试',
+            method: 'fetch_json',
+            params: {
+                url: 'https://jsonplaceholder.typicode.com/todos/1',
+                debug: true
+            }
+        },
+        {
+            name: 'TXT 获取测试',
+            method: 'fetch_txt',
+            params: {
+                url: 'https://example.com',
+                debug: true
+            }
+        },
+        {
+            name: 'Markdown 获取测试',
+            method: 'fetch_markdown',
+            params: {
+                url: 'https://example.com',
+                debug: true
+            }
+        },
+        {
+            name: '错误处理测试 - 无效URL',
+            method: 'fetch_html',
+            params: {
+                url: 'https://this-domain-does-not-exist-123456789.com',
+                debug: true,
+                timeout: 5000
+            }
+        }
+    ];
+    // 依次执行每个测试用例
+    for (const testCase of testCases) {
+        console.log(`\n执行测试: ${testCase.name}`);
+        // 构建命令行参数
+        const clientPath = path.resolve(rootDir, 'dist/src/client.js');
+        const args = [
+            clientPath,
+            testCase.method,
+            JSON.stringify(testCase.params)
+        ];
+        console.log(`运行命令: node ${args.join(' ')}`);
+        // 执行命令
+        const result = await new Promise((resolve) => {
+            const childProcess = spawn('node', args, {
+                stdio: ['ignore', 'pipe', 'pipe']
+            });
+            let stdout = '';
+            let stderr = '';
+            childProcess.stdout.on('data', (data) => {
+                stdout += data.toString();
+            });
+            childProcess.stderr.on('data', (data) => {
+                stderr += data.toString();
+            });
+            childProcess.on('close', (code) => {
+                resolve({
+                    code: code || 0,
+                    stdout,
+                    stderr
+                });
+            });
+        });
+        // 输出结果
+        console.log(`退出代码: ${result.code}`);
+        if (result.code === 0) {
+            console.log('测试成功!');
+            // 尝试解析JSON结果
+            try {
+                const jsonResult = JSON.parse(result.stdout);
+                console.log('结果类型:', typeof jsonResult);
+                if (typeof jsonResult === 'string') {
+                    console.log('内容长度:', jsonResult.length);
+                    console.log('内容预览:', jsonResult.substring(0, 200) + '...');
+                }
+                else {
+                    console.log('结果:', JSON.stringify(jsonResult, null, 2));
+                }
+            }
+            catch (error) {
+                console.log('原始输出:');
+                console.log(result.stdout);
+            }
+        }
+        else {
+            console.error('测试失败!');
+            console.error('错误输出:');
+            console.error(result.stderr);
+        }
+    }
+    console.log('\n所有测试完成!');
+}
+// 运行测试
+testDirectClient().catch(error => {
+    console.error('运行测试时出错:', error);
+    process.exit(1);
+});

package/dist/tests/test-mcp-methods.js

@@ -0,0 +1,114 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
+import path from 'path';
+import { fileURLToPath } from 'url';
+// 获取当前文件的目录路径 (Get the directory path of the current file)
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+/**
+ * 测试 MCP 方法 (Test MCP methods)
+ * 测试 resources/list 和 prompts/list 方法 (Test resources/list and prompts/list methods)
+ */
+async function testMcpMethods() {
+    console.log('开始测试 MCP 方法...\n');
+    // 创建服务器进程 (Create server process)
+    const serverPath = path.resolve(path.dirname(__dirname), 'index.js');
+    console.log(`使用服务器路径: ${serverPath}`);
+    // 创建客户端传输层 (Create client transport layer)
+    const transport = new StdioClientTransport({
+        command: 'node',
+        args: [serverPath],
+        stderr: 'inherit',
+        env: {
+            ...process.env, // 传递所有环境变量，包括MCP_LANG
+        }
+    });
+    // 创建客户端 (Create client)
+    const client = new Client({
+        name: "mcp-methods-test-client",
+        version: "1.0.0"
+    });
+    try {
+        // 连接到传输层 (Connect to transport layer)
+        console.log('连接到 MCP 服务器...');
+        await client.connect(transport);
+        console.log('连接成功!\n');
+        // 测试 resources/list 方法 (Test resources/list method)
+        console.log('测试 resources/list 方法...');
+        try {
+            const resourcesResult = await client.listResources({});
+            console.log('resources/list 结果:');
+            console.log(JSON.stringify(resourcesResult, null, 2));
+            console.log('resources/list 测试成功!\n');
+        }
+        catch (error) {
+            console.error('resources/list 测试失败:', error);
+        }
+        // 测试 prompts/list 方法 (Test prompts/list method)
+        console.log('测试 prompts/list 方法...');
+        try {
+            const promptsResult = await client.listPrompts({});
+            console.log('prompts/list 结果:');
+            console.log(JSON.stringify(promptsResult, null, 2));
+            console.log('prompts/list 测试成功!\n');
+        }
+        catch (error) {
+            console.error('prompts/list 测试失败:', error);
+        }
+        /*
+        // 测试 resources/read 方法 (Test resources/read method)
+        // 注释掉此部分，因为我们没有找到可用的资源 (Comment out this part because we didn't find available resources)
+        console.log('测试 resources/read 方法...');
+        try {
+          const readResult = await client.readResource({
+            uri: 'file:///docs/readme'
+          });
+          console.log('resources/read 结果 (前100个字符):');
+          if (readResult.contents && readResult.contents[0] && readResult.contents[0].text) {
+            console.log((readResult.contents[0].text as string).substring(0, 100) + '...');
+          } else {
+            console.log(readResult);
+          }
+          console.log('resources/read 测试成功!\n');
+        } catch (error) {
+          console.error('resources/read 测试失败:', error);
+        }
+        */
+        // 测试 prompts/get 方法 (Test prompts/get method)
+        console.log('测试 prompts/get 方法...');
+        try {
+            const getPromptResult = await client.getPrompt({
+                name: 'fetch-website',
+                arguments: {
+                    url: 'https://example.com',
+                    format: 'html',
+                    useBrowser: 'false'
+                }
+            });
+            console.log('prompts/get 结果:');
+            console.log(JSON.stringify(getPromptResult, null, 2));
+            console.log('prompts/get 测试成功!\n');
+        }
+        catch (error) {
+            console.error('prompts/get 测试失败:', error);
+        }
+    }
+    catch (error) {
+        console.error('测试过程中发生错误:', error);
+    }
+    finally {
+        // 关闭客户端连接 (Close client connection)
+        await client.close();
+        console.log('测试完成，客户端连接已关闭');
+    }
+}
+// 执行测试 (Execute test)
+testMcpMethods().catch(error => {
+    console.error('测试脚本执行失败:', error);
+    process.exit(1);
+});