@lmcc-dev/mult-fetch-mcp-server 1.0.3 → 1.2.0

package/dist/src/lib/NodeFetcher.js

@@ -55,16 +55,16 @@ export class NodeFetcher {
             return undefined;
         }
         // 检查环境变量 (Check environment variables)
-        log('fetcher.checkingProxyEnv', true, {}, COMPONENTS.NODE_FETCH);
+        log('node.checkingProxyEnv', true, {}, COMPONENTS.NODE_FETCH);
         const envVars = ['https_proxy', 'HTTPS_PROXY', 'http_proxy', 'HTTP_PROXY'];
         for (const envVar of envVars) {
-            log('fetcher.envVarValue', true, {
+            log('node.envVarValue', true, {
                 envVar,
                 value: process.env[envVar]
             }, COMPONENTS.NODE_FETCH);
             if (process.env[envVar]) {
                 const proxyUrl = process.env[envVar];
-                log('fetcher.foundSystemProxy', true, { proxy: proxyUrl }, COMPONENTS.NODE_FETCH);
+                log('node.foundSystemProxy', true, { proxy: proxyUrl }, COMPONENTS.NODE_FETCH);
                 return proxyUrl;
             }
         }
@@ -72,22 +72,22 @@ export class NodeFetcher {
         try {
             const platform = process.platform;
             let proxyUrl;
-            log('fetcher.checkingSystemEnvVars', true, { platform }, COMPONENTS.NODE_FETCH);
+            log('node.checkingSystemEnvVars', true, { platform }, COMPONENTS.NODE_FETCH);
             if (platform === 'win32') {
                 // Windows系统 - 使用set命令 (Windows - use set command)
                 try {
                     // 使用set命令获取代理环境变量 (Use set command to get proxy environment variables)
                     const setOutput = execSync('set http_proxy & set https_proxy & set HTTP_PROXY & set HTTPS_PROXY').toString();
-                    log('fetcher.windowsEnvVars', true, { output: setOutput.trim() }, COMPONENTS.NODE_FETCH);
+                    log('node.windowsEnvVars', true, { output: setOutput.trim() }, COMPONENTS.NODE_FETCH);
                     // 解析输出找到代理设置 (Parse output to find proxy settings)
                     const proxyMatch = setOutput.match(/(?:http_proxy|https_proxy|HTTP_PROXY|HTTPS_PROXY)=(https?:\/\/[^=\r\n]+)/i);
                     if (proxyMatch && proxyMatch[1]) {
                         proxyUrl = proxyMatch[1].trim();
-                        log('fetcher.foundWindowsEnvProxy', true, { proxy: proxyUrl }, COMPONENTS.NODE_FETCH);
+                        log('node.foundWindowsEnvProxy', true, { proxy: proxyUrl }, COMPONENTS.NODE_FETCH);
                     }
                 }
                 catch (winError) {
-                    log('fetcher.errorGettingWindowsEnvVars', true, { error: String(winError) }, COMPONENTS.NODE_FETCH);
+                    log('node.errorGettingWindowsEnvVars', true, { error: String(winError) }, COMPONENTS.NODE_FETCH);
                 }
             }
             else {
@@ -95,38 +95,168 @@ export class NodeFetcher {
                 try {
                     // 使用env命令获取所有环境变量 (Use env command to get all environment variables)
                     const envOutput = execSync('env').toString();
-                    log('fetcher.unixEnvVars', true, { output: envOutput.length > 200 ? envOutput.substring(0, 200) + '...' : envOutput }, COMPONENTS.NODE_FETCH);
+                    log('node.unixEnvVars', true, { output: envOutput.length > 200 ? envOutput.substring(0, 200) + '...' : envOutput }, COMPONENTS.NODE_FETCH);
                     // 解析输出找到代理设置 (Parse output to find proxy settings)
                     const proxyMatch = envOutput.match(/(?:http_proxy|https_proxy|HTTP_PROXY|HTTPS_PROXY)=(https?:\/\/[^=\n]+)/i);
                     if (proxyMatch && proxyMatch[1]) {
                         proxyUrl = proxyMatch[1].trim();
-                        log('fetcher.foundUnixEnvProxy', true, { proxy: proxyUrl }, COMPONENTS.NODE_FETCH);
+                        log('node.foundUnixEnvProxy', true, { proxy: proxyUrl }, COMPONENTS.NODE_FETCH);
                     }
                 }
                 catch (unixError) {
-                    log('fetcher.errorGettingUnixEnvVars', true, { error: String(unixError) }, COMPONENTS.NODE_FETCH);
+                    log('node.errorGettingUnixEnvVars', true, { error: String(unixError) }, COMPONENTS.NODE_FETCH);
                 }
             }
             if (proxyUrl) {
                 return proxyUrl;
             }
             // 如果没有找到代理，记录日志 (If no proxy is found, log a message)
-            log('fetcher.noSystemProxyFound', true, {}, COMPONENTS.NODE_FETCH);
+            log('node.noSystemProxyFound', true, {}, COMPONENTS.NODE_FETCH);
         }
         catch (error) {
-            log('fetcher.errorGettingSystemEnvVars', true, { error: String(error) }, COMPONENTS.NODE_FETCH);
+            log('node.errorGettingSystemEnvVars', true, { error: String(error) }, COMPONENTS.NODE_FETCH);
         }
         // 检查NO_PROXY环境变量 (Check NO_PROXY environment variable)
         const noProxy = process.env.NO_PROXY || process.env.no_proxy;
         if (noProxy) {
-            log('fetcher.foundNoProxy', true, { noProxy }, COMPONENTS.NODE_FETCH);
+            log('node.foundNoProxy', true, { noProxy }, COMPONENTS.NODE_FETCH);
         }
         return undefined;
     }
     /**
-     * 执行带重定向处理的HTTP请求 (Perform HTTP request with redirect handling)
-     * @param requestPayload 请求参数 (Request parameters)
-     * @returns 响应数据 (Response data)
+     * 处理请求延迟 (Handle request delay)
+     * @param noDelay 是否禁用延迟 (Whether to disable delay)
+     * @param isRedirect 是否是重定向请求 (Whether it's a redirect request)
+     * @param debug 是否启用调试模式 (Whether debug mode is enabled)
+     */
+    static async handleDelay(noDelay, isRedirect, debug) {
+        if (!noDelay && isRedirect) {
+            await this.randomDelay();
+        }
+    }
+    /**
+     * 准备请求头 (Prepare request headers)
+     * @param headers 原始请求头 (Original headers)
+     * @param debug 是否启用调试模式 (Whether debug mode is enabled)
+     * @returns 处理后的请求头 (Processed headers)
+     */
+    static prepareHeaders(headers = {}, debug) {
+        const requestHeaders = { ...headers };
+        // 添加随机User-Agent (Add random User-Agent)
+        if (!requestHeaders['User-Agent']) {
+            const userAgent = this.getRandomUserAgent();
+            requestHeaders['User-Agent'] = userAgent;
+            log('node.usingUserAgent', debug, { userAgent }, COMPONENTS.NODE_FETCH);
+        }
+        return requestHeaders;
+    }
+    /**
+     * 设置代理 (Set proxy)
+     * @param url 请求URL (Request URL)
+     * @param proxy 代理URL (Proxy URL)
+     * @param debug 是否启用调试模式 (Whether debug mode is enabled)
+     * @returns 代理Agent (Proxy agent)
+     */
+    static setupProxy(url, proxy, debug) {
+        if (!proxy)
+            return undefined;
+        let agent = undefined;
+        if (url.startsWith('https://')) {
+            agent = new HttpsProxyAgent(proxy);
+            log('node.usingHttpsProxy', debug, {}, COMPONENTS.NODE_FETCH);
+        }
+        else {
+            agent = new HttpProxyAgent(proxy);
+            log('node.usingHttpProxy', debug, {}, COMPONENTS.NODE_FETCH);
+        }
+        return agent;
+    }
+    /**
+     * 处理重定向URL (Handle redirect URL)
+     * @param location 重定向位置 (Redirect location)
+     * @param currentUrl 当前URL (Current URL)
+     * @param debug 是否启用调试模式 (Whether debug mode is enabled)
+     * @returns 完整的重定向URL (Complete redirect URL)
+     */
+    static buildRedirectUrl(location, currentUrl, debug) {
+        let redirectUrl = location;
+        if (location.startsWith('/')) {
+            const urlObj = new URL(currentUrl);
+            redirectUrl = `${urlObj.protocol}//${urlObj.host}${location}`;
+        }
+        else if (!location.startsWith('http')) {
+            redirectUrl = new URL(location, currentUrl).toString();
+        }
+        log('node.constructedFullRedirectUrl', debug, { redirectUrl }, COMPONENTS.NODE_FETCH);
+        return redirectUrl;
+    }
+    /**
+     * 处理错误响应 (Handle error response)
+     * @param response 响应对象 (Response object)
+     * @param debug 是否启用调试模式 (Whether debug mode is enabled)
+     * @returns Promise<never> 抛出错误 (Throws error)
+     */
+    static async handleErrorResponse(response, debug) {
+        log('node.errorResponse', debug, { status: response.status, statusText: response.statusText }, COMPONENTS.NODE_FETCH);
+        // 尝试读取错误响应体 (Try to read error response body)
+        let errorText = '';
+        try {
+            errorText = await response.text();
+            log('node.errorResponseBody', debug, { body: errorText.substring(0, 200) + (errorText.length > 200 ? '...' : '') }, COMPONENTS.NODE_FETCH);
+        }
+        catch (textError) {
+            log('node.errorReadingBody', debug, { error: String(textError) }, COMPONENTS.NODE_FETCH);
+        }
+        // 创建错误对象 (Create error object)
+        const error = new Error(`HTTP Error ${response.status}: ${response.statusText}`);
+        error.status = response.status;
+        error.statusText = response.statusText;
+        error.body = errorText;
+        throw error;
+    }
+    /**
+     * 处理请求错误 (Handle request error)
+     * @param error 错误对象 (Error object)
+     * @param fetchStart 请求开始时间 (Request start time)
+     * @param timeout 超时时间 (Timeout)
+     * @param redirectCount 重定向计数 (Redirect count)
+     * @param maxRedirects 最大重定向次数 (Maximum redirects)
+     * @param debug 是否启用调试模式 (Whether debug mode is enabled)
+     * @returns Promise<never> 抛出错误 (Throws error)
+     */
+    static handleRequestError(error, fetchStart, timeout, redirectCount, maxRedirects, debug) {
+        // 记录错误 (Log error)
+        log('node.fetchError', debug, { error: String(error) }, COMPONENTS.NODE_FETCH);
+        // 处理超时错误 (Handle timeout error)
+        if (error.name === 'AbortError') {
+            log('node.requestAborted', debug, { duration: Date.now() - fetchStart }, COMPONENTS.NODE_FETCH);
+            const timeoutError = new Error(`Request timeout after ${timeout}ms`);
+            timeoutError.code = 'ETIMEDOUT';
+            timeoutError.timeout = timeout;
+            throw timeoutError;
+        }
+        // 处理网络错误 (Handle network error)
+        if (error.code) {
+            log('node.networkError', debug, { code: error.code || error.name }, COMPONENTS.NODE_FETCH);
+            const networkError = new Error(`Network error: ${error.code || error.message}`);
+            networkError.code = error.code;
+            networkError.originalError = error;
+            throw networkError;
+        }
+        // 处理重定向错误 (Handle redirect error)
+        if (redirectCount >= maxRedirects) {
+            log('node.tooManyRedirects', debug, { redirects: maxRedirects }, COMPONENTS.NODE_FETCH);
+            const redirectError = new Error(`Too many redirects (${maxRedirects})`);
+            redirectError.code = 'EMAXREDIRECTS';
+            redirectError.redirects = redirectCount;
+            throw redirectError;
+        }
+        // 重新抛出其他错误 (Rethrow other errors)
+        throw error;
+    }
+    /**
+     * 执行带重定向处理的HTTP请求 (Execute HTTP request with redirect handling)
+     * 支持自动处理重定向、超时和代理 (Supports automatic handling of redirects, timeouts, and proxies)
      */
     static async _fetchWithRedirects({ url, headers = {}, proxy, noDelay, timeout = 30000, // 默认30秒超时
     maxRedirects = 10, // 最大重定向次数
@@ -152,20 +282,12 @@ export class NodeFetcher {
             // 处理重定向循环 (Handle redirect loop)
             while (redirectCount < maxRedirects) {
                 log('node.fetchingUrl', debug, { url: currentUrl, redirect: redirectCount }, COMPONENTS.NODE_FETCH);
-                // 如果启用了随机延迟且不是第一个请求，则添加延迟 (Add delay if random delay is enabled and not the first request)
-                if (!noDelay && redirectCount > 0) {
-                    await this.randomDelay();
-                }
+                // 处理延迟 (Handle delay)
+                await this.handleDelay(noDelay, redirectCount > 0, debug);
                 // 准备请求头 (Prepare request headers)
-                const requestHeaders = {
-                    ...headers
-                };
-                // 添加随机User-Agent (Add random User-Agent)
-                if (!requestHeaders['User-Agent']) {
-                    const userAgent = this.getRandomUserAgent();
-                    requestHeaders['User-Agent'] = userAgent;
-                    log('node.usingUserAgent', debug, { userAgent }, COMPONENTS.NODE_FETCH);
-                }
+                const requestHeaders = this.prepareHeaders(headers, debug);
+                // 设置代理 (Set proxy)
+                const agent = this.setupProxy(currentUrl, finalProxy, debug);
                 // 准备请求选项 (Prepare request options)
                 const fetchOptions = {
                     method: 'GET',
@@ -173,17 +295,7 @@ export class NodeFetcher {
                     timeout,
                     signal: controller.signal,
                 };
-                // 设置代理 (Set proxy)
-                let agent = undefined;
-                if (finalProxy) {
-                    if (currentUrl.startsWith('https://')) {
-                        agent = new HttpsProxyAgent(finalProxy);
-                        log('node.usingHttpsProxy', debug, {}, COMPONENTS.NODE_FETCH);
-                    }
-                    else {
-                        agent = new HttpProxyAgent(finalProxy);
-                        log('node.usingHttpProxy', debug, {}, COMPONENTS.NODE_FETCH);
-                    }
+                if (agent) {
                     fetchOptions.agent = agent;
                 }
                 // 记录请求详情 (Log request details)
@@ -209,17 +321,7 @@ export class NodeFetcher {
                     const location = response.headers.get('location');
                     log('node.redirectingTo', debug, { location }, COMPONENTS.NODE_FETCH);
                     // 构建完整的重定向URL (Build complete redirect URL)
-                    let redirectUrl = location;
-                    if (location.startsWith('/')) {
-                        const urlObj = new URL(currentUrl);
-                        redirectUrl = `${urlObj.protocol}//${urlObj.host}${location}`;
-                    }
-                    else if (!location.startsWith('http')) {
-                        redirectUrl = new URL(location, currentUrl).toString();
-                    }
-                    log('node.constructedFullRedirectUrl', debug, { redirectUrl }, COMPONENTS.NODE_FETCH);
-                    // 更新当前URL为重定向URL (Update current URL to redirect URL)
-                    currentUrl = redirectUrl;
+                    currentUrl = this.buildRedirectUrl(location, currentUrl, debug);
                     continue;
                 }
                 // 如果响应成功，返回响应 (If response is successful, return response)
@@ -229,58 +331,15 @@ export class NodeFetcher {
                 }
                 else {
                     // 处理错误响应 (Handle error response)
-                    log('node.errorResponse', debug, { status: response.status, statusText: response.statusText }, COMPONENTS.NODE_FETCH);
-                    // 尝试读取错误响应体 (Try to read error response body)
-                    let errorText = '';
-                    try {
-                        errorText = await response.text();
-                        log('node.errorResponseBody', debug, { body: errorText.substring(0, 200) + (errorText.length > 200 ? '...' : '') }, COMPONENTS.NODE_FETCH);
-                    }
-                    catch (textError) {
-                        log('node.errorReadingBody', debug, { error: String(textError) }, COMPONENTS.NODE_FETCH);
-                    }
-                    // 创建错误对象 (Create error object)
-                    const error = new Error(`HTTP Error ${response.status}: ${response.statusText}`);
-                    error.status = response.status;
-                    error.statusText = response.statusText;
-                    error.body = errorText;
-                    throw error;
+                    return await this.handleErrorResponse(response, debug);
                 }
             }
             // 如果达到最大重定向次数，抛出错误 (If maximum redirects reached, throw error)
             throw new Error(`Too many redirects (${maxRedirects})`);
         }
         catch (error) {
-            // 清除超时定时器 (Clear timeout timer)
-            clearTimeout(timeoutId);
-            // 记录错误 (Log error)
-            log('node.fetchError', debug, { error: String(error) }, COMPONENTS.NODE_FETCH);
-            // 处理超时错误 (Handle timeout error)
-            if (error.name === 'AbortError') {
-                log('node.requestAborted', debug, { duration: Date.now() - fetchStart }, COMPONENTS.NODE_FETCH);
-                const timeoutError = new Error(`Request timeout after ${timeout}ms`);
-                timeoutError.code = 'ETIMEDOUT';
-                timeoutError.timeout = timeout;
-                throw timeoutError;
-            }
-            // 处理网络错误 (Handle network error)
-            if (error.code) {
-                log('node.networkError', debug, { code: error.code || error.name }, COMPONENTS.NODE_FETCH);
-                const networkError = new Error(`Network error: ${error.code || error.message}`);
-                networkError.code = error.code;
-                networkError.originalError = error;
-                throw networkError;
-            }
-            // 处理重定向错误 (Handle redirect error)
-            if (redirectCount >= maxRedirects) {
-                log('node.tooManyRedirects', debug, { redirects: maxRedirects }, COMPONENTS.NODE_FETCH);
-                const redirectError = new Error(`Too many redirects (${maxRedirects})`);
-                redirectError.code = 'EMAXREDIRECTS';
-                redirectError.redirects = redirectCount;
-                throw redirectError;
-            }
-            // 重新抛出其他错误 (Rethrow other errors)
-            throw error;
+            // 处理请求错误 (Handle request error)
+            this.handleRequestError(error, fetchStart, timeout, redirectCount, maxRedirects, debug);
         }
         finally {
             // 清除超时定时器 (Clear timeout timer)
@@ -373,14 +432,14 @@ export class NodeFetcher {
      */
     static async txt(requestPayload) {
         const { debug = false } = requestPayload;
-        log('fetcher.startingTxtFetch', debug, {}, COMPONENTS.NODE_FETCH);
+        log('node.startingTxtFetch', debug, {}, COMPONENTS.NODE_FETCH);
         try {
             // 执行请求 (Execute request)
             const response = await this._fetchWithRedirects(requestPayload);
             // 读取响应文本 (Read response text)
-            log('fetcher.readingText', debug, {}, COMPONENTS.NODE_FETCH);
+            log('node.readingText', debug, {}, COMPONENTS.NODE_FETCH);
             const text = await response.text();
-            log('fetcher.textContentLength', debug, { length: text.length }, COMPONENTS.NODE_FETCH);
+            log('node.textContentLength', debug, { length: text.length }, COMPONENTS.NODE_FETCH);
             // 返回纯文本内容 (Return plain text content)
             return {
                 text,
@@ -401,21 +460,21 @@ export class NodeFetcher {
      */
     static async markdown(requestPayload) {
         const { debug = false } = requestPayload;
-        log('fetcher.startingMarkdownFetch', debug, {}, COMPONENTS.NODE_FETCH);
+        log('node.startingMarkdownFetch', debug, {}, COMPONENTS.NODE_FETCH);
         try {
             // 执行请求 (Execute request)
             const response = await this._fetchWithRedirects(requestPayload);
             // 读取响应文本 (Read response text)
-            log('fetcher.readingText', debug, {}, COMPONENTS.NODE_FETCH);
+            log('node.readingText', debug, {}, COMPONENTS.NODE_FETCH);
             const html = await response.text();
-            log('fetcher.htmlContentLength', debug, { length: html.length }, COMPONENTS.NODE_FETCH);
+            log('node.htmlContentLength', debug, { length: html.length }, COMPONENTS.NODE_FETCH);
             // 创建Turndown服务 (Create Turndown service)
-            log('fetcher.creatingTurndown', debug, {}, COMPONENTS.NODE_FETCH);
+            log('node.creatingTurndown', debug, {}, COMPONENTS.NODE_FETCH);
             const turndownService = new TurndownService();
             // 将HTML转换为Markdown (Convert HTML to Markdown)
-            log('fetcher.convertingToMarkdown', debug, {}, COMPONENTS.NODE_FETCH);
+            log('node.convertingToMarkdown', debug, {}, COMPONENTS.NODE_FETCH);
             const markdown = turndownService.turndown(html);
-            log('fetcher.markdownContentLength', debug, { length: markdown.length }, COMPONENTS.NODE_FETCH);
+            log('node.markdownContentLength', debug, { length: markdown.length }, COMPONENTS.NODE_FETCH);
             // 返回Markdown内容 (Return Markdown content)
             return {
                 markdown,

package/dist/src/lib/fetch.js

@@ -0,0 +1,40 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { Fetcher } from './fetchers/index.js';
+/**
+ * 获取HTML内容 (Get HTML content)
+ * @param requestPayload 请求参数 (Request parameters)
+ * @returns HTML内容 (HTML content)
+ */
+export async function fetchHtml(requestPayload) {
+    return await Fetcher.html(requestPayload);
+}
+/**
+ * 获取JSON内容 (Get JSON content)
+ * @param requestPayload 请求参数 (Request parameters)
+ * @returns JSON内容 (JSON content)
+ */
+export async function fetchJson(requestPayload) {
+    return await Fetcher.json(requestPayload);
+}
+/**
+ * 获取纯文本内容 (Get plain text content)
+ * @param requestPayload 请求参数 (Request parameters)
+ * @returns 纯文本内容 (Plain text content)
+ */
+export async function fetchTxt(requestPayload) {
+    return await Fetcher.txt(requestPayload);
+}
+/**
+ * 获取Markdown内容 (Get Markdown content)
+ * @param requestPayload 请求参数 (Request parameters)
+ * @returns Markdown内容 (Markdown content)
+ */
+export async function fetchMarkdown(requestPayload) {
+    return await Fetcher.markdown(requestPayload);
+}
+// 导出所有类型和类 (Export all types and classes)
+export * from './fetchers/index.js';

package/dist/src/lib/fetchers/FetcherFactory.js

@@ -0,0 +1,43 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { BrowserFetcher } from './browser/BrowserFetcher.js';
+import { NodeFetcher } from './node/NodeFetcher.js';
+import { log, COMPONENTS } from '../logger.js';
+/**
+ * 抓取器工厂类 (Fetcher factory class)
+ * 根据请求参数创建合适的抓取器实例 (Create appropriate fetcher instance based on request parameters)
+ */
+export class FetcherFactory {
+    /**
+     * 创建抓取器实例 (Create fetcher instance)
+     * @param requestPayload 请求参数 (Request parameters)
+     * @returns 抓取器实例 (Fetcher instance)
+     */
+    static createFetcher(requestPayload) {
+        const { useBrowser = false, useNodeFetch = false, autoDetectMode = true, debug = false } = requestPayload;
+        // 如果明确指定使用浏览器模式 (If browser mode is explicitly specified)
+        if (useBrowser) {
+            log('factory.using_browser_mode', debug, {}, COMPONENTS.FETCHER_FACTORY);
+            return new BrowserFetcher();
+        }
+        // 如果明确指定使用Node模式 (If Node mode is explicitly specified)
+        if (useNodeFetch) {
+            log('factory.using_node_mode', debug, {}, COMPONENTS.FETCHER_FACTORY);
+            return new NodeFetcher();
+        }
+        // 如果启用自动检测模式 (If auto-detect mode is enabled)
+        if (autoDetectMode) {
+            // 这里可以添加更复杂的逻辑来决定使用哪种模式
+            // 例如，根据URL特征、请求头等判断
+            // 目前简单地默认使用Node模式，因为它更轻量级
+            log('factory.auto_detecting_mode', debug, { defaultToNode: true }, COMPONENTS.FETCHER_FACTORY);
+            return new NodeFetcher();
+        }
+        // 默认使用Node模式 (Default to Node mode)
+        log('factory.defaultingToNodeMode', debug, {}, COMPONENTS.FETCHER_FACTORY);
+        return new NodeFetcher();
+    }
+}