@lmcc-dev/mult-fetch-mcp-server 1.3.1 → 1.3.3

package/dist/src/lib/fetchers/browser/BrowserFetcher.js

@@ -11,6 +11,7 @@ import { getRandomUserAgent, getSystemProxy } from '../common/utils.js';
 import { ContentSizeManager } from '../../utils/ContentSizeManager.js';
 import { BaseFetcher } from '../common/BaseFetcher.js';
 import { ContentProcessor } from '../../utils/ContentProcessor.js';
+import { validateFetchUrl } from '../../utils/UrlValidator.js';
 /**
  * 浏览器模式获取器类 (Browser mode fetcher class)
  * 使用Puppeteer实现浏览器模式的网页获取 (Implements webpage fetching in browser mode using Puppeteer)
@@ -75,6 +76,7 @@ export class BrowserFetcher extends BaseFetcher {
             if (effectiveProxy) {
                 log('browser.usingProxy', debug, { proxy: effectiveProxy }, COMPONENTS.BROWSER_FETCH);
             }
+            await validateFetchUrl(url);
             // 导航到URL
             log('browser.navigating', debug, { url }, COMPONENTS.BROWSER_FETCH);
             await page.goto(url, {
@@ -144,7 +146,7 @@ export class BrowserFetcher extends BaseFetcher {
      * @returns 获取结果 (Fetch result)
      */
     async html(requestPayload) {
-        const { url, debug = false, closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
+        const { url, debug = false, closeBrowser: _closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0, extractContent = false, includeMetadata = true, fallbackToOriginal = true } = requestPayload;
         // 如果提供了分段ID和起始游标，则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
         if (chunkId && startCursor !== undefined) {
             return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.BROWSER_FETCH);
@@ -159,13 +161,28 @@ export class BrowserFetcher extends BaseFetcher {
             }
             // 获取HTML内容 (Get HTML content)
             const html = result.content[0].text;
+            // 处理内容提取 (Process content extraction)
+            const { content: processedContent, metadata } = this.processWithContentExtraction(html, url, {
+                extractContent,
+                includeMetadata,
+                fallbackToOriginal
+            }, debug, COMPONENTS.BROWSER_FETCH);
             // 检查内容大小并处理 (Check content size and process)
-            const chunkingResult = this.handleContentChunking(html, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.BROWSER_FETCH);
+            const chunkingResult = this.handleContentChunking(processedContent, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.BROWSER_FETCH);
             if (chunkingResult) {
+                // 如果有元数据，添加到分块结果中 (If there is metadata, add it to the chunk result)
+                if (metadata) {
+                    chunkingResult.metadata = metadata;
+                }
                 return chunkingResult;
             }
-            // 返回HTML内容 (Return HTML content)
-            return result;
+            // 创建新的结果（使用处理后的内容） (Create new result with processed content)
+            const processedResult = BaseFetcher.createSuccessResponse(processedContent);
+            // 如果有元数据，添加到结果中 (If there is metadata, add it to the result)
+            if (metadata) {
+                processedResult.metadata = metadata;
+            }
+            return processedResult;
         }
         catch (error) {
             // 处理错误 (Handle error)
@@ -180,7 +197,7 @@ export class BrowserFetcher extends BaseFetcher {
      * @returns 获取结果 (Fetch result)
      */
     async json(requestPayload) {
-        const { url, debug = false, closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
+        const { url, debug = false, closeBrowser: _closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
         // 如果提供了分段ID和起始游标，则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
         if (chunkId && startCursor !== undefined) {
             return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.BROWSER_FETCH);
@@ -226,7 +243,7 @@ export class BrowserFetcher extends BaseFetcher {
      * @returns 获取结果 (Fetch result)
      */
     async txt(requestPayload) {
-        const { url, debug = false, closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
+        const { url, debug = false, closeBrowser: _closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
         // 如果提供了分段ID和起始游标，则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
         if (chunkId && startCursor !== undefined) {
             return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.BROWSER_FETCH);
@@ -266,7 +283,7 @@ export class BrowserFetcher extends BaseFetcher {
      * @returns 纯文本响应 (Plain text response)
      */
     async plainText(requestPayload) {
-        const { url, debug = false, closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
+        const { url: _url, debug = false, closeBrowser: _closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
         // 如果提供了分段ID和起始游标，则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
         if (chunkId && startCursor !== undefined) {
             return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.BROWSER_FETCH);
@@ -280,7 +297,7 @@ export class BrowserFetcher extends BaseFetcher {
         // 获取HTML内容 (Get HTML content)
         const html = result.content[0].text;
         // 使用ContentProcessor将HTML转换为纯文本 (Use ContentProcessor to convert HTML to plain text)
-        const plainText = ContentProcessor.htmlToText(html, debug, COMPONENTS.BROWSER_FETCH);
+        const plainText = ContentProcessor.htmlToText(html, debug);
         // 检查内容大小并处理 (Check content size and process)
         const chunkingResult = this.handleContentChunking(plainText, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.BROWSER_FETCH);
         if (chunkingResult) {
@@ -295,7 +312,7 @@ export class BrowserFetcher extends BaseFetcher {
      * @returns Markdown内容 (Markdown content)
      */
     async markdown(requestPayload) {
-        const { url, debug = false, closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
+        const { url: _url, debug = false, closeBrowser: _closeBrowser = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
         // 如果提供了分段ID和起始游标，则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
         if (chunkId && startCursor !== undefined) {
             return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.BROWSER_FETCH);
@@ -309,7 +326,7 @@ export class BrowserFetcher extends BaseFetcher {
         // 获取HTML内容 (Get HTML content)
         const html = result.content[0].text;
         // 使用ContentProcessor将HTML转换为Markdown (Use ContentProcessor to convert HTML to Markdown)
-        const markdown = ContentProcessor.htmlToMarkdown(html, debug, COMPONENTS.BROWSER_FETCH);
+        const markdown = ContentProcessor.htmlToMarkdown(html, debug);
         // 检查内容大小并处理 (Check content size and process)
         const chunkingResult = this.handleContentChunking(markdown, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.BROWSER_FETCH);
         if (chunkingResult) {

package/dist/src/lib/fetchers/browser/BrowserInstance.js

@@ -88,13 +88,13 @@ export class BrowserInstance {
         // 标记浏览器正在启动 (Mark browser as starting)
         this.browserStarting = true;
         // 创建启动Promise (Create startup Promise)
-        this.browserStartPromise = new Promise(async (resolve, reject) => {
-            try {
-                log('browser.startingBrowser', debug, {}, COMPONENTS.BROWSER_FETCH);
-                // 准备浏览器启动参数 (Prepare browser startup parameters)
-                const launchOptions = {
-                    headless: 'new',
-                    args: [
+        this.browserStartPromise = new Promise((resolve, reject) => {
+            // 内部定义并立即调用异步函数
+            (async () => {
+                try {
+                    log('browser.startingBrowser', debug, {}, COMPONENTS.BROWSER_FETCH);
+                    // 准备浏览器启动参数 (Prepare browser startup parameters)
+                    const browserArgs = [
                         '--no-sandbox',
                         '--disable-setuid-sandbox',
                         '--disable-dev-shm-usage',
@@ -112,51 +112,57 @@ export class BrowserInstance {
                         '--disable-features=IsolateOrigins,site-per-process',
                         '--disable-web-security',
                         '--disable-site-isolation-trials',
-                        '--disable-features=BlockInsecurePrivateNetworkRequests',
                         '--disable-features=IsolateOrigins',
                         '--disable-features=site-per-process',
                         '--disable-blink-features=AutomationControlled',
                         '--user-agent=' + getRandomUserAgent()
-                    ],
-                    ignoreHTTPSErrors: true,
-                    defaultViewport: {
-                        width: 1920,
-                        height: 1080
+                    ];
+                    if (process.env.ALLOW_PRIVATE_NETWORK_REQUESTS === 'true') {
+                        browserArgs.push('--disable-features=BlockInsecurePrivateNetworkRequests');
                     }
-                };
-                // 检查是否有环境变量指定的Chrome路径 (Check if there is a Chrome path specified by environment variables)
-                const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
-                if (executablePath) {
-                    log('browser.usingCustomChromePath', debug, { path: executablePath }, COMPONENTS.BROWSER_FETCH);
-                    launchOptions.executablePath = executablePath;
-                    launchOptions.channel = undefined;
+                    const launchOptions = {
+                        headless: 'new',
+                        args: browserArgs,
+                        ignoreHTTPSErrors: true,
+                        defaultViewport: {
+                            width: 1920,
+                            height: 1080
+                        }
+                    };
+                    // 检查是否有环境变量指定的Chrome路径 (Check if there is a Chrome path specified by environment variables)
+                    const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH;
+                    if (executablePath) {
+                        log('browser.usingCustomChromePath', debug, { path: executablePath }, COMPONENTS.BROWSER_FETCH);
+                        launchOptions.executablePath = executablePath;
+                        launchOptions.channel = undefined;
+                    }
+                    // 启动浏览器 (Launch browser)
+                    // @ts-ignore - 忽略puppeteer-extra的类型错误
+                    this.browser = await puppeteerExtra.launch(launchOptions);
+                    log('browser.browserStarted', debug, {}, COMPONENTS.BROWSER_FETCH);
+                    // 设置浏览器关闭事件处理 (Set browser close event handling)
+                    this.browser.on('disconnected', () => {
+                        log('browser.browserDisconnected', debug, {}, COMPONENTS.BROWSER_FETCH);
+                        this.browser = null;
+                        this.browserStarting = false;
+                        this.browserStartPromise = null;
+                    });
+                    // 重置标志 (Reset flags)
+                    this.browserStarting = false;
+                    // 解析Promise (Resolve Promise)
+                    resolve(this.browser);
                 }
-                // 启动浏览器 (Launch browser)
-                // @ts-ignore - 忽略puppeteer-extra的类型错误
-                this.browser = await puppeteerExtra.launch(launchOptions);
-                log('browser.browserStarted', debug, {}, COMPONENTS.BROWSER_FETCH);
-                // 设置浏览器关闭事件处理 (Set browser close event handling)
-                this.browser.on('disconnected', () => {
-                    log('browser.browserDisconnected', debug, {}, COMPONENTS.BROWSER_FETCH);
+                catch (error) {
+                    // 记录错误 (Log error)
+                    log('browser.browserStartError', true, { error: String(error) }, COMPONENTS.BROWSER_FETCH);
+                    // 重置标志 (Reset flags)
                     this.browser = null;
                     this.browserStarting = false;
                     this.browserStartPromise = null;
-                });
-                // 重置标志 (Reset flags)
-                this.browserStarting = false;
-                // 解析Promise (Resolve Promise)
-                resolve(this.browser);
-            }
-            catch (error) {
-                // 记录错误 (Log error)
-                log('browser.browserStartError', true, { error: String(error) }, COMPONENTS.BROWSER_FETCH);
-                // 重置标志 (Reset flags)
-                this.browser = null;
-                this.browserStarting = false;
-                this.browserStartPromise = null;
-                // 拒绝Promise (Reject Promise)
-                reject(error);
-            }
+                    // 拒绝Promise (Reject Promise)
+                    reject(error);
+                }
+            })();
         });
         return this.browserStartPromise;
     }

package/dist/src/lib/fetchers/common/BaseFetcher.js

@@ -3,10 +3,11 @@
  * Co-Author: AI Assistant (Claude)
  * Description: This code was collaboratively developed by Martin and AI Assistant.
  */
-import { log } from "../../logger.js";
-import { ContentSizeManager } from "../../utils/ContentSizeManager.js";
-import { ChunkManager } from "../../utils/ChunkManager.js";
-import { TemplateUtils } from "../../utils/TemplateUtils.js";
+import { log } from '../../logger.js';
+import { ChunkManager } from '../../utils/ChunkManager.js';
+import { ContentSizeManager } from '../../utils/ContentSizeManager.js';
+import { TemplateUtils } from '../../utils/TemplateUtils.js';
+import { ContentExtractor } from '../../utils/ContentExtractor.js';
 /**
  * 基础获取器类 (Base fetcher class)
  * 提供所有获取器共用的基础功能 (Provides basic functionality used by all fetchers)
@@ -80,7 +81,14 @@ export class BaseFetcher {
             const estimatedRequests = Math.ceil(remainingBytes / contentSizeLimit);
             // 添加分段提示 (Add chunk prompt)
             const isFirstRequest = true;
-            const chunkWithPrompt = firstChunk + TemplateUtils.generateSizeBasedChunkPrompt(fetchedBytes, totalSize, chunkId, remainingBytes, estimatedRequests, contentSizeLimit, isFirstRequest);
+            let chunkWithPrompt;
+            // 检查是否已经检索了全部内容
+            if (remainingBytes <= 0) {
+                chunkWithPrompt = firstChunk + TemplateUtils.generateSizeBasedLastChunkPrompt(fetchedBytes, totalSize, isFirstRequest);
+            }
+            else {
+                chunkWithPrompt = firstChunk + TemplateUtils.generateSizeBasedChunkPrompt(fetchedBytes, totalSize, chunkId, remainingBytes, estimatedRequests, contentSizeLimit, isFirstRequest);
+            }
             // 创建响应 (Create response)
             return {
                 isError: false,
@@ -136,12 +144,26 @@ export class BaseFetcher {
         }
         // 获取相关信息 (Get related information)
         const { content, fetchedBytes, remainingBytes, isLastChunk, totalBytes } = chunkResult;
+        // 记录是否为最后一个分块的详细信息
+        log('fetcher.chunkInfo', debug, {
+            chunkId,
+            fetchedBytes,
+            totalBytes,
+            remainingBytes,
+            isLastChunk,
+            percentage: `${Math.round((fetchedBytes / totalBytes) * 100)}%`
+        }, component);
         // 计算预计还需要的请求次数 (Calculate estimated number of requests needed)
         const estimatedRequests = Math.ceil(remainingBytes / sizeLimit);
         // 添加分段提示 (Add chunk prompt)
         let contentWithPrompt;
-        if (isLastChunk) {
-            // 如果是最后一个分段 (If it's the last chunk)
+        // 如果是最后一个分段或没有剩余内容 (If it's the last chunk or there's no remaining content)
+        if (isLastChunk || remainingBytes <= 0) {
+            log('fetcher.lastChunkDetected', debug, {
+                fetchedBytes,
+                totalBytes,
+                remainingBytes
+            }, component);
             contentWithPrompt = content + TemplateUtils.generateSizeBasedLastChunkPrompt(fetchedBytes, totalBytes, startCursor === 0 // 如果startCursor为0，则为首次请求 (If startCursor is 0, it's the first request)
             );
         }
@@ -164,7 +186,8 @@ export class BaseFetcher {
             totalBytes,
             fetchedBytes,
             remainingBytes,
-            hasMoreChunks: !isLastChunk
+            hasMoreChunks: remainingBytes > 0,
+            isLastChunk // 明确添加isLastChunk属性以便客户端使用
         };
     }
     /**
@@ -184,8 +207,18 @@ export class BaseFetcher {
                 return response;
             }
             // 使用基于字节的提示 (Use byte-based prompt)
-            const promptText = TemplateUtils.generateSizeBasedChunkPrompt(response.fetchedBytes, response.totalBytes, chunkId, response.remainingBytes, estimatedRequests, currentSizeLimit, false // 设置为false表示这不是首次请求 (Set to false indicating this is not the first request)
-            );
+            let promptText;
+            // 检查是否还有剩余字节 (Check if there are remaining bytes)
+            if (response.remainingBytes <= 0) {
+                // 没有剩余字节，使用最后一块的提示 (No remaining bytes, use last chunk prompt)
+                promptText = TemplateUtils.generateSizeBasedLastChunkPrompt(response.fetchedBytes, response.totalBytes, false // 设置为false表示这不是首次请求 (Set to false indicating this is not the first request)
+                );
+            }
+            else {
+                // 仍有剩余字节，使用普通分块提示 (Still has remaining bytes, use regular chunk prompt)
+                promptText = TemplateUtils.generateSizeBasedChunkPrompt(response.fetchedBytes, response.totalBytes, chunkId, response.remainingBytes, estimatedRequests, currentSizeLimit, false // 设置为false表示这不是首次请求 (Set to false indicating this is not the first request)
+                );
+            }
             // 创建新的响应对象，避免修改原始对象 (Create new response object to avoid modifying the original)
             return {
                 ...response,
@@ -200,4 +233,67 @@ export class BaseFetcher {
         // 如果不需要添加提示，返回原始响应 (If no prompt needed, return original response)
         return response;
     }
+    /**
+     * 处理内容提取 (Process content extraction)
+     * 根据参数决定是否提取内容，返回处理后的内容和元数据 (Decide whether to extract content based on parameters, return processed content and metadata)
+     * @param html 原始HTML内容 (Original HTML content)
+     * @param url 页面URL (Page URL)
+     * @param options 内容提取选项 (Content extraction options)
+     * @param debug 是否启用调试 (Whether to enable debugging)
+     * @param component 组件名称，用于日志 (Component name for logging)
+     * @returns 处理后的内容和元数据 (Processed content and metadata)
+     */
+    processWithContentExtraction(html, url, options, debug = false, component) {
+        const { extractContent = false, includeMetadata = true, fallbackToOriginal = true } = options;
+        // 如果不需要提取内容，直接返回原始内容 (If content extraction is not needed, return original content)
+        if (!extractContent) {
+            return { content: html, metadata: null };
+        }
+        log('fetcher.extractingContent', debug, { url }, component);
+        try {
+            // 提取内容 (Extract content)
+            const extractResult = ContentExtractor.extractContent(html, url, debug);
+            // 如果提取成功且有内容 (If extraction is successful and has content)
+            if (extractResult.content) {
+                // 准备元数据 (Prepare metadata)
+                const metadata = includeMetadata ? {
+                    title: extractResult.title,
+                    byline: extractResult.byline,
+                    siteName: extractResult.siteName,
+                    excerpt: extractResult.excerpt,
+                    isReaderable: extractResult.isReaderable,
+                    length: extractResult.length
+                } : null;
+                log('fetcher.extractionSuccess', debug, {
+                    contentLength: extractResult.content.length,
+                    title: extractResult.title
+                }, component);
+                // 直接返回处理后的内容 (Directly return the processed content)
+                return { content: extractResult.content, metadata };
+            }
+            else {
+                log('fetcher.extractionFailed', debug, { url }, component);
+                // 如果提取失败且不允许回退到原始内容 (If extraction failed and fallback is not allowed)
+                if (!fallbackToOriginal) {
+                    log('fetcher.noFallback', debug, {}, component);
+                    throw new Error("Content extraction failed, and fallback is disabled");
+                }
+                log('fetcher.usingOriginalContent', debug, {}, component);
+                return { content: html, metadata: null };
+            }
+        }
+        catch (error) {
+            log('fetcher.extractionError', debug, {
+                error: error instanceof Error ? error.message : String(error),
+                url
+            }, component);
+            // 如果不允许回退到原始内容，则抛出错误 (If fallback is not allowed, throw error)
+            if (!fallbackToOriginal) {
+                throw error;
+            }
+            // 回退到原始内容 (Fallback to original content)
+            log('fetcher.fallbackToOriginal', debug, {}, component);
+            return { content: html, metadata: null };
+        }
+    }
 }

package/dist/src/lib/fetchers/common/types.js

@@ -17,6 +17,15 @@ export const BrowserParamsSchema = z.object({
     autoDetectMode: z.boolean().optional(),
     closeBrowser: z.boolean().optional(),
 });
+/**
+ * 通用提取选项 (Common extraction options)
+ * 用于配置内容提取行为 (Used to configure content extraction behavior)
+ */
+export const ContentExtractionOptionsSchema = z.object({
+    extractContent: z.boolean().optional(),
+    includeMetadata: z.boolean().optional(),
+    fallbackToOriginal: z.boolean().optional()
+});
 /**
  * 请求参数模式 (Request parameters schema)
  */
@@ -42,4 +51,4 @@ export const RequestPayloadSchema = z.object({
     enableContentSplitting: z.boolean().optional(),
     chunkId: z.string().optional(),
     startCursor: z.number().optional(),
-}).merge(BrowserParamsSchema);
+}).merge(BrowserParamsSchema).merge(ContentExtractionOptionsSchema);

package/dist/src/lib/fetchers/common/utils.js

@@ -51,7 +51,7 @@ export function getDomain(url) {
         const urlObj = new URL(url);
         return urlObj.hostname;
     }
-    catch (error) {
+    catch (_error) {
         return '';
     }
 }

package/dist/src/lib/fetchers/node/HttpClient.js

@@ -8,6 +8,7 @@ import { HttpProxyAgent } from "http-proxy-agent";
 import { HttpsProxyAgent } from "https-proxy-agent";
 import { log, COMPONENTS } from '../../logger.js';
 import { getRandomUserAgent, randomDelay, getSystemProxy } from '../common/utils.js';
+import { validateFetchUrl, UrlValidationError } from '../../utils/UrlValidator.js';
 /**
  * HTTP客户端类 (HTTP client class)
  * 处理HTTP请求、重定向和错误 (Handle HTTP requests, redirects and errors)
@@ -58,6 +59,7 @@ export class HttpClient {
      */
     static async handleDelay(noDelay, isRedirect, debug) {
         if (!noDelay && isRedirect) {
+            log('node.addingRandomDelay', debug, {}, COMPONENTS.NODE_FETCH);
             await randomDelay();
         }
     }
@@ -163,6 +165,7 @@ export class HttpClient {
         // 初始化重定向计数器 (Initialize redirect counter)
         let redirectCount = 0;
         let currentUrl = url;
+        await validateFetchUrl(currentUrl);
         // 记录请求开始时间 (Record request start time)
         const fetchStart = Date.now();
         // 创建AbortController用于超时控制 (Create AbortController for timeout control)
@@ -202,38 +205,61 @@ export class HttpClient {
                 log('node.responseStatus', debug, {
                     status: response.status,
                     statusText: response.statusText,
-                    headers: Object.fromEntries(response.headers.entries()),
+                    duration: Date.now() - fetchStart,
                 }, COMPONENTS.NODE_FETCH);
-                // 处理重定向 (Handle redirects)
+                // 处理错误响应 (Handle error response)
+                if (response.status >= 400) {
+                    await this.handleErrorResponse(response, debug);
+                }
+                // 检查是否需要重定向 (Check if redirect is needed)
                 if (response.status >= 300 && response.status < 400 && response.headers.has('location')) {
-                    redirectCount++;
                     // 获取重定向URL (Get redirect URL)
                     const location = response.headers.get('location');
-                    log('node.redirectingTo', debug, { location }, COMPONENTS.NODE_FETCH);
-                    // 构建完整的重定向URL (Build complete redirect URL)
                     currentUrl = this.buildRedirectUrl(location, currentUrl, debug);
+                    await validateFetchUrl(currentUrl);
+                    // 增加重定向计数 (Increase redirect count)
+                    redirectCount++;
+                    // 记录重定向 (Log redirect)
+                    log('node.redirecting', debug, {
+                        to: currentUrl,
+                        redirectCount,
+                    }, COMPONENTS.NODE_FETCH);
+                    // 关闭body读取器 (Close body reader)
+                    await response.text();
                     continue;
                 }
-                // 如果响应成功，返回响应 (If response is successful, return response)
-                if (response.ok) {
-                    log('node.requestSuccess', debug, {}, COMPONENTS.NODE_FETCH);
-                    return response;
-                }
-                else {
-                    // 处理错误响应 (Handle error response)
-                    return await this.handleErrorResponse(response, debug);
-                }
+                // 返回成功响应 (Return successful response)
+                clearTimeout(timeoutId);
+                return response;
             }
-            // 如果达到最大重定向次数，抛出错误 (If maximum redirects reached, throw error)
-            throw new Error(`Too many redirects (${maxRedirects})`);
+            // 重定向次数超过限制 (Redirect count exceeds limit)
+            const error = new Error(`Too many redirects (${maxRedirects})`);
+            error.code = 'EMAXREDIRECTS';
+            throw error;
         }
         catch (error) {
-            // 处理请求错误 (Handle request error)
-            this.handleRequestError(error, fetchStart, timeout, redirectCount, maxRedirects, debug);
-        }
-        finally {
-            // 清除超时定时器 (Clear timeout timer)
             clearTimeout(timeoutId);
+            if (error instanceof UrlValidationError) {
+                throw error;
+            }
+            return this.handleRequestError(error, fetchStart, timeout, redirectCount, maxRedirects, debug);
         }
     }
+    /**
+     * 计算代理通道等待时间 (Calculate proxy channel wait time)
+     * @param proxyType 代理类型 (Proxy type)
+     * @param proxy 代理服务器 (Proxy server)
+     * @param _debug 调试模式 (Debug mode)
+     * @returns 等待时间（毫秒） (Wait time in milliseconds)
+     */
+    static calculateProxyDelay(proxyType, proxy, _debug) {
+        // 根据代理类型和代理服务器地址计算合适的延迟时间
+        if (!proxy)
+            return 0;
+        // 基础延迟根据代理类型不同
+        const baseDelay = proxyType === 'http' ? 2000 : 3000;
+        // 计算延迟变量 (0-1000ms)
+        const variableDelay = Math.floor(Math.random() * 1000);
+        return baseDelay + variableDelay;
+    }
 }

package/dist/src/lib/fetchers/node/NodeFetcher.js

@@ -9,6 +9,7 @@ import { ErrorHandler } from '../../utils/ErrorHandler.js';
 import { ContentSizeManager } from '../../utils/ContentSizeManager.js';
 import { BaseFetcher } from "../common/BaseFetcher.js";
 import { ContentProcessor } from "../../utils/ContentProcessor.js";
+// import { ContentExtractor } from '../../utils/ContentExtractor.js';
 /**
  * Node模式获取器类 (Node mode fetcher class)
  * 使用node-fetch实现标准模式的网页获取 (Implements webpage fetching in standard mode using node-fetch)
@@ -20,7 +21,7 @@ export class NodeFetcher extends BaseFetcher {
      * @returns HTML内容 (HTML content)
      */
     async html(requestPayload) {
-        const { debug = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
+        const { debug = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0, extractContent = false, includeMetadata = true, fallbackToOriginal = true } = requestPayload;
         // 如果提供了分段ID和起始游标，则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
         if (chunkId && startCursor !== undefined) {
             return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.NODE_FETCH);
@@ -33,13 +34,28 @@ export class NodeFetcher extends BaseFetcher {
             log('node.readingText', debug, {}, COMPONENTS.NODE_FETCH);
             const html = await response.text();
             log('node.htmlContentLength', debug, { length: html.length }, COMPONENTS.NODE_FETCH);
+            // 处理内容提取 (Process content extraction)
+            const { content: processedContent, metadata } = this.processWithContentExtraction(html, requestPayload.url, {
+                extractContent,
+                includeMetadata,
+                fallbackToOriginal
+            }, debug, COMPONENTS.NODE_FETCH);
             // 检查内容大小并处理 (Check content size and process)
-            const chunkingResult = this.handleContentChunking(html, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.NODE_FETCH);
+            const chunkingResult = this.handleContentChunking(processedContent, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.NODE_FETCH);
             if (chunkingResult) {
+                // 如果有元数据，添加到分块结果中 (If there is metadata, add it to the chunk result)
+                if (metadata) {
+                    chunkingResult.metadata = metadata;
+                }
                 return chunkingResult;
             }
-            // 返回HTML内容 (Return HTML content)
-            return BaseFetcher.createSuccessResponse(html);
+            // 返回内容 (Return content)
+            const result = BaseFetcher.createSuccessResponse(processedContent);
+            // 如果有元数据，添加到结果中 (If there is metadata, add it to the result)
+            if (metadata) {
+                result.metadata = metadata;
+            }
+            return result;
         }
         catch (error) {
             // 使用ErrorHandler处理错误 (Use ErrorHandler to handle error)
@@ -151,8 +167,8 @@ export class NodeFetcher extends BaseFetcher {
             const response = await HttpClient.fetchWithRedirects(requestPayload);
             // 读取响应文本 (Read response text)
             const html = await response.text();
-            // 使用ContentProcessor将HTML转换为纯文本 (Use ContentProcessor to convert HTML to plain text)
-            const plainText = ContentProcessor.htmlToText(html, debug, COMPONENTS.NODE_FETCH);
+            // 将HTML转换为纯文本 (Convert HTML to plain text)
+            const plainText = ContentProcessor.htmlToText(html, debug);
             // 检查内容大小并处理 (Check content size and process)
             const chunkingResult = this.handleContentChunking(plainText, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.NODE_FETCH);
             if (chunkingResult) {
@@ -185,8 +201,8 @@ export class NodeFetcher extends BaseFetcher {
             const response = await HttpClient.fetchWithRedirects(requestPayload);
             // 读取响应文本 (Read response text)
             const html = await response.text();
-            // 使用ContentProcessor将HTML转换为Markdown (Use ContentProcessor to convert HTML to Markdown)
-            const markdown = ContentProcessor.htmlToMarkdown(html, debug, COMPONENTS.NODE_FETCH);
+            // 将HTML转换为Markdown (Convert HTML to Markdown)
+            const markdown = ContentProcessor.htmlToMarkdown(html, debug);
             // 检查内容大小并处理 (Check content size and process)
             const chunkingResult = this.handleContentChunking(markdown, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.NODE_FETCH);
             if (chunkingResult) {

package/dist/src/lib/i18n/index.js

@@ -63,7 +63,7 @@ const initializeI18n = () => {
         saveMissing: false, // 不保存缺失的翻译键 (Don't save missing translation keys)
         keySeparator: false, // 禁用键分隔符，使用完整的键路径 (Disable key separator, use full key path)
         nsSeparator: false, // 禁用命名空间分隔符 (Disable namespace separator)
-        missingKeyHandler: (lng, ns, key, fallbackValue) => {
+        missingKeyHandler: (_lng, _ns, _key, _fallbackValue) => {
             // 不输出任何日志，由上层调用决定是否输出 (Don't output any logs, let the upper-level call decide whether to output)
         }
     });
@@ -89,7 +89,7 @@ export const t = (key, options) => {
         const result = i18n.t(key, options);
         return result;
     }
-    catch (error) {
+    catch (_error) {
         return key; // 出错时返回原始键 (Return the original key when an error occurs)
     }
 };

package/dist/src/lib/i18n/keys/client.js

@@ -54,6 +54,7 @@ export const CLIENT_KEYS = (() => {
         responseStructure: keyGen('responseStructure'),
         parsedByteChunkInfo: keyGen('parsedByteChunkInfo'),
         parsedChunkInfo: keyGen('parsedChunkInfo'),
+        chunkInfoParsed: keyGen('chunkInfoParsed'),
         chunkLimitNotice: keyGen('chunkLimitNotice'),
         chunkLimitHint: keyGen('chunkLimitHint'),
         fetchingChunkProgress: keyGen('fetchingChunkProgress'),