@lmcc-dev/mult-fetch-mcp-server 1.2.1 → 1.3.1

package/dist/src/lib/utils/ContentProcessor.js

@@ -0,0 +1,134 @@
+/*
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import TurndownService from 'turndown';
+import { log, COMPONENTS } from '../logger.js';
+import { ToolError, ErrorType } from '../utils/errors.js';
+import { convert as htmlToText } from 'html-to-text';
+/**
+ * 内容处理器类 (Content processor class)
+ * 处理不同格式的内容转换和处理 (Process and convert content in different formats)
+ */
+export class ContentProcessor {
+    /**
+     * 将HTML转换为Markdown (Convert HTML to Markdown)
+     * @param html HTML内容 (HTML content)
+     * @param debug 是否开启调试 (Whether to enable debugging)
+     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
+     * @returns Markdown内容 (Markdown content)
+     */
+    static htmlToMarkdown(html, debug, component) {
+        // 始终使用COMPONENTS.PROCESSOR作为组件标识符 (Always use COMPONENTS.PROCESSOR as component identifier)
+        log('processor.creatingTurndown', debug, {}, COMPONENTS.PROCESSOR);
+        const turndownService = new TurndownService({
+            headingStyle: 'atx',
+            codeBlockStyle: 'fenced',
+            bulletListMarker: '-'
+        });
+        // 添加表格支持 (Add table support)
+        turndownService.addRule('tables', {
+            filter: ['table'],
+            replacement: function (content) {
+                const tableContent = content.trim();
+                return '\n\n' + tableContent + '\n\n';
+            }
+        });
+        // 将HTML转换为Markdown (Convert HTML to Markdown)
+        log('processor.convertingToMarkdown', debug, {}, COMPONENTS.PROCESSOR);
+        const markdown = turndownService.turndown(html);
+        log('processor.markdownContentLength', debug, { length: markdown.length }, COMPONENTS.PROCESSOR);
+        return markdown;
+    }
+    /**
+     * 将HTML转换为纯文本 (Convert HTML to plain text)
+     * @param html HTML内容 (HTML content)
+     * @param debug 是否开启调试 (Whether to enable debugging)
+     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
+     * @returns 纯文本内容 (Plain text content)
+     */
+    static htmlToText(html, debug, component) {
+        // 始终使用COMPONENTS.PROCESSOR作为组件标识符 (Always use COMPONENTS.PROCESSOR as component identifier)
+        log('processor.creatingHtmlToText', debug, {}, COMPONENTS.PROCESSOR);
+        // 配置html-to-text选项 (Configure html-to-text options)
+        const options = {
+            wordwrap: false,
+            selectors: [
+                { selector: 'a', options: { hideLinkHrefIfSameAsText: true } },
+                { selector: 'img', format: 'skip' },
+                { selector: 'table', options: { uppercaseHeaderCells: false } }
+            ]
+        };
+        // 将HTML转换为纯文本 (Convert HTML to plain text)
+        log('processor.convertingToText', debug, {}, COMPONENTS.PROCESSOR);
+        const text = htmlToText(html, options);
+        log('processor.textContentLength', debug, { length: text.length }, COMPONENTS.PROCESSOR);
+        return text;
+    }
+    /**
+     * 解析JSON字符串 (Parse JSON string)
+     * @param text JSON字符串 (JSON string)
+     * @param debug 是否开启调试 (Whether to enable debugging)
+     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
+     * @returns 解析结果 (Parse result - success or error with message)
+     */
+    static parseJson(text, debug, component) {
+        log('processor.parsingJson', debug, {}, COMPONENTS.PROCESSOR);
+        try {
+            const parsed = JSON.parse(text);
+            log('processor.jsonParsed', debug, {}, COMPONENTS.PROCESSOR);
+            return { success: true, result: parsed };
+        }
+        catch (parseError) {
+            const textPreview = text.length > 100 ? `${text.substring(0, 100)}...` : text;
+            const errorMessage = `Invalid JSON: ${parseError instanceof Error ? parseError.message : String(parseError)}. Text preview: "${textPreview}", length: ${text.length}`;
+            log('processor.jsonParseError', debug, {
+                error: String(parseError),
+                textPreview,
+                textLength: text.length
+            }, COMPONENTS.PROCESSOR);
+            return { success: false, error: errorMessage };
+        }
+    }
+    /**
+     * 处理文本内容，确保是UTF-8编码 (Process text content, ensure it's UTF-8 encoded)
+     * @param text 文本内容 (Text content)
+     * @param debug 是否开启调试 (Whether to enable debugging)
+     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
+     * @returns 处理后的文本 (Processed text)
+     */
+    static processTextContent(text, debug, component) {
+        log('processor.processingText', debug, { length: text.length }, COMPONENTS.PROCESSOR);
+        // 这里可以添加文本处理逻辑，如编码转换、去除特殊字符等
+        // (Add text processing logic here, such as encoding conversion, removing special characters, etc.)
+        return text;
+    }
+    /**
+     * 验证内容大小并处理过大的内容 (Validate content size and handle oversized content)
+     * @param content 内容 (Content)
+     * @param contentSizeLimit 内容大小限制 (Content size limit)
+     * @param debug 是否开启调试 (Whether to enable debugging)
+     * @param component 组件名称 (Component name for logging) - 仅用于向下兼容，实际会使用PROCESSOR组件
+     * @throws {ToolError} 如果内容太大且不允许分割 (If content is too large and splitting is not allowed)
+     */
+    static validateContentSize(content, contentSizeLimit, debug, component) {
+        const contentSize = content.length;
+        if (contentSize > contentSizeLimit) {
+            log('processor.contentTooLarge', debug, {
+                contentSize,
+                contentSizeLimit
+            }, COMPONENTS.PROCESSOR);
+            throw new ToolError(`Content size (${contentSize}) exceeds the allowed limit (${contentSizeLimit})`, ErrorType.TOOL_EXECUTION_ERROR, COMPONENTS.PROCESSOR, {
+                contentSize,
+                contentSizeLimit
+            });
+        }
+        else {
+            log('processor.contentSizeOk', debug, {
+                contentSize,
+                contentSizeLimit
+            }, COMPONENTS.PROCESSOR);
+        }
+    }
+}

package/dist/src/lib/utils/ContentSizeManager.js

@@ -0,0 +1,123 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { log, COMPONENTS } from '../logger.js';
+import { TemplateUtils } from './TemplateUtils.js';
+/**
+ * 内容大小管理器 (Content size manager)
+ * 用于处理内容大小限制，防止返回过大的内容 (Used to handle content size limits, prevent returning too large content)
+ */
+export class ContentSizeManager {
+    /**
+     * 默认大小限制，单位为字节 (Default size limit in bytes)
+     * 默认为50KB (Default is 50KB)
+     */
+    static DEFAULT_SIZE_LIMIT = 50 * 1024; // 50KB
+    /**
+     * 获取默认大小限制 (Get default size limit)
+     * @returns 默认大小限制，单位为字节 (Default size limit in bytes)
+     */
+    static getDefaultSizeLimit() {
+        return this.DEFAULT_SIZE_LIMIT;
+    }
+    /**
+     * 检查内容大小是否超过限制 (Check if content size exceeds limit)
+     * @param content 内容 (Content)
+     * @param sizeLimit 大小限制，单位为字节 (Size limit in bytes)
+     * @param debug 是否启用调试模式 (Whether debug mode is enabled)
+     * @returns 是否超过限制 (Whether exceeds limit)
+     */
+    static exceedsLimit(content, sizeLimit = this.DEFAULT_SIZE_LIMIT, debug = false) {
+        const contentSize = Buffer.byteLength(content, 'utf8');
+        const exceedsLimit = contentSize > sizeLimit;
+        if (debug) {
+            log('contentSize.checking', debug, {
+                contentSize: `${(contentSize / 1024).toFixed(2)}KB`,
+                limit: `${(sizeLimit / 1024).toFixed(2)}KB`,
+                exceedsLimit
+            }, COMPONENTS.CONTENT_SIZE);
+        }
+        return exceedsLimit;
+    }
+    /**
+     * 将内容分割成多个片段，不添加分段信息 (Split content into multiple chunks without adding chunk information)
+     * 这个方法用于内部处理，返回原始分段 (This method is for internal processing, returns raw chunks)
+     * @param content 原始内容 (Original content)
+     * @param sizeLimit 每个片段的大小限制，单位为字节 (Size limit for each chunk in bytes)
+     * @param debug 是否启用调试模式 (Whether debug mode is enabled)
+     * @returns 内容片段数组 (Array of content chunks)
+     */
+    static splitContentIntoRawChunks(content, sizeLimit = this.DEFAULT_SIZE_LIMIT, debug = false) {
+        if (!this.exceedsLimit(content, sizeLimit, debug)) {
+            return [content];
+        }
+        const contentSize = Buffer.byteLength(content, 'utf8');
+        log('contentSize.splitting', debug, {
+            originalSize: `${(contentSize / 1024).toFixed(2)}KB`,
+            chunkSize: `${(sizeLimit / 1024).toFixed(2)}KB`
+        }, COMPONENTS.CONTENT_SIZE);
+        // 分割内容 (Split content)
+        const chunks = [];
+        let currentChunk = '';
+        let currentSize = 0;
+        const chars = [...content];
+        for (const char of chars) {
+            const charSize = Buffer.byteLength(char, 'utf8');
+            // 如果添加这个字符会超过限制，创建新的片段 (If adding this character would exceed the limit, create a new chunk)
+            if (currentSize + charSize > sizeLimit && currentChunk.length > 0) {
+                chunks.push(currentChunk);
+                currentChunk = '';
+                currentSize = 0;
+            }
+            currentChunk += char;
+            currentSize += charSize;
+        }
+        // 添加最后一个片段 (Add the last chunk)
+        if (currentChunk.length > 0) {
+            chunks.push(currentChunk);
+        }
+        log('contentSize.splitComplete', debug, {
+            chunks: chunks.length,
+            avgChunkSize: `${(contentSize / chunks.length / 1024).toFixed(2)}KB`
+        }, COMPONENTS.CONTENT_SIZE);
+        return chunks;
+    }
+    /**
+     * 将内容分割成多个片段，并添加分段信息 (Split content into multiple chunks and add chunk information)
+     * @param content 原始内容 (Original content)
+     * @param sizeLimit 每个片段的大小限制，单位为字节 (Size limit for each chunk in bytes)
+     * @param debug 是否启用调试模式 (Whether debug mode is enabled)
+     * @param offset 当前偏移量，用于确定是首次请求还是后续请求 (Current offset, used to determine if it's initial or subsequent request)
+     * @returns 内容片段数组 (Array of content chunks)
+     */
+    static splitContentIntoChunks(content, sizeLimit = this.DEFAULT_SIZE_LIMIT, debug = false, offset = 0) {
+        // 计算内容总字节大小 (Calculate total size of content in bytes)
+        const totalBytes = Buffer.byteLength(content, 'utf8');
+        // 使用TemplateUtils中的常量和方法生成示例模板以计算大小
+        // (Use constants and methods from TemplateUtils to generate example templates for size calculation)
+        const sampleChunkId = 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx';
+        const samplePrompt = TemplateUtils.generateSizeBasedChunkPrompt(50000, // fetchedBytes 
+        totalBytes, sampleChunkId, totalBytes - 50000, // remainingBytes
+        Math.ceil((totalBytes - 50000) / sizeLimit), // estimatedRequests
+        sizeLimit, true // isFirstRequest
+        );
+        const maxChunkInfoSize = Buffer.byteLength(samplePrompt, 'utf8');
+        // 计算实际可用的分段大小 (Calculate actual available chunk size)
+        const effectiveChunkSize = sizeLimit - maxChunkInfoSize;
+        // 分割内容 (Split content)
+        const rawChunks = this.splitContentIntoRawChunks(content, effectiveChunkSize, debug);
+        // 不需要添加分段信息，这将在加载时动态添加
+        // (No need to add chunk information, it will be added dynamically when loading)
+        log('contentSize.splitIntoChunks', debug, {
+            totalChunks: rawChunks.length,
+            chunkCount: rawChunks.length,
+            chunkSize: effectiveChunkSize,
+            totalBytes,
+            effectiveChunkSize,
+            sizeLimit
+        }, COMPONENTS.CONTENT_SIZE);
+        return { chunks: rawChunks, totalBytes };
+    }
+}

package/dist/src/lib/utils/TemplateUtils.js

@@ -0,0 +1,71 @@
+/*
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+/**
+ * 模板工具类 (Template utilities class)
+ * 提供处理模板替换的通用方法 (Provides common methods for template replacement)
+ */
+export class TemplateUtils {
+    /**
+     * 替换内容中的占位符 (Replace placeholders in content)
+     * @param content 原始内容 (Original content)
+     * @param replacements 替换项 (Replacements)
+     * @returns 替换后的内容 (Content after replacement)
+     */
+    static replaceTemplateVariables(content, replacements) {
+        let result = content;
+        // 遍历所有替换项 (Iterate through all replacements)
+        for (const [key, value] of Object.entries(replacements)) {
+            // 创建正则表达式 (Create regular expression)
+            const regex = new RegExp(`{{${key}}}`, 'g');
+            // 替换所有匹配项 (Replace all matches)
+            result = result.replace(regex, value);
+        }
+        return result;
+    }
+    /**
+     * 系统提示相关的常量 (Constants related to system prompts)
+     */
+    static SYSTEM_NOTE = {
+        START: "=== SYSTEM NOTE ===",
+        END: "==================="
+    };
+    /**
+     * 生成基于字节大小的分块内容提示文本 (Generate size-based chunk prompt text)
+     * @param fetchedBytes 已获取的字节数 (Bytes already fetched)
+     * @param totalBytes 总字节数 (Total bytes)
+     * @param chunkId 块ID (Chunk ID)
+     * @param remainingBytes 剩余字节数 (Remaining bytes)
+     * @param estimatedRequests 预计需要的请求次数 (Estimated number of requests needed)
+     * @param currentSizeLimit 当前大小限制 (Current size limit)
+     * @param isFirstRequest 是否为首次请求 (Whether it's the first request)
+     * @returns 格式化的提示文本 (Formatted prompt text)
+     */
+    static generateSizeBasedChunkPrompt(fetchedBytes, totalBytes, chunkId, remainingBytes, estimatedRequests, currentSizeLimit, isFirstRequest = true) {
+        const prefix = isFirstRequest ? 'Content is too long and has been split. ' : '';
+        const fetchedPercent = Math.round((fetchedBytes / totalBytes) * 100);
+        return `\n\n${TemplateUtils.SYSTEM_NOTE.START}\n${prefix}You've retrieved ${fetchedBytes.toLocaleString()} bytes (${fetchedPercent}% of total ${totalBytes.toLocaleString()} bytes). ${remainingBytes.toLocaleString()} bytes remaining. With current contentSizeLimit=${currentSizeLimit.toLocaleString()}, approximately ${estimatedRequests} more requests needed to retrieve all content. To continue, use the same tool function with parameters chunkId="${chunkId}" and startCursor=${fetchedBytes}.\n${TemplateUtils.SYSTEM_NOTE.END}`;
+    }
+    /**
+     * 生成最后一个分块的基于字节大小的提示信息 (Generate size-based prompt for the last chunk)
+     * @param fetchedBytes 已获取的字节数 (Bytes already fetched)
+     * @param totalBytes 总字节数 (Total bytes)
+     * @param isFirstRequest 是否为首次请求 (Whether it's the first request)
+     * @returns 分段提示信息 (Chunk prompt information)
+     */
+    static generateSizeBasedLastChunkPrompt(fetchedBytes, totalBytes, isFirstRequest = true) {
+        const prefix = isFirstRequest ? 'Content is too long and has been split. ' : '';
+        const fetchedPercent = Math.round((fetchedBytes / totalBytes) * 100);
+        return `\n\n${TemplateUtils.SYSTEM_NOTE.START}\n${prefix}You've retrieved ${fetchedBytes.toLocaleString()} bytes (100% of total ${totalBytes.toLocaleString()} bytes).\nThis is the last part of the content.\n${TemplateUtils.SYSTEM_NOTE.END}`;
+    }
+    /**
+     * 检查内容是否已包含系统提示 (Check if content already contains system prompt)
+     * @param content 要检查的内容 (Content to check)
+     * @returns 是否包含系统提示 (Whether it contains system prompt)
+     */
+    static hasSystemPrompt(content) {
+        return content.includes(TemplateUtils.SYSTEM_NOTE.START) && content.includes(TemplateUtils.SYSTEM_NOTE.END);
+    }
+}

package/dist/src/lib/utils/errors.js

@@ -5,6 +5,7 @@
  */
 import { log, COMPONENTS } from '../logger.js';
 import { isAccessDeniedError, isNetworkError } from './errorDetection.js';
+import { BaseFetcher } from '../fetchers/common/BaseFetcher.js';
 // 扩展组件常量 (Extend component constants)
 const EXTENDED_COMPONENTS = {
     ...COMPONENTS,
@@ -90,14 +91,7 @@ export class FetchError extends Error {
      * @returns 标准响应对象 (Standard response object)
      */
     toResponse() {
-        return {
-            isError: true,
-            content: [
-                {
-                    text: this.message
-                }
-            ]
-        };
+        return BaseFetcher.createErrorResponse(this.message);
     }
 }
 /**

package/dist/tests/fetchers/plain-text.test.js

@@ -0,0 +1,146 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { describe, test, expect, vi } from 'vitest';
+import { fetchPlainText } from '../../src/lib/fetch.js';
+import { Fetcher } from '../../src/lib/fetchers/index.js';
+// Mock the Fetcher.plainText function
+vi.mock('../../src/lib/fetchers/index.js', () => ({
+    Fetcher: {
+        plainText: vi.fn()
+    }
+}));
+describe('plain-text fetcher 测试 (plain-text fetcher tests)', () => {
+    test('应该正确获取并转换内容为纯文本 (should correctly fetch and convert content to plain text)', async () => {
+        // 模拟 Fetcher.plainText 的返回值
+        Fetcher.plainText.mockResolvedValue({
+            content: [{ type: 'text', text: '<div>Example content</div>' }],
+            isError: false
+        });
+        // 调用被测试的函数
+        const result = await fetchPlainText({ url: 'https://example.com', startCursor: 0 });
+        // 验证 Fetcher.plainText 被调用并传递了正确的参数
+        expect(Fetcher.plainText).toHaveBeenCalledWith({
+            url: 'https://example.com',
+            startCursor: 0
+        });
+        // 验证返回结果正确
+        expect(result).toHaveProperty('content');
+        expect(result).toHaveProperty('isError', false);
+        expect(result.content[0]).toHaveProperty('type', 'text');
+        expect(result.content[0]).toHaveProperty('text', '<div>Example content</div>');
+    });
+    test('应该正确处理大型内容并返回分块信息 (should handle large content and return chunk information)', async () => {
+        // 模拟返回值，包含分块信息
+        const chunkInfo = {
+            isChunked: true,
+            chunkId: 'test-chunk-id',
+            startCursor: 0,
+            totalBytes: 10000,
+            fetchedBytes: 1000,
+            remainingBytes: 9000,
+            isLastChunk: false
+        };
+        Fetcher.plainText.mockResolvedValue({
+            content: [{
+                    type: 'text',
+                    text: 'Large content example',
+                    metadata: { chunkInfo }
+                }],
+            isError: false
+        });
+        // 调用被测试的函数
+        const result = await fetchPlainText({
+            url: 'https://example.com/large',
+            enableContentSplitting: true,
+            contentSizeLimit: 1000,
+            startCursor: 0
+        });
+        // 验证 Fetcher.plainText 被调用并传递了正确的参数
+        expect(Fetcher.plainText).toHaveBeenCalledWith({
+            url: 'https://example.com/large',
+            enableContentSplitting: true,
+            contentSizeLimit: 1000,
+            startCursor: 0
+        });
+        // 验证返回结果包含分块信息
+        expect(result).toHaveProperty('content');
+        expect(result).toHaveProperty('isError', false);
+        // 由于类型问题，使用toEqual代替单独的属性断言
+        expect(result.content[0]).toEqual({
+            type: 'text',
+            text: 'Large content example',
+            metadata: { chunkInfo }
+        });
+    });
+    test('应该正确处理后续分块请求 (should handle subsequent chunk requests)', async () => {
+        // 模拟返回值，包含后续分块信息
+        const chunkInfo = {
+            isChunked: true,
+            chunkId: 'test-chunk-id',
+            startCursor: 1000,
+            totalBytes: 10000,
+            fetchedBytes: 1000,
+            remainingBytes: 8000,
+            isLastChunk: false
+        };
+        Fetcher.plainText.mockResolvedValue({
+            content: [{
+                    type: 'text',
+                    text: 'Subsequent chunk content',
+                    metadata: { chunkInfo }
+                }],
+            isError: false
+        });
+        // 调用被测试的函数，包含chunkId和startCursor
+        const result = await fetchPlainText({
+            url: 'https://example.com/large',
+            chunkId: 'test-chunk-id',
+            startCursor: 1000
+        });
+        // 验证 Fetcher.plainText 被调用并传递了正确的参数
+        expect(Fetcher.plainText).toHaveBeenCalledWith({
+            url: 'https://example.com/large',
+            chunkId: 'test-chunk-id',
+            startCursor: 1000
+        });
+        // 验证返回结果包含分块信息
+        expect(result).toHaveProperty('content');
+        expect(result).toHaveProperty('isError', false);
+        // 由于类型问题，使用toEqual代替单独的属性断言
+        expect(result.content[0]).toEqual({
+            type: 'text',
+            text: 'Subsequent chunk content',
+            metadata: { chunkInfo }
+        });
+    });
+    test('当URL无效时应抛出错误 (should throw error when URL is invalid)', async () => {
+        // 模拟 Fetcher.plainText 抛出错误
+        const errorMessage = 'Invalid URL or unable to fetch';
+        Fetcher.plainText.mockRejectedValue(new Error(errorMessage));
+        // 将 fetchPlainText 包装在 try/catch 中来获取生成的错误响应
+        let result;
+        try {
+            await fetchPlainText({ url: 'https://example.com/error', startCursor: 0 });
+            // 如果没有抛出错误，测试应该失败
+            expect(true).toBe(false);
+        }
+        catch (error) {
+            // 处理错误，获取生成的错误响应
+            result = {
+                isError: true,
+                content: [{ type: 'text', text: errorMessage }]
+            };
+        }
+        // 验证 Fetcher.plainText 被调用并传递了正确的参数
+        expect(Fetcher.plainText).toHaveBeenCalledWith({
+            url: 'https://example.com/error',
+            startCursor: 0
+        });
+        // 验证返回结果包含错误信息
+        expect(result).toHaveProperty('isError', true);
+        expect(result.content[0]).toHaveProperty('text', errorMessage);
+    });
+});