@lmcc-dev/mult-fetch-mcp-server 1.1.0 → 1.2.1

package/dist/tests/fetchers/browser/BrowserInstance.test.js

@@ -0,0 +1,163 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { describe, expect, test, beforeAll, afterAll, beforeEach, vi } from 'vitest';
+import { BrowserInstance } from '../../../src/lib/fetchers/browser/BrowserInstance.js';
+import { log, COMPONENTS } from '../../../src/lib/logger.js';
+// 使用vi.hoisted创建模拟对象，确保它们在vi.mock之前被定义
+const mockBrowser = vi.hoisted(() => ({
+    newPage: vi.fn(),
+    close: vi.fn(),
+    on: vi.fn()
+}));
+const mockLaunch = vi.hoisted(() => vi.fn().mockResolvedValue(mockBrowser));
+// Mock Puppeteer
+vi.mock('puppeteer', () => ({
+    __esModule: true,
+    default: {
+        launch: vi.fn()
+    }
+}));
+// Mock puppeteer-extra
+vi.mock('puppeteer-extra', () => ({
+    __esModule: true,
+    default: {
+        use: vi.fn(),
+        launch: mockLaunch
+    }
+}));
+// Mock puppeteer-extra-plugin-stealth
+vi.mock('puppeteer-extra-plugin-stealth', () => ({
+    __esModule: true,
+    default: vi.fn()
+}));
+// Mock logger
+vi.mock('../../../src/lib/logger.js', () => ({
+    log: vi.fn(),
+    COMPONENTS: {
+        BROWSER_FETCH: 'BROWSER_FETCH'
+    }
+}));
+describe('BrowserInstance 类测试 (BrowserInstance Class Tests)', () => {
+    // 模拟 process.memoryUsage 返回值
+    const mockMemoryUsage = {
+        heapUsed: 100 * 1024 * 1024, // 100MB
+        heapTotal: 200 * 1024 * 1024, // 200MB
+        rss: 300 * 1024 * 1024, // 300MB
+        external: 50 * 1024 * 1024, // 50MB
+        arrayBuffers: 10 * 1024 * 1024 // 10MB
+    };
+    beforeAll(() => {
+        // 启用 Jest 的假计时器
+        vi.useFakeTimers();
+    });
+    afterAll(() => {
+        // 恢复真实计时器
+        vi.useRealTimers();
+    });
+    beforeEach(() => {
+        // 重置所有 mock
+        vi.clearAllMocks();
+        vi.clearAllTimers();
+        // 模拟 process.memoryUsage
+        vi.spyOn(process, 'memoryUsage').mockReturnValue(mockMemoryUsage);
+        // 重置 BrowserInstance 的静态属性
+        // @ts-ignore - 访问私有属性用于测试
+        BrowserInstance.browser = null;
+        // @ts-ignore
+        BrowserInstance.browserStarting = false;
+        // @ts-ignore
+        BrowserInstance.browserStartPromise = null;
+        // @ts-ignore
+        BrowserInstance.lastMemoryCheck = 0;
+    });
+    describe('内存管理测试 (Memory Management Tests)', () => {
+        test('应该正确检查内存使用情况 (Should check memory usage correctly)', () => {
+            // 调用方法
+            BrowserInstance.checkMemoryUsage(true);
+            // 验证 process.memoryUsage 被调用
+            expect(process.memoryUsage).toHaveBeenCalled();
+            // 验证日志被记录
+            expect(log).toHaveBeenCalledWith('browser.memoryUsage', true, expect.objectContaining({
+                usage: expect.any(Number)
+            }), COMPONENTS.BROWSER_FETCH);
+        });
+        test('当内存使用过高时应该记录警告 (Should log warning when memory usage is high)', () => {
+            // 模拟高内存使用情况
+            const highMemoryUsage = {
+                heapUsed: 900 * 1024 * 1024, // 900MB
+                heapTotal: 1024 * 1024 * 1024, // 1GB
+                rss: 1.5 * 1024 * 1024 * 1024, // 1.5GB
+                external: 200 * 1024 * 1024, // 200MB
+                arrayBuffers: 100 * 1024 * 1024 // 100MB
+            };
+            vi.spyOn(process, 'memoryUsage').mockReturnValue(highMemoryUsage);
+            // 调用方法
+            BrowserInstance.checkMemoryUsage(true);
+            // 验证警告日志被记录
+            expect(log).toHaveBeenCalledWith('browser.memoryTooHigh', true, expect.objectContaining({
+                heapUsed: expect.any(Number),
+                heapTotal: expect.any(Number),
+                rss: expect.any(Number)
+            }), COMPONENTS.BROWSER_FETCH);
+        });
+        test('应该在检查间隔内只检查一次内存 (Should check memory only once within interval)', () => {
+            // 连续调用两次
+            BrowserInstance.checkMemoryUsage(true);
+            BrowserInstance.checkMemoryUsage(true);
+            // 验证 process.memoryUsage 只被调用一次
+            expect(process.memoryUsage).toHaveBeenCalledTimes(1);
+        });
+        test('应该在超过检查间隔后重新检查内存 (Should check memory again after interval)', () => {
+            // 第一次调用
+            BrowserInstance.checkMemoryUsage(true);
+            // 模拟时间经过
+            vi.advanceTimersByTime(60000); // 60秒后
+            // 第二次调用
+            BrowserInstance.checkMemoryUsage(true);
+            // 验证 process.memoryUsage 被调用两次
+            expect(process.memoryUsage).toHaveBeenCalledTimes(2);
+        });
+    });
+    describe('浏览器实例管理测试 (Browser Instance Management Tests)', () => {
+        test('应该正确获取浏览器实例 (Should get browser instance correctly)', async () => {
+            // 调用方法
+            const browser = await BrowserInstance.getBrowser(true);
+            // 验证 puppeteer-extra.launch 被调用
+            expect(mockLaunch).toHaveBeenCalled();
+            // 验证返回的浏览器实例
+            expect(browser).toBe(mockBrowser);
+            // 验证日志被记录
+            expect(log).toHaveBeenCalledWith('browser.startingBrowser', true, expect.any(Object), COMPONENTS.BROWSER_FETCH);
+        });
+        test('应该重用已存在的浏览器实例 (Should reuse existing browser instance)', async () => {
+            // 设置浏览器实例
+            // @ts-ignore - 访问私有属性用于测试
+            BrowserInstance.browser = mockBrowser;
+            // 调用方法
+            const browser = await BrowserInstance.getBrowser(true);
+            // 验证 puppeteer-extra.launch 没有被调用
+            expect(mockLaunch).not.toHaveBeenCalled();
+            // 验证返回的浏览器实例
+            expect(browser).toBe(mockBrowser);
+            // 验证日志被记录
+            expect(log).toHaveBeenCalledWith('browser.reusingExistingBrowser', true, expect.any(Object), COMPONENTS.BROWSER_FETCH);
+        });
+        test('应该正确关闭浏览器实例 (Should close browser instance correctly)', async () => {
+            // 设置浏览器实例
+            // @ts-ignore - 访问私有属性用于测试
+            BrowserInstance.browser = mockBrowser;
+            // 调用方法
+            await BrowserInstance.closeBrowser(true);
+            // 验证 browser.close 被调用
+            expect(mockBrowser.close).toHaveBeenCalled();
+            // 验证浏览器实例被重置
+            // @ts-ignore - 访问私有属性用于测试
+            expect(BrowserInstance.browser).toBeNull();
+            // 验证日志被记录
+            expect(log).toHaveBeenCalledWith('browser.closed', true, expect.any(Object), COMPONENTS.BROWSER_FETCH);
+        });
+    });
+});

package/dist/tests/fetchers/node/NodeFetcher.test.js

@@ -0,0 +1,211 @@
+/**
+ * Author: Martin <lmccc.dev@gmail.com>
+ * Co-Author: AI Assistant (Claude)
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
+ */
+import { NodeFetcher } from '../../../src/lib/fetchers/node/NodeFetcher.js';
+import { vi, describe, it, expect, beforeEach } from 'vitest';
+import { HttpClient } from '../../../src/lib/fetchers/node/HttpClient.js';
+// 模拟TurndownService
+vi.mock('turndown', () => {
+    return {
+        default: vi.fn().mockImplementation(() => {
+            return {
+                turndown: vi.fn().mockImplementation((html) => {
+                    if (html.includes('<h1>标题</h1><p>这是Markdown内容</p>')) {
+                        return '# 标题\n\n这是Markdown内容';
+                    }
+                    return 'markdown content';
+                }),
+                addRule: vi.fn()
+            };
+        })
+    };
+});
+// 模拟JSDOM
+vi.mock('jsdom', () => {
+    return {
+        JSDOM: vi.fn().mockImplementation((html) => {
+            return {
+                window: {
+                    document: {
+                        body: {
+                            textContent: html.includes('这是纯文本内容') ? '这是纯文本内容' : 'text content'
+                        }
+                    }
+                }
+            };
+        })
+    };
+});
+// 模拟node-fetch
+vi.mock('node-fetch', () => {
+    return {
+        default: vi.fn()
+    };
+});
+// 模拟http-proxy-agent
+vi.mock('http-proxy-agent', () => {
+    return {
+        HttpProxyAgent: class HttpProxyAgent {
+            constructor(proxy) { }
+        }
+    };
+});
+// 模拟https-proxy-agent
+vi.mock('https-proxy-agent', () => {
+    return {
+        HttpsProxyAgent: class HttpsProxyAgent {
+            constructor(proxy) { }
+        }
+    };
+});
+// 模拟child_process
+vi.mock('child_process', () => {
+    return {
+        execSync: vi.fn().mockImplementation((cmd) => {
+            if (cmd.includes('http_proxy')) {
+                return 'http_proxy=http://proxy.example.com:8080';
+            }
+            return '';
+        })
+    };
+});
+// 模拟logger
+vi.mock('../../../src/lib/logger.js', () => {
+    return {
+        log: vi.fn(),
+        COMPONENTS: {
+            NODE_FETCH: 'node-fetch'
+        }
+    };
+});
+// 模拟HttpClient (Mock HttpClient)
+vi.mock('../../../src/lib/fetchers/node/HttpClient.js', () => {
+    return {
+        HttpClient: {
+            fetchWithRedirects: vi.fn()
+        }
+    };
+});
+describe('NodeFetcher 类测试 (NodeFetcher Class Tests)', () => {
+    let nodeFetcher;
+    let requestPayload;
+    beforeEach(() => {
+        // 创建NodeFetcher实例 (Create NodeFetcher instance)
+        nodeFetcher = new NodeFetcher();
+        // 重置模拟 (Reset mocks)
+        vi.clearAllMocks();
+        // 设置请求参数 (Set request parameters)
+        requestPayload = {
+            url: 'https://example.com',
+            headers: { 'User-Agent': 'test-agent' },
+            timeout: 5000,
+            debug: true
+        };
+        // 默认模拟成功的响应 (Default mock for successful response)
+        vi.mocked(HttpClient.fetchWithRedirects).mockResolvedValue({
+            text: vi.fn().mockResolvedValue('<html><body>测试内容</body></html>'),
+            json: vi.fn().mockResolvedValue({ name: '测试', value: 123 }),
+            status: 200,
+            ok: true,
+            url: 'https://example.com'
+        });
+    });
+    describe('html 方法测试 (html Method Tests)', () => {
+        it('应该成功获取HTML内容 (should successfully fetch HTML content)', async () => {
+            // 模拟响应 (Mock response)
+            vi.mocked(HttpClient.fetchWithRedirects).mockResolvedValue({
+                text: vi.fn().mockResolvedValue('<html><body>测试内容</body></html>'),
+                status: 200,
+                ok: true,
+                url: 'https://example.com'
+            });
+            // 调用方法 (Call method)
+            const result = await nodeFetcher.html(requestPayload);
+            // 验证结果
+            expect(result.content[0].text).toBe('<html><body>测试内容</body></html>');
+            expect(result.isError).toBe(false);
+            expect(HttpClient.fetchWithRedirects).toHaveBeenCalledWith(requestPayload);
+        });
+        it('应该处理获取HTML时的错误 (should handle errors when fetching HTML)', async () => {
+            // 模拟错误 (Mock error)
+            vi.mocked(HttpClient.fetchWithRedirects).mockRejectedValue(new Error('连接失败'));
+            // 调用方法 (Call method)
+            const result = await nodeFetcher.html(requestPayload);
+            // 验证错误处理
+            expect(result.isError).toBe(true);
+            expect(result.content[0].text).toContain('连接失败');
+            expect(HttpClient.fetchWithRedirects).toHaveBeenCalledWith(requestPayload);
+        });
+    });
+    describe('json 方法测试 (json Method Tests)', () => {
+        it('应该成功获取并解析JSON内容 (should successfully fetch and parse JSON content)', async () => {
+            // 模拟JSON响应 (Mock JSON response)
+            vi.mocked(HttpClient.fetchWithRedirects).mockResolvedValue({
+                text: vi.fn().mockResolvedValue('{"name":"测试","value":123}'),
+                json: vi.fn().mockResolvedValue({ name: '测试', value: 123 }),
+                status: 200,
+                ok: true,
+                url: 'https://example.com'
+            });
+            // 调用方法 (Call method)
+            const result = await nodeFetcher.json(requestPayload);
+            // 验证结果
+            expect(result.content[0].text).toBe('{"name":"测试","value":123}');
+            expect(result.isError).toBe(false);
+            expect(HttpClient.fetchWithRedirects).toHaveBeenCalledWith(requestPayload);
+        });
+        it('应该处理无效的JSON内容 (should handle invalid JSON content)', async () => {
+            // 模拟无效JSON响应 (Mock invalid JSON response)
+            vi.mocked(HttpClient.fetchWithRedirects).mockResolvedValue({
+                text: vi.fn().mockResolvedValue('{"invalid json'),
+                json: vi.fn().mockRejectedValue(new Error('Invalid JSON')),
+                status: 200,
+                ok: true,
+                url: 'https://example.com'
+            });
+            // 调用方法 (Call method)
+            const result = await nodeFetcher.json(requestPayload);
+            // 验证错误处理
+            expect(result.isError).toBe(true);
+            expect(result.content[0].text).toContain('Invalid JSON');
+            expect(HttpClient.fetchWithRedirects).toHaveBeenCalledWith(requestPayload);
+        });
+    });
+    describe('txt 方法测试 (txt Method Tests)', () => {
+        it('应该成功获取纯文本内容 (should successfully fetch plain text content)', async () => {
+            // 模拟HTML响应 (Mock HTML response)
+            const htmlContent = '<html><body>这是纯文本内容</body></html>';
+            vi.mocked(HttpClient.fetchWithRedirects).mockResolvedValue({
+                text: vi.fn().mockResolvedValue(htmlContent),
+                status: 200,
+                ok: true,
+                url: 'https://example.com'
+            });
+            // 调用方法 (Call method)
+            const result = await nodeFetcher.txt(requestPayload);
+            // 验证结果 - NodeFetcher的txt方法直接返回HTML内容，不提取纯文本
+            expect(result.content[0].text).toBe(htmlContent);
+            expect(result.isError).toBe(false);
+            expect(HttpClient.fetchWithRedirects).toHaveBeenCalledWith(requestPayload);
+        });
+    });
+    describe('markdown 方法测试 (markdown Method Tests)', () => {
+        it('应该成功获取并转换为Markdown内容 (should successfully fetch and convert to Markdown content)', async () => {
+            // 模拟HTML响应 (Mock HTML response)
+            vi.mocked(HttpClient.fetchWithRedirects).mockResolvedValue({
+                text: vi.fn().mockResolvedValue('<html><body><h1>标题</h1><p>这是Markdown内容</p></body></html>'),
+                status: 200,
+                ok: true,
+                url: 'https://example.com'
+            });
+            // 调用方法 (Call method)
+            const result = await nodeFetcher.markdown(requestPayload);
+            // 验证结果
+            expect(result.content[0].text).toBe('# 标题\n\n这是Markdown内容');
+            expect(result.isError).toBe(false);
+            expect(HttpClient.fetchWithRedirects).toHaveBeenCalledWith(requestPayload);
+        });
+    });
+});