@lmcc-dev/mult-fetch-mcp-server 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -29
- package/README.zh.md +80 -29
- package/dist/i18n-test-report.json +2 -2
- package/dist/i18n-unused-keys-report.json +4 -4
- package/dist/src/client.js +528 -119
- package/dist/src/index.js +0 -0
- package/dist/src/lib/fetch.js +8 -0
- package/dist/src/lib/fetchers/browser/BrowserFetcher.js +139 -199
- package/dist/src/lib/fetchers/common/BaseFetcher.js +203 -0
- package/dist/src/lib/fetchers/common/types.js +4 -0
- package/dist/src/lib/fetchers/common/utils.js +1 -1
- package/dist/src/lib/fetchers/index.js +11 -0
- package/dist/src/lib/fetchers/node/NodeFetcher.js +91 -66
- package/dist/src/lib/i18n/keys/browser.js +40 -2
- package/dist/src/lib/i18n/keys/chunkManager.js +43 -0
- package/dist/src/lib/i18n/keys/client.js +53 -1
- package/dist/src/lib/i18n/keys/contentSize.js +20 -0
- package/dist/src/lib/i18n/keys/fetcher.js +88 -79
- package/dist/src/lib/i18n/keys/index.js +4 -0
- package/dist/src/lib/i18n/keys/node.js +7 -1
- package/dist/src/lib/i18n/keys/processor.js +30 -0
- package/dist/src/lib/i18n/keys/tools.js +5 -1
- package/dist/src/lib/i18n/keys/url.js +45 -0
- package/dist/src/lib/i18n/keys.js +25 -0
- package/dist/src/lib/i18n/locales/en/browser.js +52 -14
- package/dist/src/lib/i18n/locales/en/chunkManager.js +41 -0
- package/dist/src/lib/i18n/locales/en/client.js +62 -9
- package/dist/src/lib/i18n/locales/en/contentSize.js +17 -0
- package/dist/src/lib/i18n/locales/en/fetcher.js +50 -30
- package/dist/src/lib/i18n/locales/en/index.js +9 -1
- package/dist/src/lib/i18n/locales/en/node.js +26 -20
- package/dist/src/lib/i18n/locales/en/processor.js +28 -0
- package/dist/src/lib/i18n/locales/en/tools.js +5 -1
- package/dist/src/lib/i18n/locales/en/url.js +40 -0
- package/dist/src/lib/i18n/locales/en.js +51 -0
- package/dist/src/lib/i18n/locales/zh/browser.js +85 -47
- package/dist/src/lib/i18n/locales/zh/chunkManager.js +41 -0
- package/dist/src/lib/i18n/locales/zh/client.js +53 -1
- package/dist/src/lib/i18n/locales/zh/contentSize.js +17 -0
- package/dist/src/lib/i18n/locales/zh/fetcher.js +67 -47
- package/dist/src/lib/i18n/locales/zh/index.js +9 -1
- package/dist/src/lib/i18n/locales/zh/node.js +50 -44
- package/dist/src/lib/i18n/locales/zh/processor.js +28 -0
- package/dist/src/lib/i18n/locales/zh/tools.js +5 -1
- package/dist/src/lib/i18n/locales/zh/url.js +40 -0
- package/dist/src/lib/i18n/locales/zh.js +51 -0
- package/dist/src/lib/logger.js +11 -2
- package/dist/src/lib/server/browser.js +40 -8
- package/dist/src/lib/server/fetcher.js +96 -15
- package/dist/src/lib/server/tools.js +202 -52
- package/dist/src/lib/types.js +5 -0
- package/dist/src/lib/utils/ChunkManager.js +177 -0
- package/dist/src/lib/utils/ContentProcessor.js +134 -0
- package/dist/src/lib/utils/ContentSizeManager.js +123 -0
- package/dist/src/lib/utils/TemplateUtils.js +71 -0
- package/dist/src/lib/utils/errors.js +2 -8
- package/dist/src/mcp-server.js +0 -0
- package/dist/src/test-i18n.js +0 -0
- package/dist/tests/BrowserFetcher.test.js +0 -0
- package/dist/tests/NodeFetcher.test.js +0 -0
- package/dist/tests/client.test.js +0 -0
- package/dist/tests/fetch.test.js +0 -0
- package/dist/tests/fetchers/plain-text.test.js +146 -0
- package/dist/tests/i18n-missing-keys.js +0 -0
- package/dist/tests/i18n-remove-unused-keys.js +0 -0
- package/dist/tests/i18n-unused-keys.js +0 -0
- package/dist/tests/i18n.test.js +0 -0
- package/dist/tests/logger.test.js +0 -0
- package/dist/tests/mcp-server.test.js +0 -0
- package/dist/tests/server/fetcher.test.js +69 -15
- package/dist/tests/server/tools.test.js +165 -16
- package/dist/tests/setup.js +0 -0
- package/dist/tests/test-direct-client.js +113 -5
- package/dist/tests/test-i18n.js +0 -0
- package/dist/tests/test-mcp-methods.js +67 -1
- package/dist/tests/test-mcp.js +83 -3
- package/dist/tests/test-mini4k.js +90 -3
- package/dist/tests/types.test.js +0 -0
- package/dist/tests/utils/ChunkManager.test.js +163 -0
- package/dist/tests/utils/ContentSizeManager.test.js +170 -0
- package/dist/vitest.config.js +0 -0
- package/package.json +27 -23
|
@@ -24,7 +24,8 @@ async function testDirectClient() {
|
|
|
24
24
|
method: 'fetch_html',
|
|
25
25
|
params: {
|
|
26
26
|
url: 'https://example.com',
|
|
27
|
-
debug: true
|
|
27
|
+
debug: true,
|
|
28
|
+
startCursor: 0
|
|
28
29
|
}
|
|
29
30
|
},
|
|
30
31
|
{
|
|
@@ -32,7 +33,8 @@ async function testDirectClient() {
|
|
|
32
33
|
method: 'fetch_json',
|
|
33
34
|
params: {
|
|
34
35
|
url: 'https://jsonplaceholder.typicode.com/todos/1',
|
|
35
|
-
debug: true
|
|
36
|
+
debug: true,
|
|
37
|
+
startCursor: 0
|
|
36
38
|
}
|
|
37
39
|
},
|
|
38
40
|
{
|
|
@@ -40,7 +42,8 @@ async function testDirectClient() {
|
|
|
40
42
|
method: 'fetch_txt',
|
|
41
43
|
params: {
|
|
42
44
|
url: 'https://example.com',
|
|
43
|
-
debug: true
|
|
45
|
+
debug: true,
|
|
46
|
+
startCursor: 0
|
|
44
47
|
}
|
|
45
48
|
},
|
|
46
49
|
{
|
|
@@ -48,7 +51,28 @@ async function testDirectClient() {
|
|
|
48
51
|
method: 'fetch_markdown',
|
|
49
52
|
params: {
|
|
50
53
|
url: 'https://example.com',
|
|
51
|
-
debug: true
|
|
54
|
+
debug: true,
|
|
55
|
+
startCursor: 0
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
name: 'HTML 转纯文本测试',
|
|
60
|
+
method: 'fetch_plaintext',
|
|
61
|
+
params: {
|
|
62
|
+
url: 'https://example.com',
|
|
63
|
+
debug: true,
|
|
64
|
+
startCursor: 0
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
name: '分块测试 - 第一个分块',
|
|
69
|
+
method: 'fetch_html',
|
|
70
|
+
params: {
|
|
71
|
+
url: 'https://example.com',
|
|
72
|
+
debug: true,
|
|
73
|
+
contentSizeLimit: 2000,
|
|
74
|
+
enableContentSplitting: true,
|
|
75
|
+
startCursor: 0
|
|
52
76
|
}
|
|
53
77
|
},
|
|
54
78
|
{
|
|
@@ -57,10 +81,14 @@ async function testDirectClient() {
|
|
|
57
81
|
params: {
|
|
58
82
|
url: 'https://this-domain-does-not-exist-123456789.com',
|
|
59
83
|
debug: true,
|
|
60
|
-
timeout: 5000
|
|
84
|
+
timeout: 5000,
|
|
85
|
+
startCursor: 0
|
|
61
86
|
}
|
|
62
87
|
}
|
|
63
88
|
];
|
|
89
|
+
// 获取分块ID和字节位置,用于第二个分块测试
|
|
90
|
+
let chunkId = null;
|
|
91
|
+
let fetchedBytes = 0;
|
|
64
92
|
// 依次执行每个测试用例
|
|
65
93
|
for (const testCase of testCases) {
|
|
66
94
|
console.log(`\n执行测试: ${testCase.name}`);
|
|
@@ -95,6 +123,86 @@ async function testDirectClient() {
|
|
|
95
123
|
});
|
|
96
124
|
// 输出结果
|
|
97
125
|
console.log(`退出代码: ${result.code}`);
|
|
126
|
+
if (result.code === 0) {
|
|
127
|
+
console.log('测试成功!');
|
|
128
|
+
// 尝试解析JSON结果
|
|
129
|
+
try {
|
|
130
|
+
const jsonResult = JSON.parse(result.stdout);
|
|
131
|
+
console.log('结果类型:', typeof jsonResult);
|
|
132
|
+
if (typeof jsonResult === 'string') {
|
|
133
|
+
console.log('内容长度:', jsonResult.length);
|
|
134
|
+
console.log('内容预览:', jsonResult.substring(0, 200) + '...');
|
|
135
|
+
// 如果是分块测试,获取分块ID和字节位置
|
|
136
|
+
if (testCase.name === '分块测试 - 第一个分块') {
|
|
137
|
+
const systemNoteMatch = jsonResult.match(/===\s*SYSTEM\s*NOTE\s*===.*?====================/s);
|
|
138
|
+
if (systemNoteMatch) {
|
|
139
|
+
console.log('检测到系统注释,表示有更多分块可用');
|
|
140
|
+
const chunkIdMatch = systemNoteMatch[0].match(/chunkId:\s*"([^"]+)"/);
|
|
141
|
+
const fetchedBytesMatch = systemNoteMatch[0].match(/fetchedBytes:\s*(\d+)/);
|
|
142
|
+
if (chunkIdMatch && fetchedBytesMatch) {
|
|
143
|
+
chunkId = chunkIdMatch[1];
|
|
144
|
+
fetchedBytes = parseInt(fetchedBytesMatch[1]);
|
|
145
|
+
console.log(`获取到 chunkId: ${chunkId}, fetchedBytes: ${fetchedBytes}`);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
console.log('结果:', JSON.stringify(jsonResult, null, 2));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
catch (error) {
|
|
155
|
+
console.log('原始输出:');
|
|
156
|
+
console.log(result.stdout);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
console.error('测试失败!');
|
|
161
|
+
console.error('错误输出:');
|
|
162
|
+
console.error(result.stderr);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
// 如果有分块ID,测试获取第二个分块
|
|
166
|
+
if (chunkId && fetchedBytes > 0) {
|
|
167
|
+
console.log(`\n执行测试: 分块测试 - 第二个分块`);
|
|
168
|
+
const clientPath = path.resolve(rootDir, 'dist/src/client.js');
|
|
169
|
+
const secondChunkParams = {
|
|
170
|
+
url: 'https://example.com',
|
|
171
|
+
debug: true,
|
|
172
|
+
contentSizeLimit: 2000,
|
|
173
|
+
enableContentSplitting: true,
|
|
174
|
+
chunkId: chunkId,
|
|
175
|
+
startCursor: fetchedBytes
|
|
176
|
+
};
|
|
177
|
+
const args = [
|
|
178
|
+
clientPath,
|
|
179
|
+
'fetch_html',
|
|
180
|
+
JSON.stringify(secondChunkParams)
|
|
181
|
+
];
|
|
182
|
+
console.log(`运行命令: node ${args.join(' ')}`);
|
|
183
|
+
// 执行命令
|
|
184
|
+
const result = await new Promise((resolve) => {
|
|
185
|
+
const childProcess = spawn('node', args, {
|
|
186
|
+
stdio: ['ignore', 'pipe', 'pipe']
|
|
187
|
+
});
|
|
188
|
+
let stdout = '';
|
|
189
|
+
let stderr = '';
|
|
190
|
+
childProcess.stdout.on('data', (data) => {
|
|
191
|
+
stdout += data.toString();
|
|
192
|
+
});
|
|
193
|
+
childProcess.stderr.on('data', (data) => {
|
|
194
|
+
stderr += data.toString();
|
|
195
|
+
});
|
|
196
|
+
childProcess.on('close', (code) => {
|
|
197
|
+
resolve({
|
|
198
|
+
code: code || 0,
|
|
199
|
+
stdout,
|
|
200
|
+
stderr
|
|
201
|
+
});
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
// 输出结果
|
|
205
|
+
console.log(`退出代码: ${result.code}`);
|
|
98
206
|
if (result.code === 0) {
|
|
99
207
|
console.log('测试成功!');
|
|
100
208
|
// 尝试解析JSON结果
|
package/dist/tests/test-i18n.js
CHANGED
|
File without changes
|
|
@@ -10,6 +10,8 @@ import { fileURLToPath } from 'url';
|
|
|
10
10
|
// 获取当前文件的目录路径 (Get the directory path of the current file)
|
|
11
11
|
const __filename = fileURLToPath(import.meta.url);
|
|
12
12
|
const __dirname = path.dirname(__filename);
|
|
13
|
+
// 获取项目根目录 (Get the project root directory)
|
|
14
|
+
const rootDir = process.cwd();
|
|
13
15
|
/**
|
|
14
16
|
* 测试 MCP 方法 (Test MCP methods)
|
|
15
17
|
* 测试 resources/list 和 prompts/list 方法 (Test resources/list and prompts/list methods)
|
|
@@ -17,7 +19,7 @@ const __dirname = path.dirname(__filename);
|
|
|
17
19
|
async function testMcpMethods() {
|
|
18
20
|
console.log('开始测试 MCP 方法...\n');
|
|
19
21
|
// 创建服务器进程 (Create server process)
|
|
20
|
-
const serverPath = path.resolve(
|
|
22
|
+
const serverPath = path.resolve(rootDir, 'dist/index.js');
|
|
21
23
|
console.log(`使用服务器路径: ${serverPath}`);
|
|
22
24
|
// 创建客户端传输层 (Create client transport layer)
|
|
23
25
|
const transport = new StdioClientTransport({
|
|
@@ -97,6 +99,70 @@ async function testMcpMethods() {
|
|
|
97
99
|
catch (error) {
|
|
98
100
|
console.error('prompts/get 测试失败:', error);
|
|
99
101
|
}
|
|
102
|
+
// 测试 HTML 转纯文本功能 (Test HTML to plaintext conversion)
|
|
103
|
+
console.log('测试 HTML 转纯文本功能...');
|
|
104
|
+
try {
|
|
105
|
+
const plaintextResult = await client.callTool({
|
|
106
|
+
name: 'fetch_plaintext',
|
|
107
|
+
arguments: {
|
|
108
|
+
url: 'https://example.com',
|
|
109
|
+
debug: true,
|
|
110
|
+
startCursor: 0
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
if (plaintextResult.isError) {
|
|
114
|
+
console.error('获取纯文本失败:', plaintextResult.content[0].text);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
console.log(`获取纯文本成功,内容长度: ${plaintextResult.content[0].text.length} 字节`);
|
|
118
|
+
console.log(`纯文本内容预览: ${plaintextResult.content[0].text.substring(0, 100)}...`);
|
|
119
|
+
}
|
|
120
|
+
console.log('HTML 转纯文本测试成功!\n');
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
console.error('HTML 转纯文本测试失败:', error);
|
|
124
|
+
}
|
|
125
|
+
// 测试分块功能 (Test chunking functionality)
|
|
126
|
+
console.log('测试分块功能...');
|
|
127
|
+
try {
|
|
128
|
+
const chunkResult = await client.callTool({
|
|
129
|
+
name: 'fetch_html',
|
|
130
|
+
arguments: {
|
|
131
|
+
url: 'https://example.com',
|
|
132
|
+
debug: true,
|
|
133
|
+
contentSizeLimit: 2000, // 设置较小的内容大小限制来触发分块
|
|
134
|
+
startCursor: 0,
|
|
135
|
+
enableContentSplitting: true
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
if (chunkResult.isError) {
|
|
139
|
+
console.error('分块测试失败:', chunkResult.content[0].text);
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
console.log(`第一个分块获取成功,内容长度: ${chunkResult.content[0].text.length} 字节`);
|
|
143
|
+
// 检查是否有更多分块
|
|
144
|
+
const systemNoteMatch = chunkResult.content[0].text.match(/===\s*SYSTEM\s*NOTE\s*===.*?====================/s);
|
|
145
|
+
if (systemNoteMatch) {
|
|
146
|
+
console.log('检测到系统注释,表示有更多分块可用');
|
|
147
|
+
console.log(systemNoteMatch[0]);
|
|
148
|
+
// 提取 chunkId 和 fetchedBytes
|
|
149
|
+
const chunkIdMatch = systemNoteMatch[0].match(/chunkId:\s*"([^"]+)"/);
|
|
150
|
+
const fetchedBytesMatch = systemNoteMatch[0].match(/fetchedBytes:\s*(\d+)/);
|
|
151
|
+
if (chunkIdMatch && fetchedBytesMatch) {
|
|
152
|
+
const chunkId = chunkIdMatch[1];
|
|
153
|
+
const fetchedBytes = parseInt(fetchedBytesMatch[1]);
|
|
154
|
+
console.log(`获取到 chunkId: ${chunkId}, fetchedBytes: ${fetchedBytes}`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
else {
|
|
158
|
+
console.log('没有检测到更多分块');
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
console.log('分块功能测试成功!\n');
|
|
162
|
+
}
|
|
163
|
+
catch (error) {
|
|
164
|
+
console.error('分块功能测试失败:', error);
|
|
165
|
+
}
|
|
100
166
|
}
|
|
101
167
|
catch (error) {
|
|
102
168
|
console.error('测试过程中发生错误:', error);
|
package/dist/tests/test-mcp.js
CHANGED
|
@@ -69,7 +69,8 @@ async function testMCP() {
|
|
|
69
69
|
name: 'fetch_html',
|
|
70
70
|
arguments: {
|
|
71
71
|
url: 'https://example.com',
|
|
72
|
-
debug: true
|
|
72
|
+
debug: true,
|
|
73
|
+
startCursor: 0
|
|
73
74
|
}
|
|
74
75
|
});
|
|
75
76
|
if (htmlResult.isError) {
|
|
@@ -86,7 +87,8 @@ async function testMCP() {
|
|
|
86
87
|
name: 'fetch_json',
|
|
87
88
|
arguments: {
|
|
88
89
|
url: 'https://jsonplaceholder.typicode.com/todos/1',
|
|
89
|
-
debug: true
|
|
90
|
+
debug: true,
|
|
91
|
+
startCursor: 0
|
|
90
92
|
}
|
|
91
93
|
});
|
|
92
94
|
if (jsonResult.isError) {
|
|
@@ -95,6 +97,83 @@ async function testMCP() {
|
|
|
95
97
|
else {
|
|
96
98
|
console.log(`获取 JSON 成功,内容: ${jsonResult.content[0].text}`);
|
|
97
99
|
}
|
|
100
|
+
// 测试 fetch_plaintext 方法
|
|
101
|
+
console.log('\n测试 fetch_plaintext 方法...');
|
|
102
|
+
const plaintextResult = await client.callTool({
|
|
103
|
+
name: 'fetch_plaintext',
|
|
104
|
+
arguments: {
|
|
105
|
+
url: 'https://example.com',
|
|
106
|
+
debug: true,
|
|
107
|
+
startCursor: 0
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
if (plaintextResult.isError) {
|
|
111
|
+
console.error('获取纯文本失败:', plaintextResult.content[0].text);
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
console.log(`获取纯文本成功,内容长度: ${plaintextResult.content[0].text.length} 字节`);
|
|
115
|
+
console.log(`纯文本内容预览: ${plaintextResult.content[0].text.substring(0, 100)}...`);
|
|
116
|
+
}
|
|
117
|
+
console.log('');
|
|
118
|
+
// 测试分块获取功能
|
|
119
|
+
console.log('\n测试分块获取功能...');
|
|
120
|
+
// 设置较小的 contentSizeLimit 触发分块
|
|
121
|
+
const chunkTestResult = await client.callTool({
|
|
122
|
+
name: 'fetch_html',
|
|
123
|
+
arguments: {
|
|
124
|
+
url: 'https://example.com',
|
|
125
|
+
debug: true,
|
|
126
|
+
contentSizeLimit: 2000, // 设置较小的内容大小限制来触发分块
|
|
127
|
+
startCursor: 0,
|
|
128
|
+
enableContentSplitting: true
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
if (chunkTestResult.isError) {
|
|
132
|
+
console.error('分块测试失败:', chunkTestResult.content[0].text);
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
console.log('第一个分块获取成功:');
|
|
136
|
+
console.log(`内容长度: ${chunkTestResult.content[0].text.length} 字节`);
|
|
137
|
+
console.log(`内容预览: ${chunkTestResult.content[0].text.substring(0, 100)}...`);
|
|
138
|
+
// 检查是否包含系统注释,表示有更多分块
|
|
139
|
+
const systemNoteMatch = chunkTestResult.content[0].text.match(/===\s*SYSTEM\s*NOTE\s*===.*?====================/s);
|
|
140
|
+
if (systemNoteMatch) {
|
|
141
|
+
console.log('检测到系统注释,表示有更多分块可用');
|
|
142
|
+
console.log(systemNoteMatch[0]);
|
|
143
|
+
// 提取 chunkId 和 fetchedBytes
|
|
144
|
+
const chunkIdMatch = systemNoteMatch[0].match(/chunkId:\s*"([^"]+)"/);
|
|
145
|
+
const fetchedBytesMatch = systemNoteMatch[0].match(/fetchedBytes:\s*(\d+)/);
|
|
146
|
+
if (chunkIdMatch && fetchedBytesMatch) {
|
|
147
|
+
const chunkId = chunkIdMatch[1];
|
|
148
|
+
const fetchedBytes = parseInt(fetchedBytesMatch[1]);
|
|
149
|
+
console.log(`获取到 chunkId: ${chunkId}, fetchedBytes: ${fetchedBytes}`);
|
|
150
|
+
// 获取下一个分块
|
|
151
|
+
console.log('\n获取下一个分块...');
|
|
152
|
+
const nextChunkResult = await client.callTool({
|
|
153
|
+
name: 'fetch_html',
|
|
154
|
+
arguments: {
|
|
155
|
+
url: 'https://example.com',
|
|
156
|
+
debug: true,
|
|
157
|
+
chunkId: chunkId,
|
|
158
|
+
startCursor: fetchedBytes,
|
|
159
|
+
contentSizeLimit: 2000,
|
|
160
|
+
enableContentSplitting: true
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
if (nextChunkResult.isError) {
|
|
164
|
+
console.error('获取下一个分块失败:', nextChunkResult.content[0].text);
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
console.log('下一个分块获取成功:');
|
|
168
|
+
console.log(`内容长度: ${nextChunkResult.content[0].text.length} 字节`);
|
|
169
|
+
console.log(`内容预览: ${nextChunkResult.content[0].text.substring(0, 100)}...`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
console.log('没有检测到更多分块,内容可能小于设置的限制或分块功能未正确工作');
|
|
175
|
+
}
|
|
176
|
+
}
|
|
98
177
|
// 关闭浏览器实例(如果有的话)
|
|
99
178
|
console.log('\n关闭浏览器实例...');
|
|
100
179
|
await client.callTool({
|
|
@@ -103,7 +182,8 @@ async function testMCP() {
|
|
|
103
182
|
url: 'about:blank',
|
|
104
183
|
debug: true,
|
|
105
184
|
useBrowser: true,
|
|
106
|
-
closeBrowser: true
|
|
185
|
+
closeBrowser: true,
|
|
186
|
+
startCursor: 0
|
|
107
187
|
}
|
|
108
188
|
});
|
|
109
189
|
console.log('\n所有测试完成!');
|
|
@@ -65,7 +65,8 @@ async function testMini4k() {
|
|
|
65
65
|
debug: true,
|
|
66
66
|
timeout: 30000,
|
|
67
67
|
waitForTimeout: 3000,
|
|
68
|
-
scrollToBottom: false // 禁用滚动到底部
|
|
68
|
+
scrollToBottom: false, // 禁用滚动到底部
|
|
69
|
+
startCursor: 0
|
|
69
70
|
}
|
|
70
71
|
});
|
|
71
72
|
console.log(`标准模式结果 - 是否错误: ${standardResult.isError}`);
|
|
@@ -86,7 +87,8 @@ async function testMini4k() {
|
|
|
86
87
|
useBrowser: true,
|
|
87
88
|
timeout: 30000,
|
|
88
89
|
waitForTimeout: 3000,
|
|
89
|
-
scrollToBottom: false // 禁用滚动到底部
|
|
90
|
+
scrollToBottom: false, // 禁用滚动到底部
|
|
91
|
+
startCursor: 0
|
|
90
92
|
}
|
|
91
93
|
});
|
|
92
94
|
console.log(`浏览器模式结果 - 是否错误: ${browserResult.isError}`);
|
|
@@ -97,6 +99,90 @@ async function testMini4k() {
|
|
|
97
99
|
else if (browserResult.isError) {
|
|
98
100
|
console.log(`浏览器模式错误: ${browserResult.content[0].text}`);
|
|
99
101
|
}
|
|
102
|
+
// 测试 HTML 转纯文本功能
|
|
103
|
+
console.log('\n测试 HTML 转纯文本功能...');
|
|
104
|
+
const plaintextResult = await client.callTool({
|
|
105
|
+
name: 'fetch_plaintext',
|
|
106
|
+
arguments: {
|
|
107
|
+
url: 'https://mini4k.com',
|
|
108
|
+
debug: true,
|
|
109
|
+
useBrowser: true, // 使用浏览器模式,因为标准模式可能会失败
|
|
110
|
+
timeout: 30000,
|
|
111
|
+
waitForTimeout: 3000,
|
|
112
|
+
startCursor: 0
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
console.log(`纯文本结果 - 是否错误: ${plaintextResult.isError}`);
|
|
116
|
+
if (!plaintextResult.isError && plaintextResult.content && plaintextResult.content[0]) {
|
|
117
|
+
console.log(`纯文本内容长度: ${plaintextResult.content[0].text.length} 字节`);
|
|
118
|
+
console.log(`纯文本内容预览: ${plaintextResult.content[0].text.substring(0, 200)}...`);
|
|
119
|
+
}
|
|
120
|
+
else if (plaintextResult.isError) {
|
|
121
|
+
console.log(`纯文本错误: ${plaintextResult.content[0].text}`);
|
|
122
|
+
}
|
|
123
|
+
// 测试分块功能
|
|
124
|
+
console.log('\n测试分块功能...');
|
|
125
|
+
const chunkResult = await client.callTool({
|
|
126
|
+
name: 'fetch_html',
|
|
127
|
+
arguments: {
|
|
128
|
+
url: 'https://mini4k.com',
|
|
129
|
+
debug: true,
|
|
130
|
+
useBrowser: true,
|
|
131
|
+
timeout: 30000,
|
|
132
|
+
waitForTimeout: 3000,
|
|
133
|
+
contentSizeLimit: 3000, // 设置较小的内容大小限制来触发分块
|
|
134
|
+
startCursor: 0,
|
|
135
|
+
enableContentSplitting: true
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
console.log(`分块结果 - 是否错误: ${chunkResult.isError}`);
|
|
139
|
+
if (!chunkResult.isError && chunkResult.content && chunkResult.content[0]) {
|
|
140
|
+
console.log(`第一个分块内容长度: ${chunkResult.content[0].text.length} 字节`);
|
|
141
|
+
console.log(`第一个分块内容预览: ${chunkResult.content[0].text.substring(0, 100)}...`);
|
|
142
|
+
// 检查是否有更多分块
|
|
143
|
+
const systemNoteMatch = chunkResult.content[0].text.match(/===\s*SYSTEM\s*NOTE\s*===.*?====================/s);
|
|
144
|
+
if (systemNoteMatch) {
|
|
145
|
+
console.log('检测到系统注释,表示有更多分块可用');
|
|
146
|
+
console.log(systemNoteMatch[0]);
|
|
147
|
+
// 提取 chunkId 和 fetchedBytes
|
|
148
|
+
const chunkIdMatch = systemNoteMatch[0].match(/chunkId:\s*"([^"]+)"/);
|
|
149
|
+
const fetchedBytesMatch = systemNoteMatch[0].match(/fetchedBytes:\s*(\d+)/);
|
|
150
|
+
if (chunkIdMatch && fetchedBytesMatch) {
|
|
151
|
+
const chunkId = chunkIdMatch[1];
|
|
152
|
+
const fetchedBytes = parseInt(fetchedBytesMatch[1]);
|
|
153
|
+
console.log(`获取到 chunkId: ${chunkId}, fetchedBytes: ${fetchedBytes}`);
|
|
154
|
+
// 获取第二个分块
|
|
155
|
+
console.log('\n获取第二个分块...');
|
|
156
|
+
const secondChunkResult = await client.callTool({
|
|
157
|
+
name: 'fetch_html',
|
|
158
|
+
arguments: {
|
|
159
|
+
url: 'https://mini4k.com',
|
|
160
|
+
debug: true,
|
|
161
|
+
useBrowser: true,
|
|
162
|
+
timeout: 30000,
|
|
163
|
+
chunkId: chunkId,
|
|
164
|
+
startCursor: fetchedBytes,
|
|
165
|
+
contentSizeLimit: 3000,
|
|
166
|
+
enableContentSplitting: true
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
console.log(`第二个分块结果 - 是否错误: ${secondChunkResult.isError}`);
|
|
170
|
+
if (!secondChunkResult.isError && secondChunkResult.content && secondChunkResult.content[0]) {
|
|
171
|
+
console.log(`第二个分块内容长度: ${secondChunkResult.content[0].text.length} 字节`);
|
|
172
|
+
console.log(`第二个分块内容预览: ${secondChunkResult.content[0].text.substring(0, 100)}...`);
|
|
173
|
+
}
|
|
174
|
+
else if (secondChunkResult.isError) {
|
|
175
|
+
console.log(`第二个分块错误: ${secondChunkResult.content[0].text}`);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
console.log('没有检测到更多分块');
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
else if (chunkResult.isError) {
|
|
184
|
+
console.log(`分块错误: ${chunkResult.content[0].text}`);
|
|
185
|
+
}
|
|
100
186
|
// 关闭浏览器实例
|
|
101
187
|
console.log('\n关闭浏览器实例...');
|
|
102
188
|
await client.callTool({
|
|
@@ -105,7 +191,8 @@ async function testMini4k() {
|
|
|
105
191
|
url: 'about:blank',
|
|
106
192
|
debug: true,
|
|
107
193
|
useBrowser: true,
|
|
108
|
-
closeBrowser: true
|
|
194
|
+
closeBrowser: true,
|
|
195
|
+
startCursor: 0
|
|
109
196
|
}
|
|
110
197
|
});
|
|
111
198
|
console.log('\n所有测试完成!');
|
package/dist/tests/types.test.js
CHANGED
|
File without changes
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Author: Martin <lmccc.dev@gmail.com>
|
|
3
|
+
* Co-Author: AI Assistant (Claude)
|
|
4
|
+
* Description: This code was collaboratively developed by Martin and AI Assistant.
|
|
5
|
+
*/
|
|
6
|
+
import { describe, test, expect, vi, beforeEach } from 'vitest';
|
|
7
|
+
import { ChunkManager } from '../../src/lib/utils/ChunkManager.js';
|
|
8
|
+
// 模拟logger
|
|
9
|
+
vi.mock('../../src/lib/logger.js', () => ({
|
|
10
|
+
log: vi.fn(),
|
|
11
|
+
COMPONENTS: {
|
|
12
|
+
CHUNK_MANAGER: 'chunk-manager'
|
|
13
|
+
}
|
|
14
|
+
}));
|
|
15
|
+
// 模拟uuid
|
|
16
|
+
vi.mock('uuid', () => ({
|
|
17
|
+
v4: vi.fn().mockReturnValue('test-uuid')
|
|
18
|
+
}));
|
|
19
|
+
describe('ChunkManager 测试 (ChunkManager Tests)', () => {
|
|
20
|
+
beforeEach(() => {
|
|
21
|
+
// 清理mocks
|
|
22
|
+
vi.clearAllMocks();
|
|
23
|
+
// 重置ChunkManager的静态属性
|
|
24
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
25
|
+
ChunkManager['chunks'] = new Map();
|
|
26
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
27
|
+
ChunkManager['expirations'] = new Map();
|
|
28
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
29
|
+
ChunkManager['sizeInfo'] = new Map();
|
|
30
|
+
});
|
|
31
|
+
test('storeChunks应正确存储分块内容和大小信息 (storeChunks should correctly store chunked content and size information)', () => {
|
|
32
|
+
// 测试数据
|
|
33
|
+
const chunks = ['chunk1', 'chunk2', 'chunk3'];
|
|
34
|
+
const totalBytes = 1000;
|
|
35
|
+
// Mock Buffer.byteLength 以返回固定的字节大小
|
|
36
|
+
const originalByteLength = Buffer.byteLength;
|
|
37
|
+
global.Buffer.byteLength = vi.fn()
|
|
38
|
+
.mockReturnValueOnce(200) // chunk1
|
|
39
|
+
.mockReturnValueOnce(300) // chunk2
|
|
40
|
+
.mockReturnValueOnce(500); // chunk3
|
|
41
|
+
// 调用测试方法
|
|
42
|
+
const chunkId = ChunkManager.storeChunks(chunks, totalBytes);
|
|
43
|
+
// 恢复原始函数
|
|
44
|
+
global.Buffer.byteLength = originalByteLength;
|
|
45
|
+
// 验证返回的chunkId
|
|
46
|
+
expect(chunkId).toBe('test-uuid');
|
|
47
|
+
// 验证内容被正确存储
|
|
48
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
49
|
+
expect(ChunkManager['chunks'].get(chunkId)).toEqual(chunks);
|
|
50
|
+
// 验证大小信息被正确存储
|
|
51
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
52
|
+
expect(ChunkManager['sizeInfo'].get(chunkId)).toEqual({
|
|
53
|
+
totalBytes: 1000,
|
|
54
|
+
fetchedBytes: [200, 300, 500]
|
|
55
|
+
});
|
|
56
|
+
// 验证过期时间被设置
|
|
57
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
58
|
+
expect(ChunkManager['expirations'].has(chunkId)).toBe(true);
|
|
59
|
+
});
|
|
60
|
+
test('getChunkBySize应返回正确的分块内容和大小信息 (getChunkBySize should return correct chunk content and size information)', () => {
|
|
61
|
+
// 准备测试数据
|
|
62
|
+
const chunks = ['chunk1', 'chunk2', 'chunk3'];
|
|
63
|
+
const chunkId = 'test-chunk-id';
|
|
64
|
+
const fetchedBytes = [100, 200, 300];
|
|
65
|
+
const totalBytes = 600;
|
|
66
|
+
// 手动设置内部状态
|
|
67
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
68
|
+
ChunkManager['chunks'].set(chunkId, chunks);
|
|
69
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
70
|
+
ChunkManager['sizeInfo'].set(chunkId, { totalBytes, fetchedBytes });
|
|
71
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
72
|
+
ChunkManager['expirations'].set(chunkId, Date.now() + 60000); // 设置为未过期
|
|
73
|
+
// 使用startCursor=0获取从头开始的内容
|
|
74
|
+
const resultStart = ChunkManager.getChunkBySize(chunkId, 0, 200);
|
|
75
|
+
// 验证从头开始的结果
|
|
76
|
+
expect(resultStart).not.toBeNull();
|
|
77
|
+
expect(resultStart?.content).toBe('chunk1chunk2');
|
|
78
|
+
expect(resultStart?.fetchedBytes).toBe(300);
|
|
79
|
+
expect(resultStart?.remainingBytes).toBe(300);
|
|
80
|
+
expect(resultStart?.isLastChunk).toBe(false);
|
|
81
|
+
expect(resultStart?.totalBytes).toBe(600);
|
|
82
|
+
// 使用startCursor=300获取从中间开始的内容
|
|
83
|
+
const resultMiddle = ChunkManager.getChunkBySize(chunkId, 300, 300);
|
|
84
|
+
// 验证从中间开始的结果
|
|
85
|
+
expect(resultMiddle).not.toBeNull();
|
|
86
|
+
expect(resultMiddle?.content).toBe('chunk3');
|
|
87
|
+
expect(resultMiddle?.fetchedBytes).toBe(600);
|
|
88
|
+
expect(resultMiddle?.remainingBytes).toBe(0);
|
|
89
|
+
expect(resultMiddle?.isLastChunk).toBe(true);
|
|
90
|
+
expect(resultMiddle?.totalBytes).toBe(600);
|
|
91
|
+
});
|
|
92
|
+
test('getChunkBySize应处理无效chunkId和startCursor (getChunkBySize should handle invalid chunkId and startCursor)', () => {
|
|
93
|
+
// 无效的chunkId
|
|
94
|
+
expect(ChunkManager.getChunkBySize('non-existent-id', 0)).toBeNull();
|
|
95
|
+
// 准备测试数据
|
|
96
|
+
const chunks = ['chunk1', 'chunk2', 'chunk3'];
|
|
97
|
+
const chunkId = 'test-chunk-id';
|
|
98
|
+
const fetchedBytes = [100, 200, 300];
|
|
99
|
+
const totalBytes = 600;
|
|
100
|
+
// 设置内部状态
|
|
101
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
102
|
+
ChunkManager['chunks'].set(chunkId, chunks);
|
|
103
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
104
|
+
ChunkManager['sizeInfo'].set(chunkId, { totalBytes, fetchedBytes });
|
|
105
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
106
|
+
ChunkManager['expirations'].set(chunkId, Date.now() + 60000); // 设置为未过期
|
|
107
|
+
// 无效的startCursor - 负数
|
|
108
|
+
expect(ChunkManager.getChunkBySize(chunkId, -1)).toBeNull();
|
|
109
|
+
// 无效的startCursor - 超出总字节数
|
|
110
|
+
expect(ChunkManager.getChunkBySize(chunkId, 700)).toBeNull();
|
|
111
|
+
});
|
|
112
|
+
test('getSizeInfo应返回正确的大小信息 (getSizeInfo should return correct size information)', () => {
|
|
113
|
+
// 准备测试数据
|
|
114
|
+
const chunkId = 'test-chunk-id';
|
|
115
|
+
const fetchedBytes = [100, 200, 300];
|
|
116
|
+
const totalBytes = 600;
|
|
117
|
+
// 设置内部状态
|
|
118
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
119
|
+
ChunkManager['sizeInfo'].set(chunkId, { totalBytes, fetchedBytes });
|
|
120
|
+
// 获取大小信息
|
|
121
|
+
const sizeInfo = ChunkManager.getSizeInfo(chunkId);
|
|
122
|
+
// 验证结果
|
|
123
|
+
expect(sizeInfo).not.toBeNull();
|
|
124
|
+
expect(sizeInfo?.totalBytes).toBe(600);
|
|
125
|
+
expect(sizeInfo?.fetchedBytes).toEqual([100, 200, 300]);
|
|
126
|
+
// 测试无效的chunkId
|
|
127
|
+
expect(ChunkManager.getSizeInfo('non-existent-id')).toBeNull();
|
|
128
|
+
});
|
|
129
|
+
test('cleanupExpiredChunks应清理过期的分块和大小信息 (cleanupExpiredChunks should clean up expired chunks and size information)', () => {
|
|
130
|
+
// 准备测试数据:过期的分块
|
|
131
|
+
const expiredChunkId = 'expired-chunk-id';
|
|
132
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
133
|
+
ChunkManager['chunks'].set(expiredChunkId, ['expired-chunk']);
|
|
134
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
135
|
+
ChunkManager['sizeInfo'].set(expiredChunkId, { totalBytes: 100, fetchedBytes: [100] });
|
|
136
|
+
// @ts-ignore - 设置为过期时间(过去的时间)
|
|
137
|
+
ChunkManager['expirations'].set(expiredChunkId, Date.now() - 1000);
|
|
138
|
+
// 准备测试数据:未过期的分块
|
|
139
|
+
const validChunkId = 'valid-chunk-id';
|
|
140
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
141
|
+
ChunkManager['chunks'].set(validChunkId, ['valid-chunk']);
|
|
142
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
143
|
+
ChunkManager['sizeInfo'].set(validChunkId, { totalBytes: 200, fetchedBytes: [200] });
|
|
144
|
+
// @ts-ignore - 设置为未过期时间(未来的时间)
|
|
145
|
+
ChunkManager['expirations'].set(validChunkId, Date.now() + 60000);
|
|
146
|
+
// 执行清理
|
|
147
|
+
// @ts-ignore - 访问私有方法用于测试
|
|
148
|
+
ChunkManager['cleanupExpiredChunks']();
|
|
149
|
+
// 验证结果:过期的分块应该被删除,未过期的分块仍然存在
|
|
150
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
151
|
+
expect(ChunkManager['chunks'].has(expiredChunkId)).toBe(false);
|
|
152
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
153
|
+
expect(ChunkManager['sizeInfo'].has(expiredChunkId)).toBe(false);
|
|
154
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
155
|
+
expect(ChunkManager['expirations'].has(expiredChunkId)).toBe(false);
|
|
156
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
157
|
+
expect(ChunkManager['chunks'].has(validChunkId)).toBe(true);
|
|
158
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
159
|
+
expect(ChunkManager['sizeInfo'].has(validChunkId)).toBe(true);
|
|
160
|
+
// @ts-ignore - 访问私有属性用于测试
|
|
161
|
+
expect(ChunkManager['expirations'].has(validChunkId)).toBe(true);
|
|
162
|
+
});
|
|
163
|
+
});
|