@lmcc-dev/mult-fetch-mcp-server 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +80 -29
  2. package/README.zh.md +80 -29
  3. package/dist/i18n-test-report.json +2 -2
  4. package/dist/i18n-unused-keys-report.json +4 -4
  5. package/dist/src/client.js +528 -119
  6. package/dist/src/index.js +0 -0
  7. package/dist/src/lib/fetch.js +8 -0
  8. package/dist/src/lib/fetchers/browser/BrowserFetcher.js +139 -199
  9. package/dist/src/lib/fetchers/common/BaseFetcher.js +203 -0
  10. package/dist/src/lib/fetchers/common/types.js +4 -0
  11. package/dist/src/lib/fetchers/common/utils.js +1 -1
  12. package/dist/src/lib/fetchers/index.js +11 -0
  13. package/dist/src/lib/fetchers/node/NodeFetcher.js +91 -66
  14. package/dist/src/lib/i18n/keys/browser.js +40 -2
  15. package/dist/src/lib/i18n/keys/chunkManager.js +43 -0
  16. package/dist/src/lib/i18n/keys/client.js +53 -1
  17. package/dist/src/lib/i18n/keys/contentSize.js +20 -0
  18. package/dist/src/lib/i18n/keys/fetcher.js +88 -79
  19. package/dist/src/lib/i18n/keys/index.js +4 -0
  20. package/dist/src/lib/i18n/keys/node.js +7 -1
  21. package/dist/src/lib/i18n/keys/processor.js +30 -0
  22. package/dist/src/lib/i18n/keys/tools.js +5 -1
  23. package/dist/src/lib/i18n/keys/url.js +45 -0
  24. package/dist/src/lib/i18n/keys.js +25 -0
  25. package/dist/src/lib/i18n/locales/en/browser.js +52 -14
  26. package/dist/src/lib/i18n/locales/en/chunkManager.js +41 -0
  27. package/dist/src/lib/i18n/locales/en/client.js +62 -9
  28. package/dist/src/lib/i18n/locales/en/contentSize.js +17 -0
  29. package/dist/src/lib/i18n/locales/en/fetcher.js +50 -30
  30. package/dist/src/lib/i18n/locales/en/index.js +9 -1
  31. package/dist/src/lib/i18n/locales/en/node.js +26 -20
  32. package/dist/src/lib/i18n/locales/en/processor.js +28 -0
  33. package/dist/src/lib/i18n/locales/en/tools.js +5 -1
  34. package/dist/src/lib/i18n/locales/en/url.js +40 -0
  35. package/dist/src/lib/i18n/locales/en.js +51 -0
  36. package/dist/src/lib/i18n/locales/zh/browser.js +85 -47
  37. package/dist/src/lib/i18n/locales/zh/chunkManager.js +41 -0
  38. package/dist/src/lib/i18n/locales/zh/client.js +53 -1
  39. package/dist/src/lib/i18n/locales/zh/contentSize.js +17 -0
  40. package/dist/src/lib/i18n/locales/zh/fetcher.js +67 -47
  41. package/dist/src/lib/i18n/locales/zh/index.js +9 -1
  42. package/dist/src/lib/i18n/locales/zh/node.js +50 -44
  43. package/dist/src/lib/i18n/locales/zh/processor.js +28 -0
  44. package/dist/src/lib/i18n/locales/zh/tools.js +5 -1
  45. package/dist/src/lib/i18n/locales/zh/url.js +40 -0
  46. package/dist/src/lib/i18n/locales/zh.js +51 -0
  47. package/dist/src/lib/logger.js +11 -2
  48. package/dist/src/lib/server/browser.js +40 -8
  49. package/dist/src/lib/server/fetcher.js +96 -15
  50. package/dist/src/lib/server/tools.js +202 -52
  51. package/dist/src/lib/types.js +5 -0
  52. package/dist/src/lib/utils/ChunkManager.js +177 -0
  53. package/dist/src/lib/utils/ContentProcessor.js +134 -0
  54. package/dist/src/lib/utils/ContentSizeManager.js +123 -0
  55. package/dist/src/lib/utils/TemplateUtils.js +71 -0
  56. package/dist/src/lib/utils/errors.js +2 -8
  57. package/dist/src/mcp-server.js +0 -0
  58. package/dist/src/test-i18n.js +0 -0
  59. package/dist/tests/BrowserFetcher.test.js +0 -0
  60. package/dist/tests/NodeFetcher.test.js +0 -0
  61. package/dist/tests/client.test.js +0 -0
  62. package/dist/tests/fetch.test.js +0 -0
  63. package/dist/tests/fetchers/plain-text.test.js +146 -0
  64. package/dist/tests/i18n-missing-keys.js +0 -0
  65. package/dist/tests/i18n-remove-unused-keys.js +0 -0
  66. package/dist/tests/i18n-unused-keys.js +0 -0
  67. package/dist/tests/i18n.test.js +0 -0
  68. package/dist/tests/logger.test.js +0 -0
  69. package/dist/tests/mcp-server.test.js +0 -0
  70. package/dist/tests/server/fetcher.test.js +69 -15
  71. package/dist/tests/server/tools.test.js +165 -16
  72. package/dist/tests/setup.js +0 -0
  73. package/dist/tests/test-direct-client.js +113 -5
  74. package/dist/tests/test-i18n.js +0 -0
  75. package/dist/tests/test-mcp-methods.js +67 -1
  76. package/dist/tests/test-mcp.js +83 -3
  77. package/dist/tests/test-mini4k.js +90 -3
  78. package/dist/tests/types.test.js +0 -0
  79. package/dist/tests/utils/ChunkManager.test.js +163 -0
  80. package/dist/tests/utils/ContentSizeManager.test.js +170 -0
  81. package/dist/vitest.config.js +0 -0
  82. package/package.json +27 -23
@@ -24,7 +24,8 @@ async function testDirectClient() {
24
24
  method: 'fetch_html',
25
25
  params: {
26
26
  url: 'https://example.com',
27
- debug: true
27
+ debug: true,
28
+ startCursor: 0
28
29
  }
29
30
  },
30
31
  {
@@ -32,7 +33,8 @@ async function testDirectClient() {
32
33
  method: 'fetch_json',
33
34
  params: {
34
35
  url: 'https://jsonplaceholder.typicode.com/todos/1',
35
- debug: true
36
+ debug: true,
37
+ startCursor: 0
36
38
  }
37
39
  },
38
40
  {
@@ -40,7 +42,8 @@ async function testDirectClient() {
40
42
  method: 'fetch_txt',
41
43
  params: {
42
44
  url: 'https://example.com',
43
- debug: true
45
+ debug: true,
46
+ startCursor: 0
44
47
  }
45
48
  },
46
49
  {
@@ -48,7 +51,28 @@ async function testDirectClient() {
48
51
  method: 'fetch_markdown',
49
52
  params: {
50
53
  url: 'https://example.com',
51
- debug: true
54
+ debug: true,
55
+ startCursor: 0
56
+ }
57
+ },
58
+ {
59
+ name: 'HTML 转纯文本测试',
60
+ method: 'fetch_plaintext',
61
+ params: {
62
+ url: 'https://example.com',
63
+ debug: true,
64
+ startCursor: 0
65
+ }
66
+ },
67
+ {
68
+ name: '分块测试 - 第一个分块',
69
+ method: 'fetch_html',
70
+ params: {
71
+ url: 'https://example.com',
72
+ debug: true,
73
+ contentSizeLimit: 2000,
74
+ enableContentSplitting: true,
75
+ startCursor: 0
52
76
  }
53
77
  },
54
78
  {
@@ -57,10 +81,14 @@ async function testDirectClient() {
57
81
  params: {
58
82
  url: 'https://this-domain-does-not-exist-123456789.com',
59
83
  debug: true,
60
- timeout: 5000
84
+ timeout: 5000,
85
+ startCursor: 0
61
86
  }
62
87
  }
63
88
  ];
89
+ // 获取分块ID和字节位置,用于第二个分块测试
90
+ let chunkId = null;
91
+ let fetchedBytes = 0;
64
92
  // 依次执行每个测试用例
65
93
  for (const testCase of testCases) {
66
94
  console.log(`\n执行测试: ${testCase.name}`);
@@ -95,6 +123,86 @@ async function testDirectClient() {
95
123
  });
96
124
  // 输出结果
97
125
  console.log(`退出代码: ${result.code}`);
126
+ if (result.code === 0) {
127
+ console.log('测试成功!');
128
+ // 尝试解析JSON结果
129
+ try {
130
+ const jsonResult = JSON.parse(result.stdout);
131
+ console.log('结果类型:', typeof jsonResult);
132
+ if (typeof jsonResult === 'string') {
133
+ console.log('内容长度:', jsonResult.length);
134
+ console.log('内容预览:', jsonResult.substring(0, 200) + '...');
135
+ // 如果是分块测试,获取分块ID和字节位置
136
+ if (testCase.name === '分块测试 - 第一个分块') {
137
+ const systemNoteMatch = jsonResult.match(/===\s*SYSTEM\s*NOTE\s*===.*?====================/s);
138
+ if (systemNoteMatch) {
139
+ console.log('检测到系统注释,表示有更多分块可用');
140
+ const chunkIdMatch = systemNoteMatch[0].match(/chunkId:\s*"([^"]+)"/);
141
+ const fetchedBytesMatch = systemNoteMatch[0].match(/fetchedBytes:\s*(\d+)/);
142
+ if (chunkIdMatch && fetchedBytesMatch) {
143
+ chunkId = chunkIdMatch[1];
144
+ fetchedBytes = parseInt(fetchedBytesMatch[1]);
145
+ console.log(`获取到 chunkId: ${chunkId}, fetchedBytes: ${fetchedBytes}`);
146
+ }
147
+ }
148
+ }
149
+ }
150
+ else {
151
+ console.log('结果:', JSON.stringify(jsonResult, null, 2));
152
+ }
153
+ }
154
+ catch (error) {
155
+ console.log('原始输出:');
156
+ console.log(result.stdout);
157
+ }
158
+ }
159
+ else {
160
+ console.error('测试失败!');
161
+ console.error('错误输出:');
162
+ console.error(result.stderr);
163
+ }
164
+ }
165
+ // 如果有分块ID,测试获取第二个分块
166
+ if (chunkId && fetchedBytes > 0) {
167
+ console.log(`\n执行测试: 分块测试 - 第二个分块`);
168
+ const clientPath = path.resolve(rootDir, 'dist/src/client.js');
169
+ const secondChunkParams = {
170
+ url: 'https://example.com',
171
+ debug: true,
172
+ contentSizeLimit: 2000,
173
+ enableContentSplitting: true,
174
+ chunkId: chunkId,
175
+ startCursor: fetchedBytes
176
+ };
177
+ const args = [
178
+ clientPath,
179
+ 'fetch_html',
180
+ JSON.stringify(secondChunkParams)
181
+ ];
182
+ console.log(`运行命令: node ${args.join(' ')}`);
183
+ // 执行命令
184
+ const result = await new Promise((resolve) => {
185
+ const childProcess = spawn('node', args, {
186
+ stdio: ['ignore', 'pipe', 'pipe']
187
+ });
188
+ let stdout = '';
189
+ let stderr = '';
190
+ childProcess.stdout.on('data', (data) => {
191
+ stdout += data.toString();
192
+ });
193
+ childProcess.stderr.on('data', (data) => {
194
+ stderr += data.toString();
195
+ });
196
+ childProcess.on('close', (code) => {
197
+ resolve({
198
+ code: code || 0,
199
+ stdout,
200
+ stderr
201
+ });
202
+ });
203
+ });
204
+ // 输出结果
205
+ console.log(`退出代码: ${result.code}`);
98
206
  if (result.code === 0) {
99
207
  console.log('测试成功!');
100
208
  // 尝试解析JSON结果
File without changes
@@ -10,6 +10,8 @@ import { fileURLToPath } from 'url';
10
10
  // 获取当前文件的目录路径 (Get the directory path of the current file)
11
11
  const __filename = fileURLToPath(import.meta.url);
12
12
  const __dirname = path.dirname(__filename);
13
+ // 获取项目根目录 (Get the project root directory)
14
+ const rootDir = process.cwd();
13
15
  /**
14
16
  * 测试 MCP 方法 (Test MCP methods)
15
17
  * 测试 resources/list 和 prompts/list 方法 (Test resources/list and prompts/list methods)
@@ -17,7 +19,7 @@ const __dirname = path.dirname(__filename);
17
19
  async function testMcpMethods() {
18
20
  console.log('开始测试 MCP 方法...\n');
19
21
  // 创建服务器进程 (Create server process)
20
- const serverPath = path.resolve(path.dirname(__dirname), 'index.js');
22
+ const serverPath = path.resolve(rootDir, 'dist/index.js');
21
23
  console.log(`使用服务器路径: ${serverPath}`);
22
24
  // 创建客户端传输层 (Create client transport layer)
23
25
  const transport = new StdioClientTransport({
@@ -97,6 +99,70 @@ async function testMcpMethods() {
97
99
  catch (error) {
98
100
  console.error('prompts/get 测试失败:', error);
99
101
  }
102
+ // 测试 HTML 转纯文本功能 (Test HTML to plaintext conversion)
103
+ console.log('测试 HTML 转纯文本功能...');
104
+ try {
105
+ const plaintextResult = await client.callTool({
106
+ name: 'fetch_plaintext',
107
+ arguments: {
108
+ url: 'https://example.com',
109
+ debug: true,
110
+ startCursor: 0
111
+ }
112
+ });
113
+ if (plaintextResult.isError) {
114
+ console.error('获取纯文本失败:', plaintextResult.content[0].text);
115
+ }
116
+ else {
117
+ console.log(`获取纯文本成功,内容长度: ${plaintextResult.content[0].text.length} 字节`);
118
+ console.log(`纯文本内容预览: ${plaintextResult.content[0].text.substring(0, 100)}...`);
119
+ }
120
+ console.log('HTML 转纯文本测试成功!\n');
121
+ }
122
+ catch (error) {
123
+ console.error('HTML 转纯文本测试失败:', error);
124
+ }
125
+ // 测试分块功能 (Test chunking functionality)
126
+ console.log('测试分块功能...');
127
+ try {
128
+ const chunkResult = await client.callTool({
129
+ name: 'fetch_html',
130
+ arguments: {
131
+ url: 'https://example.com',
132
+ debug: true,
133
+ contentSizeLimit: 2000, // 设置较小的内容大小限制来触发分块
134
+ startCursor: 0,
135
+ enableContentSplitting: true
136
+ }
137
+ });
138
+ if (chunkResult.isError) {
139
+ console.error('分块测试失败:', chunkResult.content[0].text);
140
+ }
141
+ else {
142
+ console.log(`第一个分块获取成功,内容长度: ${chunkResult.content[0].text.length} 字节`);
143
+ // 检查是否有更多分块
144
+ const systemNoteMatch = chunkResult.content[0].text.match(/===\s*SYSTEM\s*NOTE\s*===.*?====================/s);
145
+ if (systemNoteMatch) {
146
+ console.log('检测到系统注释,表示有更多分块可用');
147
+ console.log(systemNoteMatch[0]);
148
+ // 提取 chunkId 和 fetchedBytes
149
+ const chunkIdMatch = systemNoteMatch[0].match(/chunkId:\s*"([^"]+)"/);
150
+ const fetchedBytesMatch = systemNoteMatch[0].match(/fetchedBytes:\s*(\d+)/);
151
+ if (chunkIdMatch && fetchedBytesMatch) {
152
+ const chunkId = chunkIdMatch[1];
153
+ const fetchedBytes = parseInt(fetchedBytesMatch[1]);
154
+ console.log(`获取到 chunkId: ${chunkId}, fetchedBytes: ${fetchedBytes}`);
155
+ }
156
+ }
157
+ else {
158
+ console.log('没有检测到更多分块');
159
+ }
160
+ }
161
+ console.log('分块功能测试成功!\n');
162
+ }
163
+ catch (error) {
164
+ console.error('分块功能测试失败:', error);
165
+ }
100
166
  }
101
167
  catch (error) {
102
168
  console.error('测试过程中发生错误:', error);
@@ -69,7 +69,8 @@ async function testMCP() {
69
69
  name: 'fetch_html',
70
70
  arguments: {
71
71
  url: 'https://example.com',
72
- debug: true
72
+ debug: true,
73
+ startCursor: 0
73
74
  }
74
75
  });
75
76
  if (htmlResult.isError) {
@@ -86,7 +87,8 @@ async function testMCP() {
86
87
  name: 'fetch_json',
87
88
  arguments: {
88
89
  url: 'https://jsonplaceholder.typicode.com/todos/1',
89
- debug: true
90
+ debug: true,
91
+ startCursor: 0
90
92
  }
91
93
  });
92
94
  if (jsonResult.isError) {
@@ -95,6 +97,83 @@ async function testMCP() {
95
97
  else {
96
98
  console.log(`获取 JSON 成功,内容: ${jsonResult.content[0].text}`);
97
99
  }
100
+ // 测试 fetch_plaintext 方法
101
+ console.log('\n测试 fetch_plaintext 方法...');
102
+ const plaintextResult = await client.callTool({
103
+ name: 'fetch_plaintext',
104
+ arguments: {
105
+ url: 'https://example.com',
106
+ debug: true,
107
+ startCursor: 0
108
+ }
109
+ });
110
+ if (plaintextResult.isError) {
111
+ console.error('获取纯文本失败:', plaintextResult.content[0].text);
112
+ }
113
+ else {
114
+ console.log(`获取纯文本成功,内容长度: ${plaintextResult.content[0].text.length} 字节`);
115
+ console.log(`纯文本内容预览: ${plaintextResult.content[0].text.substring(0, 100)}...`);
116
+ }
117
+ console.log('');
118
+ // 测试分块获取功能
119
+ console.log('\n测试分块获取功能...');
120
+ // 设置较小的 contentSizeLimit 触发分块
121
+ const chunkTestResult = await client.callTool({
122
+ name: 'fetch_html',
123
+ arguments: {
124
+ url: 'https://example.com',
125
+ debug: true,
126
+ contentSizeLimit: 2000, // 设置较小的内容大小限制来触发分块
127
+ startCursor: 0,
128
+ enableContentSplitting: true
129
+ }
130
+ });
131
+ if (chunkTestResult.isError) {
132
+ console.error('分块测试失败:', chunkTestResult.content[0].text);
133
+ }
134
+ else {
135
+ console.log('第一个分块获取成功:');
136
+ console.log(`内容长度: ${chunkTestResult.content[0].text.length} 字节`);
137
+ console.log(`内容预览: ${chunkTestResult.content[0].text.substring(0, 100)}...`);
138
+ // 检查是否包含系统注释,表示有更多分块
139
+ const systemNoteMatch = chunkTestResult.content[0].text.match(/===\s*SYSTEM\s*NOTE\s*===.*?====================/s);
140
+ if (systemNoteMatch) {
141
+ console.log('检测到系统注释,表示有更多分块可用');
142
+ console.log(systemNoteMatch[0]);
143
+ // 提取 chunkId 和 fetchedBytes
144
+ const chunkIdMatch = systemNoteMatch[0].match(/chunkId:\s*"([^"]+)"/);
145
+ const fetchedBytesMatch = systemNoteMatch[0].match(/fetchedBytes:\s*(\d+)/);
146
+ if (chunkIdMatch && fetchedBytesMatch) {
147
+ const chunkId = chunkIdMatch[1];
148
+ const fetchedBytes = parseInt(fetchedBytesMatch[1]);
149
+ console.log(`获取到 chunkId: ${chunkId}, fetchedBytes: ${fetchedBytes}`);
150
+ // 获取下一个分块
151
+ console.log('\n获取下一个分块...');
152
+ const nextChunkResult = await client.callTool({
153
+ name: 'fetch_html',
154
+ arguments: {
155
+ url: 'https://example.com',
156
+ debug: true,
157
+ chunkId: chunkId,
158
+ startCursor: fetchedBytes,
159
+ contentSizeLimit: 2000,
160
+ enableContentSplitting: true
161
+ }
162
+ });
163
+ if (nextChunkResult.isError) {
164
+ console.error('获取下一个分块失败:', nextChunkResult.content[0].text);
165
+ }
166
+ else {
167
+ console.log('下一个分块获取成功:');
168
+ console.log(`内容长度: ${nextChunkResult.content[0].text.length} 字节`);
169
+ console.log(`内容预览: ${nextChunkResult.content[0].text.substring(0, 100)}...`);
170
+ }
171
+ }
172
+ }
173
+ else {
174
+ console.log('没有检测到更多分块,内容可能小于设置的限制或分块功能未正确工作');
175
+ }
176
+ }
98
177
  // 关闭浏览器实例(如果有的话)
99
178
  console.log('\n关闭浏览器实例...');
100
179
  await client.callTool({
@@ -103,7 +182,8 @@ async function testMCP() {
103
182
  url: 'about:blank',
104
183
  debug: true,
105
184
  useBrowser: true,
106
- closeBrowser: true
185
+ closeBrowser: true,
186
+ startCursor: 0
107
187
  }
108
188
  });
109
189
  console.log('\n所有测试完成!');
@@ -65,7 +65,8 @@ async function testMini4k() {
65
65
  debug: true,
66
66
  timeout: 30000,
67
67
  waitForTimeout: 3000,
68
- scrollToBottom: false // 禁用滚动到底部
68
+ scrollToBottom: false, // 禁用滚动到底部
69
+ startCursor: 0
69
70
  }
70
71
  });
71
72
  console.log(`标准模式结果 - 是否错误: ${standardResult.isError}`);
@@ -86,7 +87,8 @@ async function testMini4k() {
86
87
  useBrowser: true,
87
88
  timeout: 30000,
88
89
  waitForTimeout: 3000,
89
- scrollToBottom: false // 禁用滚动到底部
90
+ scrollToBottom: false, // 禁用滚动到底部
91
+ startCursor: 0
90
92
  }
91
93
  });
92
94
  console.log(`浏览器模式结果 - 是否错误: ${browserResult.isError}`);
@@ -97,6 +99,90 @@ async function testMini4k() {
97
99
  else if (browserResult.isError) {
98
100
  console.log(`浏览器模式错误: ${browserResult.content[0].text}`);
99
101
  }
102
+ // 测试 HTML 转纯文本功能
103
+ console.log('\n测试 HTML 转纯文本功能...');
104
+ const plaintextResult = await client.callTool({
105
+ name: 'fetch_plaintext',
106
+ arguments: {
107
+ url: 'https://mini4k.com',
108
+ debug: true,
109
+ useBrowser: true, // 使用浏览器模式,因为标准模式可能会失败
110
+ timeout: 30000,
111
+ waitForTimeout: 3000,
112
+ startCursor: 0
113
+ }
114
+ });
115
+ console.log(`纯文本结果 - 是否错误: ${plaintextResult.isError}`);
116
+ if (!plaintextResult.isError && plaintextResult.content && plaintextResult.content[0]) {
117
+ console.log(`纯文本内容长度: ${plaintextResult.content[0].text.length} 字节`);
118
+ console.log(`纯文本内容预览: ${plaintextResult.content[0].text.substring(0, 200)}...`);
119
+ }
120
+ else if (plaintextResult.isError) {
121
+ console.log(`纯文本错误: ${plaintextResult.content[0].text}`);
122
+ }
123
+ // 测试分块功能
124
+ console.log('\n测试分块功能...');
125
+ const chunkResult = await client.callTool({
126
+ name: 'fetch_html',
127
+ arguments: {
128
+ url: 'https://mini4k.com',
129
+ debug: true,
130
+ useBrowser: true,
131
+ timeout: 30000,
132
+ waitForTimeout: 3000,
133
+ contentSizeLimit: 3000, // 设置较小的内容大小限制来触发分块
134
+ startCursor: 0,
135
+ enableContentSplitting: true
136
+ }
137
+ });
138
+ console.log(`分块结果 - 是否错误: ${chunkResult.isError}`);
139
+ if (!chunkResult.isError && chunkResult.content && chunkResult.content[0]) {
140
+ console.log(`第一个分块内容长度: ${chunkResult.content[0].text.length} 字节`);
141
+ console.log(`第一个分块内容预览: ${chunkResult.content[0].text.substring(0, 100)}...`);
142
+ // 检查是否有更多分块
143
+ const systemNoteMatch = chunkResult.content[0].text.match(/===\s*SYSTEM\s*NOTE\s*===.*?====================/s);
144
+ if (systemNoteMatch) {
145
+ console.log('检测到系统注释,表示有更多分块可用');
146
+ console.log(systemNoteMatch[0]);
147
+ // 提取 chunkId 和 fetchedBytes
148
+ const chunkIdMatch = systemNoteMatch[0].match(/chunkId:\s*"([^"]+)"/);
149
+ const fetchedBytesMatch = systemNoteMatch[0].match(/fetchedBytes:\s*(\d+)/);
150
+ if (chunkIdMatch && fetchedBytesMatch) {
151
+ const chunkId = chunkIdMatch[1];
152
+ const fetchedBytes = parseInt(fetchedBytesMatch[1]);
153
+ console.log(`获取到 chunkId: ${chunkId}, fetchedBytes: ${fetchedBytes}`);
154
+ // 获取第二个分块
155
+ console.log('\n获取第二个分块...');
156
+ const secondChunkResult = await client.callTool({
157
+ name: 'fetch_html',
158
+ arguments: {
159
+ url: 'https://mini4k.com',
160
+ debug: true,
161
+ useBrowser: true,
162
+ timeout: 30000,
163
+ chunkId: chunkId,
164
+ startCursor: fetchedBytes,
165
+ contentSizeLimit: 3000,
166
+ enableContentSplitting: true
167
+ }
168
+ });
169
+ console.log(`第二个分块结果 - 是否错误: ${secondChunkResult.isError}`);
170
+ if (!secondChunkResult.isError && secondChunkResult.content && secondChunkResult.content[0]) {
171
+ console.log(`第二个分块内容长度: ${secondChunkResult.content[0].text.length} 字节`);
172
+ console.log(`第二个分块内容预览: ${secondChunkResult.content[0].text.substring(0, 100)}...`);
173
+ }
174
+ else if (secondChunkResult.isError) {
175
+ console.log(`第二个分块错误: ${secondChunkResult.content[0].text}`);
176
+ }
177
+ }
178
+ }
179
+ else {
180
+ console.log('没有检测到更多分块');
181
+ }
182
+ }
183
+ else if (chunkResult.isError) {
184
+ console.log(`分块错误: ${chunkResult.content[0].text}`);
185
+ }
100
186
  // 关闭浏览器实例
101
187
  console.log('\n关闭浏览器实例...');
102
188
  await client.callTool({
@@ -105,7 +191,8 @@ async function testMini4k() {
105
191
  url: 'about:blank',
106
192
  debug: true,
107
193
  useBrowser: true,
108
- closeBrowser: true
194
+ closeBrowser: true,
195
+ startCursor: 0
109
196
  }
110
197
  });
111
198
  console.log('\n所有测试完成!');
File without changes
@@ -0,0 +1,163 @@
1
+ /**
2
+ * Author: Martin <lmccc.dev@gmail.com>
3
+ * Co-Author: AI Assistant (Claude)
4
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
5
+ */
6
+ import { describe, test, expect, vi, beforeEach } from 'vitest';
7
+ import { ChunkManager } from '../../src/lib/utils/ChunkManager.js';
8
+ // 模拟logger
9
+ vi.mock('../../src/lib/logger.js', () => ({
10
+ log: vi.fn(),
11
+ COMPONENTS: {
12
+ CHUNK_MANAGER: 'chunk-manager'
13
+ }
14
+ }));
15
+ // 模拟uuid
16
+ vi.mock('uuid', () => ({
17
+ v4: vi.fn().mockReturnValue('test-uuid')
18
+ }));
19
+ describe('ChunkManager 测试 (ChunkManager Tests)', () => {
20
+ beforeEach(() => {
21
+ // 清理mocks
22
+ vi.clearAllMocks();
23
+ // 重置ChunkManager的静态属性
24
+ // @ts-ignore - 访问私有属性用于测试
25
+ ChunkManager['chunks'] = new Map();
26
+ // @ts-ignore - 访问私有属性用于测试
27
+ ChunkManager['expirations'] = new Map();
28
+ // @ts-ignore - 访问私有属性用于测试
29
+ ChunkManager['sizeInfo'] = new Map();
30
+ });
31
+ test('storeChunks应正确存储分块内容和大小信息 (storeChunks should correctly store chunked content and size information)', () => {
32
+ // 测试数据
33
+ const chunks = ['chunk1', 'chunk2', 'chunk3'];
34
+ const totalBytes = 1000;
35
+ // Mock Buffer.byteLength 以返回固定的字节大小
36
+ const originalByteLength = Buffer.byteLength;
37
+ global.Buffer.byteLength = vi.fn()
38
+ .mockReturnValueOnce(200) // chunk1
39
+ .mockReturnValueOnce(300) // chunk2
40
+ .mockReturnValueOnce(500); // chunk3
41
+ // 调用测试方法
42
+ const chunkId = ChunkManager.storeChunks(chunks, totalBytes);
43
+ // 恢复原始函数
44
+ global.Buffer.byteLength = originalByteLength;
45
+ // 验证返回的chunkId
46
+ expect(chunkId).toBe('test-uuid');
47
+ // 验证内容被正确存储
48
+ // @ts-ignore - 访问私有属性用于测试
49
+ expect(ChunkManager['chunks'].get(chunkId)).toEqual(chunks);
50
+ // 验证大小信息被正确存储
51
+ // @ts-ignore - 访问私有属性用于测试
52
+ expect(ChunkManager['sizeInfo'].get(chunkId)).toEqual({
53
+ totalBytes: 1000,
54
+ fetchedBytes: [200, 300, 500]
55
+ });
56
+ // 验证过期时间被设置
57
+ // @ts-ignore - 访问私有属性用于测试
58
+ expect(ChunkManager['expirations'].has(chunkId)).toBe(true);
59
+ });
60
+ test('getChunkBySize应返回正确的分块内容和大小信息 (getChunkBySize should return correct chunk content and size information)', () => {
61
+ // 准备测试数据
62
+ const chunks = ['chunk1', 'chunk2', 'chunk3'];
63
+ const chunkId = 'test-chunk-id';
64
+ const fetchedBytes = [100, 200, 300];
65
+ const totalBytes = 600;
66
+ // 手动设置内部状态
67
+ // @ts-ignore - 访问私有属性用于测试
68
+ ChunkManager['chunks'].set(chunkId, chunks);
69
+ // @ts-ignore - 访问私有属性用于测试
70
+ ChunkManager['sizeInfo'].set(chunkId, { totalBytes, fetchedBytes });
71
+ // @ts-ignore - 访问私有属性用于测试
72
+ ChunkManager['expirations'].set(chunkId, Date.now() + 60000); // 设置为未过期
73
+ // 使用startCursor=0获取从头开始的内容
74
+ const resultStart = ChunkManager.getChunkBySize(chunkId, 0, 200);
75
+ // 验证从头开始的结果
76
+ expect(resultStart).not.toBeNull();
77
+ expect(resultStart?.content).toBe('chunk1chunk2');
78
+ expect(resultStart?.fetchedBytes).toBe(300);
79
+ expect(resultStart?.remainingBytes).toBe(300);
80
+ expect(resultStart?.isLastChunk).toBe(false);
81
+ expect(resultStart?.totalBytes).toBe(600);
82
+ // 使用startCursor=300获取从中间开始的内容
83
+ const resultMiddle = ChunkManager.getChunkBySize(chunkId, 300, 300);
84
+ // 验证从中间开始的结果
85
+ expect(resultMiddle).not.toBeNull();
86
+ expect(resultMiddle?.content).toBe('chunk3');
87
+ expect(resultMiddle?.fetchedBytes).toBe(600);
88
+ expect(resultMiddle?.remainingBytes).toBe(0);
89
+ expect(resultMiddle?.isLastChunk).toBe(true);
90
+ expect(resultMiddle?.totalBytes).toBe(600);
91
+ });
92
+ test('getChunkBySize应处理无效chunkId和startCursor (getChunkBySize should handle invalid chunkId and startCursor)', () => {
93
+ // 无效的chunkId
94
+ expect(ChunkManager.getChunkBySize('non-existent-id', 0)).toBeNull();
95
+ // 准备测试数据
96
+ const chunks = ['chunk1', 'chunk2', 'chunk3'];
97
+ const chunkId = 'test-chunk-id';
98
+ const fetchedBytes = [100, 200, 300];
99
+ const totalBytes = 600;
100
+ // 设置内部状态
101
+ // @ts-ignore - 访问私有属性用于测试
102
+ ChunkManager['chunks'].set(chunkId, chunks);
103
+ // @ts-ignore - 访问私有属性用于测试
104
+ ChunkManager['sizeInfo'].set(chunkId, { totalBytes, fetchedBytes });
105
+ // @ts-ignore - 访问私有属性用于测试
106
+ ChunkManager['expirations'].set(chunkId, Date.now() + 60000); // 设置为未过期
107
+ // 无效的startCursor - 负数
108
+ expect(ChunkManager.getChunkBySize(chunkId, -1)).toBeNull();
109
+ // 无效的startCursor - 超出总字节数
110
+ expect(ChunkManager.getChunkBySize(chunkId, 700)).toBeNull();
111
+ });
112
+ test('getSizeInfo应返回正确的大小信息 (getSizeInfo should return correct size information)', () => {
113
+ // 准备测试数据
114
+ const chunkId = 'test-chunk-id';
115
+ const fetchedBytes = [100, 200, 300];
116
+ const totalBytes = 600;
117
+ // 设置内部状态
118
+ // @ts-ignore - 访问私有属性用于测试
119
+ ChunkManager['sizeInfo'].set(chunkId, { totalBytes, fetchedBytes });
120
+ // 获取大小信息
121
+ const sizeInfo = ChunkManager.getSizeInfo(chunkId);
122
+ // 验证结果
123
+ expect(sizeInfo).not.toBeNull();
124
+ expect(sizeInfo?.totalBytes).toBe(600);
125
+ expect(sizeInfo?.fetchedBytes).toEqual([100, 200, 300]);
126
+ // 测试无效的chunkId
127
+ expect(ChunkManager.getSizeInfo('non-existent-id')).toBeNull();
128
+ });
129
+ test('cleanupExpiredChunks应清理过期的分块和大小信息 (cleanupExpiredChunks should clean up expired chunks and size information)', () => {
130
+ // 准备测试数据:过期的分块
131
+ const expiredChunkId = 'expired-chunk-id';
132
+ // @ts-ignore - 访问私有属性用于测试
133
+ ChunkManager['chunks'].set(expiredChunkId, ['expired-chunk']);
134
+ // @ts-ignore - 访问私有属性用于测试
135
+ ChunkManager['sizeInfo'].set(expiredChunkId, { totalBytes: 100, fetchedBytes: [100] });
136
+ // @ts-ignore - 设置为过期时间(过去的时间)
137
+ ChunkManager['expirations'].set(expiredChunkId, Date.now() - 1000);
138
+ // 准备测试数据:未过期的分块
139
+ const validChunkId = 'valid-chunk-id';
140
+ // @ts-ignore - 访问私有属性用于测试
141
+ ChunkManager['chunks'].set(validChunkId, ['valid-chunk']);
142
+ // @ts-ignore - 访问私有属性用于测试
143
+ ChunkManager['sizeInfo'].set(validChunkId, { totalBytes: 200, fetchedBytes: [200] });
144
+ // @ts-ignore - 设置为未过期时间(未来的时间)
145
+ ChunkManager['expirations'].set(validChunkId, Date.now() + 60000);
146
+ // 执行清理
147
+ // @ts-ignore - 访问私有方法用于测试
148
+ ChunkManager['cleanupExpiredChunks']();
149
+ // 验证结果:过期的分块应该被删除,未过期的分块仍然存在
150
+ // @ts-ignore - 访问私有属性用于测试
151
+ expect(ChunkManager['chunks'].has(expiredChunkId)).toBe(false);
152
+ // @ts-ignore - 访问私有属性用于测试
153
+ expect(ChunkManager['sizeInfo'].has(expiredChunkId)).toBe(false);
154
+ // @ts-ignore - 访问私有属性用于测试
155
+ expect(ChunkManager['expirations'].has(expiredChunkId)).toBe(false);
156
+ // @ts-ignore - 访问私有属性用于测试
157
+ expect(ChunkManager['chunks'].has(validChunkId)).toBe(true);
158
+ // @ts-ignore - 访问私有属性用于测试
159
+ expect(ChunkManager['sizeInfo'].has(validChunkId)).toBe(true);
160
+ // @ts-ignore - 访问私有属性用于测试
161
+ expect(ChunkManager['expirations'].has(validChunkId)).toBe(true);
162
+ });
163
+ });