@lmcc-dev/mult-fetch-mcp-server 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -29
- package/README.zh.md +80 -29
- package/dist/i18n-test-report.json +2 -2
- package/dist/i18n-unused-keys-report.json +4 -4
- package/dist/src/client.js +522 -131
- package/dist/src/index.js +0 -0
- package/dist/src/lib/fetch.js +8 -0
- package/dist/src/lib/fetchers/browser/BrowserFetcher.js +139 -199
- package/dist/src/lib/fetchers/common/BaseFetcher.js +203 -0
- package/dist/src/lib/fetchers/common/types.js +4 -0
- package/dist/src/lib/fetchers/common/utils.js +1 -1
- package/dist/src/lib/fetchers/index.js +11 -0
- package/dist/src/lib/fetchers/node/NodeFetcher.js +91 -66
- package/dist/src/lib/i18n/keys/browser.js +40 -2
- package/dist/src/lib/i18n/keys/chunkManager.js +43 -0
- package/dist/src/lib/i18n/keys/client.js +53 -1
- package/dist/src/lib/i18n/keys/contentSize.js +20 -0
- package/dist/src/lib/i18n/keys/fetcher.js +88 -79
- package/dist/src/lib/i18n/keys/index.js +4 -0
- package/dist/src/lib/i18n/keys/node.js +7 -1
- package/dist/src/lib/i18n/keys/processor.js +30 -0
- package/dist/src/lib/i18n/keys/tools.js +5 -1
- package/dist/src/lib/i18n/keys/url.js +45 -0
- package/dist/src/lib/i18n/keys.js +25 -0
- package/dist/src/lib/i18n/locales/en/browser.js +52 -14
- package/dist/src/lib/i18n/locales/en/chunkManager.js +41 -0
- package/dist/src/lib/i18n/locales/en/client.js +62 -9
- package/dist/src/lib/i18n/locales/en/contentSize.js +17 -0
- package/dist/src/lib/i18n/locales/en/fetcher.js +50 -30
- package/dist/src/lib/i18n/locales/en/index.js +9 -1
- package/dist/src/lib/i18n/locales/en/node.js +26 -20
- package/dist/src/lib/i18n/locales/en/processor.js +28 -0
- package/dist/src/lib/i18n/locales/en/tools.js +5 -1
- package/dist/src/lib/i18n/locales/en/url.js +40 -0
- package/dist/src/lib/i18n/locales/en.js +51 -0
- package/dist/src/lib/i18n/locales/zh/browser.js +85 -47
- package/dist/src/lib/i18n/locales/zh/chunkManager.js +41 -0
- package/dist/src/lib/i18n/locales/zh/client.js +53 -1
- package/dist/src/lib/i18n/locales/zh/contentSize.js +17 -0
- package/dist/src/lib/i18n/locales/zh/fetcher.js +67 -47
- package/dist/src/lib/i18n/locales/zh/index.js +9 -1
- package/dist/src/lib/i18n/locales/zh/node.js +50 -44
- package/dist/src/lib/i18n/locales/zh/processor.js +28 -0
- package/dist/src/lib/i18n/locales/zh/tools.js +5 -1
- package/dist/src/lib/i18n/locales/zh/url.js +40 -0
- package/dist/src/lib/i18n/locales/zh.js +51 -0
- package/dist/src/lib/logger.js +11 -2
- package/dist/src/lib/server/fetcher.js +15 -5
- package/dist/src/lib/server/tools.js +202 -52
- package/dist/src/lib/types.js +5 -0
- package/dist/src/lib/utils/ChunkManager.js +177 -0
- package/dist/src/lib/utils/ContentProcessor.js +134 -0
- package/dist/src/lib/utils/ContentSizeManager.js +123 -0
- package/dist/src/lib/utils/TemplateUtils.js +71 -0
- package/dist/src/lib/utils/errors.js +2 -8
- package/dist/src/mcp-server.js +0 -0
- package/dist/src/test-i18n.js +0 -0
- package/dist/tests/BrowserFetcher.test.js +0 -0
- package/dist/tests/NodeFetcher.test.js +0 -0
- package/dist/tests/client.test.js +0 -0
- package/dist/tests/fetch.test.js +0 -0
- package/dist/tests/fetchers/plain-text.test.js +146 -0
- package/dist/tests/i18n-missing-keys.js +0 -0
- package/dist/tests/i18n-remove-unused-keys.js +0 -0
- package/dist/tests/i18n-unused-keys.js +0 -0
- package/dist/tests/i18n.test.js +0 -0
- package/dist/tests/logger.test.js +0 -0
- package/dist/tests/mcp-server.test.js +0 -0
- package/dist/tests/server/fetcher.test.js +69 -15
- package/dist/tests/server/tools.test.js +165 -16
- package/dist/tests/setup.js +0 -0
- package/dist/tests/test-direct-client.js +113 -5
- package/dist/tests/test-i18n.js +0 -0
- package/dist/tests/test-mcp-methods.js +67 -1
- package/dist/tests/test-mcp.js +83 -3
- package/dist/tests/test-mini4k.js +90 -3
- package/dist/tests/types.test.js +0 -0
- package/dist/tests/utils/ChunkManager.test.js +163 -0
- package/dist/tests/utils/ContentSizeManager.test.js +170 -0
- package/dist/vitest.config.js +0 -0
- package/package.json +28 -23
|
@@ -38,4 +38,8 @@ export const RequestPayloadSchema = z.object({
|
|
|
38
38
|
scrollToBottom: z.boolean().optional(),
|
|
39
39
|
closeBrowser: z.boolean().optional(),
|
|
40
40
|
saveCookies: z.boolean().optional(),
|
|
41
|
+
contentSizeLimit: z.number().optional(),
|
|
42
|
+
enableContentSplitting: z.boolean().optional(),
|
|
43
|
+
chunkId: z.string().optional(),
|
|
44
|
+
startCursor: z.number().optional(),
|
|
41
45
|
}).merge(BrowserParamsSchema);
|
|
@@ -77,7 +77,7 @@ export function getSystemProxy(useSystemProxy = true, debug = false, component =
|
|
|
77
77
|
log('fetcher.checkingProxyEnv', debug, {}, component);
|
|
78
78
|
for (const varName of proxyVars) {
|
|
79
79
|
log('fetcher.envVarValue', debug, {
|
|
80
|
-
|
|
80
|
+
name: varName,
|
|
81
81
|
value: process.env[varName] || t('fetcher.notSet')
|
|
82
82
|
}, component);
|
|
83
83
|
}
|
|
@@ -43,6 +43,17 @@ export class Fetcher {
|
|
|
43
43
|
const fetcher = FetcherFactory.createFetcher(requestPayload);
|
|
44
44
|
return await fetcher.txt(requestPayload);
|
|
45
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* 获取HTML并转换为纯文本 (Get HTML and convert to plain text)
|
|
48
|
+
* @param requestPayload 请求参数 (Request parameters)
|
|
49
|
+
* @returns 纯文本内容 (Plain text content)
|
|
50
|
+
*/
|
|
51
|
+
static async plainText(requestPayload) {
|
|
52
|
+
const { debug = false } = requestPayload;
|
|
53
|
+
log('fetcher.fetchingPlainText', debug, { url: requestPayload.url }, COMPONENTS.SERVER);
|
|
54
|
+
const fetcher = FetcherFactory.createFetcher(requestPayload);
|
|
55
|
+
return await fetcher.plainText(requestPayload);
|
|
56
|
+
}
|
|
46
57
|
/**
|
|
47
58
|
* 获取Markdown内容 (Get Markdown content)
|
|
48
59
|
* @param requestPayload 请求参数 (Request parameters)
|
|
@@ -3,22 +3,28 @@
|
|
|
3
3
|
* Co-Author: AI Assistant (Claude)
|
|
4
4
|
* Description: This code was collaboratively developed by Martin and AI Assistant.
|
|
5
5
|
*/
|
|
6
|
-
import TurndownService from "turndown";
|
|
7
6
|
import { log, COMPONENTS } from '../../logger.js';
|
|
8
7
|
import { HttpClient } from './HttpClient.js';
|
|
9
8
|
import { ErrorHandler } from '../../utils/ErrorHandler.js';
|
|
9
|
+
import { ContentSizeManager } from '../../utils/ContentSizeManager.js';
|
|
10
|
+
import { BaseFetcher } from "../common/BaseFetcher.js";
|
|
11
|
+
import { ContentProcessor } from "../../utils/ContentProcessor.js";
|
|
10
12
|
/**
|
|
11
13
|
* Node模式获取器类 (Node mode fetcher class)
|
|
12
14
|
* 使用node-fetch实现标准模式的网页获取 (Implements webpage fetching in standard mode using node-fetch)
|
|
13
15
|
*/
|
|
14
|
-
export class NodeFetcher {
|
|
16
|
+
export class NodeFetcher extends BaseFetcher {
|
|
15
17
|
/**
|
|
16
18
|
* 获取HTML内容 (Get HTML content)
|
|
17
19
|
* @param requestPayload 请求参数 (Request parameters)
|
|
18
20
|
* @returns HTML内容 (HTML content)
|
|
19
21
|
*/
|
|
20
22
|
async html(requestPayload) {
|
|
21
|
-
const { debug = false } = requestPayload;
|
|
23
|
+
const { debug = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
|
|
24
|
+
// 如果提供了分段ID和起始游标,则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
|
|
25
|
+
if (chunkId && startCursor !== undefined) {
|
|
26
|
+
return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.NODE_FETCH);
|
|
27
|
+
}
|
|
22
28
|
log('node.startingHtmlFetch', debug, {}, COMPONENTS.NODE_FETCH);
|
|
23
29
|
try {
|
|
24
30
|
// 执行请求 (Execute request)
|
|
@@ -27,20 +33,19 @@ export class NodeFetcher {
|
|
|
27
33
|
log('node.readingText', debug, {}, COMPONENTS.NODE_FETCH);
|
|
28
34
|
const html = await response.text();
|
|
29
35
|
log('node.htmlContentLength', debug, { length: html.length }, COMPONENTS.NODE_FETCH);
|
|
36
|
+
// 检查内容大小并处理 (Check content size and process)
|
|
37
|
+
const chunkingResult = this.handleContentChunking(html, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.NODE_FETCH);
|
|
38
|
+
if (chunkingResult) {
|
|
39
|
+
return chunkingResult;
|
|
40
|
+
}
|
|
30
41
|
// 返回HTML内容 (Return HTML content)
|
|
31
|
-
return
|
|
32
|
-
content: [{ type: 'text', text: html }],
|
|
33
|
-
isError: false
|
|
34
|
-
};
|
|
42
|
+
return BaseFetcher.createSuccessResponse(html);
|
|
35
43
|
}
|
|
36
44
|
catch (error) {
|
|
37
45
|
// 使用ErrorHandler处理错误 (Use ErrorHandler to handle error)
|
|
38
46
|
const errorMessage = ErrorHandler.handleError(error, COMPONENTS.NODE_FETCH, debug, { url: requestPayload.url, method: 'html' });
|
|
39
47
|
// 返回错误信息 (Return error message)
|
|
40
|
-
return {
|
|
41
|
-
content: [{ type: 'text', text: `Error fetching HTML: ${errorMessage}` }],
|
|
42
|
-
isError: true
|
|
43
|
-
};
|
|
48
|
+
return BaseFetcher.createErrorResponse(`Error fetching HTML: ${errorMessage}`);
|
|
44
49
|
}
|
|
45
50
|
}
|
|
46
51
|
/**
|
|
@@ -49,7 +54,11 @@ export class NodeFetcher {
|
|
|
49
54
|
* @returns JSON内容 (JSON content)
|
|
50
55
|
*/
|
|
51
56
|
async json(requestPayload) {
|
|
52
|
-
const { debug = false } = requestPayload;
|
|
57
|
+
const { debug = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
|
|
58
|
+
// 如果提供了分段ID和起始游标,则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
|
|
59
|
+
if (chunkId && startCursor !== undefined) {
|
|
60
|
+
return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.NODE_FETCH);
|
|
61
|
+
}
|
|
53
62
|
log('node.startingJsonFetch', debug, {}, COMPONENTS.NODE_FETCH);
|
|
54
63
|
try {
|
|
55
64
|
// 执行请求 (Execute request)
|
|
@@ -58,15 +67,16 @@ export class NodeFetcher {
|
|
|
58
67
|
const text = await response.text();
|
|
59
68
|
// 解析JSON (Parse JSON)
|
|
60
69
|
log('node.parsingJson', debug, {}, COMPONENTS.NODE_FETCH);
|
|
61
|
-
let json;
|
|
62
70
|
try {
|
|
63
|
-
|
|
71
|
+
JSON.parse(text);
|
|
64
72
|
log('node.jsonParsed', debug, {}, COMPONENTS.NODE_FETCH);
|
|
73
|
+
// 检查内容大小并处理 (Check content size and process)
|
|
74
|
+
const chunkingResult = this.handleContentChunking(text, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.NODE_FETCH);
|
|
75
|
+
if (chunkingResult) {
|
|
76
|
+
return chunkingResult;
|
|
77
|
+
}
|
|
65
78
|
// 返回JSON内容 (Return JSON content)
|
|
66
|
-
return
|
|
67
|
-
content: [{ type: 'text', text }],
|
|
68
|
-
isError: false
|
|
69
|
-
};
|
|
79
|
+
return BaseFetcher.createSuccessResponse(text);
|
|
70
80
|
}
|
|
71
81
|
catch (parseError) {
|
|
72
82
|
// 处理JSON解析错误 (Handle JSON parse error)
|
|
@@ -80,20 +90,14 @@ export class NodeFetcher {
|
|
|
80
90
|
textLength: text.length
|
|
81
91
|
}, COMPONENTS.NODE_FETCH);
|
|
82
92
|
// 返回错误信息 (Return error message)
|
|
83
|
-
return {
|
|
84
|
-
content: [{ type: 'text', text: `Error parsing JSON: ${error}` }],
|
|
85
|
-
isError: true
|
|
86
|
-
};
|
|
93
|
+
return BaseFetcher.createErrorResponse(`Error parsing JSON: ${error.message}`);
|
|
87
94
|
}
|
|
88
95
|
}
|
|
89
96
|
catch (error) {
|
|
90
97
|
// 使用ErrorHandler处理错误 (Use ErrorHandler to handle error)
|
|
91
98
|
const errorMessage = ErrorHandler.handleError(error, COMPONENTS.NODE_FETCH, debug, { url: requestPayload.url, method: 'json' });
|
|
92
99
|
// 返回错误信息 (Return error message)
|
|
93
|
-
return {
|
|
94
|
-
content: [{ type: 'text', text: `Error fetching JSON: ${errorMessage}` }],
|
|
95
|
-
isError: true
|
|
96
|
-
};
|
|
100
|
+
return BaseFetcher.createErrorResponse(`Error fetching JSON: ${errorMessage}`);
|
|
97
101
|
}
|
|
98
102
|
}
|
|
99
103
|
/**
|
|
@@ -102,7 +106,11 @@ export class NodeFetcher {
|
|
|
102
106
|
* @returns 纯文本内容 (Plain text content)
|
|
103
107
|
*/
|
|
104
108
|
async txt(requestPayload) {
|
|
105
|
-
const { debug = false } = requestPayload;
|
|
109
|
+
const { debug = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
|
|
110
|
+
// 如果提供了分段ID和起始游标,则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
|
|
111
|
+
if (chunkId && startCursor !== undefined) {
|
|
112
|
+
return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.NODE_FETCH);
|
|
113
|
+
}
|
|
106
114
|
log('node.startingTxtFetch', debug, {}, COMPONENTS.NODE_FETCH);
|
|
107
115
|
try {
|
|
108
116
|
// 执行请求 (Execute request)
|
|
@@ -111,20 +119,53 @@ export class NodeFetcher {
|
|
|
111
119
|
log('node.readingText', debug, {}, COMPONENTS.NODE_FETCH);
|
|
112
120
|
const text = await response.text();
|
|
113
121
|
log('node.textContentLength', debug, { length: text.length }, COMPONENTS.NODE_FETCH);
|
|
122
|
+
// 检查内容大小并处理 (Check content size and process)
|
|
123
|
+
const chunkingResult = this.handleContentChunking(text, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.NODE_FETCH);
|
|
124
|
+
if (chunkingResult) {
|
|
125
|
+
return chunkingResult;
|
|
126
|
+
}
|
|
114
127
|
// 返回纯文本内容 (Return plain text content)
|
|
115
|
-
return
|
|
116
|
-
content: [{ type: 'text', text }],
|
|
117
|
-
isError: false
|
|
118
|
-
};
|
|
128
|
+
return BaseFetcher.createSuccessResponse(text);
|
|
119
129
|
}
|
|
120
130
|
catch (error) {
|
|
121
131
|
// 使用ErrorHandler处理错误 (Use ErrorHandler to handle error)
|
|
122
132
|
const errorMessage = ErrorHandler.handleError(error, COMPONENTS.NODE_FETCH, debug, { url: requestPayload.url, method: 'txt' });
|
|
123
133
|
// 返回错误信息 (Return error message)
|
|
124
|
-
return {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
134
|
+
return BaseFetcher.createErrorResponse(`Error fetching text: ${errorMessage}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* 获取HTML并转换为纯文本 (Get HTML and convert to plain text)
|
|
139
|
+
* @param requestPayload 请求参数 (Request parameters)
|
|
140
|
+
* @returns 纯文本响应 (Plain text response)
|
|
141
|
+
*/
|
|
142
|
+
async plainText(requestPayload) {
|
|
143
|
+
const { debug = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
|
|
144
|
+
// 如果提供了分段ID和起始游标,则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
|
|
145
|
+
if (chunkId && startCursor !== undefined) {
|
|
146
|
+
return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.NODE_FETCH);
|
|
147
|
+
}
|
|
148
|
+
log('node.startingPlainTextFetch', debug, { url: requestPayload.url }, COMPONENTS.NODE_FETCH);
|
|
149
|
+
try {
|
|
150
|
+
// 执行请求 (Execute request)
|
|
151
|
+
const response = await HttpClient.fetchWithRedirects(requestPayload);
|
|
152
|
+
// 读取响应文本 (Read response text)
|
|
153
|
+
const html = await response.text();
|
|
154
|
+
// 使用ContentProcessor将HTML转换为纯文本 (Use ContentProcessor to convert HTML to plain text)
|
|
155
|
+
const plainText = ContentProcessor.htmlToText(html, debug, COMPONENTS.NODE_FETCH);
|
|
156
|
+
// 检查内容大小并处理 (Check content size and process)
|
|
157
|
+
const chunkingResult = this.handleContentChunking(plainText, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.NODE_FETCH);
|
|
158
|
+
if (chunkingResult) {
|
|
159
|
+
return chunkingResult;
|
|
160
|
+
}
|
|
161
|
+
// 返回纯文本内容 (Return plain text content)
|
|
162
|
+
return BaseFetcher.createSuccessResponse(plainText);
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
// 使用ErrorHandler处理错误 (Use ErrorHandler to handle error)
|
|
166
|
+
const errorMessage = ErrorHandler.handleError(error, COMPONENTS.NODE_FETCH, debug, { url: requestPayload.url, method: 'plainText' });
|
|
167
|
+
// 返回错误信息 (Return error message)
|
|
168
|
+
return BaseFetcher.createErrorResponse(`Error fetching plain text: ${errorMessage}`);
|
|
128
169
|
}
|
|
129
170
|
}
|
|
130
171
|
/**
|
|
@@ -133,48 +174,32 @@ export class NodeFetcher {
|
|
|
133
174
|
* @returns Markdown内容 (Markdown content)
|
|
134
175
|
*/
|
|
135
176
|
async markdown(requestPayload) {
|
|
136
|
-
const { debug = false } = requestPayload;
|
|
137
|
-
|
|
177
|
+
const { debug = false, contentSizeLimit = ContentSizeManager.getDefaultSizeLimit(), enableContentSplitting = true, chunkId, startCursor = 0 } = requestPayload;
|
|
178
|
+
// 如果提供了分段ID和起始游标,则从缓存中获取分段内容 (If chunk ID and startCursor are provided, get chunk content from cache)
|
|
179
|
+
if (chunkId && startCursor !== undefined) {
|
|
180
|
+
return this.getChunkContent(chunkId, startCursor, contentSizeLimit, debug, COMPONENTS.NODE_FETCH);
|
|
181
|
+
}
|
|
182
|
+
log('node.startingMarkdownFetch', debug, { url: requestPayload.url }, COMPONENTS.NODE_FETCH);
|
|
138
183
|
try {
|
|
139
184
|
// 执行请求 (Execute request)
|
|
140
185
|
const response = await HttpClient.fetchWithRedirects(requestPayload);
|
|
141
186
|
// 读取响应文本 (Read response text)
|
|
142
|
-
log('node.readingText', debug, {}, COMPONENTS.NODE_FETCH);
|
|
143
187
|
const html = await response.text();
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
});
|
|
152
|
-
// 添加表格支持
|
|
153
|
-
turndownService.addRule('tables', {
|
|
154
|
-
filter: ['table'],
|
|
155
|
-
replacement: function (content, node) {
|
|
156
|
-
const tableContent = content.trim();
|
|
157
|
-
return '\n\n' + tableContent + '\n\n';
|
|
158
|
-
}
|
|
159
|
-
});
|
|
160
|
-
// 将HTML转换为Markdown (Convert HTML to Markdown)
|
|
161
|
-
log('node.convertingToMarkdown', debug, {}, COMPONENTS.NODE_FETCH);
|
|
162
|
-
const markdown = turndownService.turndown(html);
|
|
163
|
-
log('node.markdownContentLength', debug, { length: markdown.length }, COMPONENTS.NODE_FETCH);
|
|
188
|
+
// 使用ContentProcessor将HTML转换为Markdown (Use ContentProcessor to convert HTML to Markdown)
|
|
189
|
+
const markdown = ContentProcessor.htmlToMarkdown(html, debug, COMPONENTS.NODE_FETCH);
|
|
190
|
+
// 检查内容大小并处理 (Check content size and process)
|
|
191
|
+
const chunkingResult = this.handleContentChunking(markdown, contentSizeLimit, enableContentSplitting, debug, COMPONENTS.NODE_FETCH);
|
|
192
|
+
if (chunkingResult) {
|
|
193
|
+
return chunkingResult;
|
|
194
|
+
}
|
|
164
195
|
// 返回Markdown内容 (Return Markdown content)
|
|
165
|
-
return
|
|
166
|
-
content: [{ type: 'text', text: markdown }],
|
|
167
|
-
isError: false
|
|
168
|
-
};
|
|
196
|
+
return BaseFetcher.createSuccessResponse(markdown);
|
|
169
197
|
}
|
|
170
198
|
catch (error) {
|
|
171
199
|
// 使用ErrorHandler处理错误 (Use ErrorHandler to handle error)
|
|
172
200
|
const errorMessage = ErrorHandler.handleError(error, COMPONENTS.NODE_FETCH, debug, { url: requestPayload.url, method: 'markdown' });
|
|
173
201
|
// 返回错误信息 (Return error message)
|
|
174
|
-
return {
|
|
175
|
-
content: [{ type: 'text', text: `Error fetching Markdown: ${errorMessage}` }],
|
|
176
|
-
isError: true
|
|
177
|
-
};
|
|
202
|
+
return BaseFetcher.createErrorResponse(`Error fetching Markdown: ${errorMessage}`);
|
|
178
203
|
}
|
|
179
204
|
}
|
|
180
205
|
}
|
|
@@ -25,6 +25,9 @@ export const BROWSER_KEYS = (() => {
|
|
|
25
25
|
scrollError: keyGen('scrollError'),
|
|
26
26
|
gettingContent: keyGen('gettingContent'),
|
|
27
27
|
contentLength: keyGen('contentLength'),
|
|
28
|
+
contentTruncated: keyGen('contentTruncated'),
|
|
29
|
+
contentSplit: keyGen('contentSplit'),
|
|
30
|
+
startingHtmlFetch: keyGen('startingHtmlFetch'),
|
|
28
31
|
// Cookie 相关 (Cookie related)
|
|
29
32
|
savingCookies: keyGen('savingCookies'),
|
|
30
33
|
cookiesSaved: keyGen('cookiesSaved'),
|
|
@@ -45,7 +48,6 @@ export const BROWSER_KEYS = (() => {
|
|
|
45
48
|
navigating: keyGen('navigating'),
|
|
46
49
|
scrolling: keyGen('scrolling'),
|
|
47
50
|
scrollCompleted: keyGen('scrollCompleted'),
|
|
48
|
-
contentTruncated: keyGen('contentTruncated'),
|
|
49
51
|
pageClosed: keyGen('pageClosed'),
|
|
50
52
|
fetchError: keyGen('fetchError'),
|
|
51
53
|
highMemory: keyGen('highMemory'),
|
|
@@ -64,6 +66,10 @@ export const BROWSER_KEYS = (() => {
|
|
|
64
66
|
cloudflareError: keyGen('cloudflareError'),
|
|
65
67
|
continuingWithoutBypass: keyGen('continuingWithoutBypass'),
|
|
66
68
|
unableToBypassCloudflare: keyGen('unableToBypassCloudflare'),
|
|
69
|
+
cloudflareBypass: keyGen('cloudflareBypass'),
|
|
70
|
+
cloudflareBypassSuccess: keyGen('cloudflareBypassSuccess'),
|
|
71
|
+
cloudflareBypassFailed: keyGen('cloudflareBypassFailed'),
|
|
72
|
+
cloudflareBypassNotNeeded: keyGen('cloudflareBypassNotNeeded'),
|
|
67
73
|
// 获取和重试相关 (Fetch and retry related)
|
|
68
74
|
fetchingWithRetry: keyGen('fetchingWithRetry'),
|
|
69
75
|
memoryUsage: keyGen('memoryUsage'),
|
|
@@ -86,6 +92,38 @@ export const BROWSER_KEYS = (() => {
|
|
|
86
92
|
reusingExistingBrowser: keyGen('reusingExistingBrowser'),
|
|
87
93
|
startingBrowser: keyGen('startingBrowser'),
|
|
88
94
|
browserStarted: keyGen('browserStarted'),
|
|
89
|
-
browserStartError: keyGen('browserStartError')
|
|
95
|
+
browserStartError: keyGen('browserStartError'),
|
|
96
|
+
// Fetch 相关 (Fetch related)
|
|
97
|
+
htmlFetchError: keyGen('htmlFetchError'),
|
|
98
|
+
jsonFetchError: keyGen('jsonFetchError'),
|
|
99
|
+
txtFetchError: keyGen('txtFetchError'),
|
|
100
|
+
markdownFetchError: keyGen('markdownFetchError'),
|
|
101
|
+
creatingTurndown: keyGen('creatingTurndown'),
|
|
102
|
+
convertingToMarkdown: keyGen('convertingToMarkdown'),
|
|
103
|
+
markdownContentLength: keyGen('markdownContentLength'),
|
|
104
|
+
startingJsonFetch: keyGen('startingJsonFetch'),
|
|
105
|
+
startingTxtFetch: keyGen('startingTxtFetch'),
|
|
106
|
+
startingMarkdownFetch: keyGen('startingMarkdownFetch'),
|
|
107
|
+
startingPlainTextFetch: keyGen('startingPlainTextFetch'),
|
|
108
|
+
extractingText: keyGen('extractingText'),
|
|
109
|
+
textExtracted: keyGen('textExtracted'),
|
|
110
|
+
jsonParsed: keyGen('jsonParsed'),
|
|
111
|
+
jsonParseError: keyGen('jsonParseError'),
|
|
112
|
+
// 代理相关 (Proxy related)
|
|
113
|
+
proxyConnected: keyGen('proxyConnected'),
|
|
114
|
+
proxyError: keyGen('proxyError'),
|
|
115
|
+
// 响应相关 (Response related)
|
|
116
|
+
responseStatus: keyGen('responseStatus'),
|
|
117
|
+
responseError: keyGen('responseError'),
|
|
118
|
+
responseSuccess: keyGen('responseSuccess'),
|
|
119
|
+
responseRedirect: keyGen('responseRedirect'),
|
|
120
|
+
responseTimeout: keyGen('responseTimeout'),
|
|
121
|
+
// 错误处理 (Error handling)
|
|
122
|
+
errorResponse: keyGen('errorResponse'),
|
|
123
|
+
errorResponseBody: keyGen('errorResponseBody'),
|
|
124
|
+
accessDenied: keyGen('accessDenied'),
|
|
125
|
+
timeoutError: keyGen('timeoutError'),
|
|
126
|
+
networkError: keyGen('networkError'),
|
|
127
|
+
unknownError: keyGen('unknownError')
|
|
90
128
|
};
|
|
91
129
|
})();
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Author: Martin <lmccc.dev@gmail.com>
|
|
3
|
+
* Co-Author: AI Assistant (Claude)
|
|
4
|
+
* Description: This code was collaboratively developed by Martin and AI Assistant.
|
|
5
|
+
*/
|
|
6
|
+
import { createKeyGenerator } from './base.js';
|
|
7
|
+
const PREFIX = 'chunkManager';
|
|
8
|
+
const keyGen = createKeyGenerator(PREFIX);
|
|
9
|
+
/**
|
|
10
|
+
* 分段管理相关的国际化键 (Internationalization keys related to chunk management)
|
|
11
|
+
*/
|
|
12
|
+
export const CHUNK_MANAGER_KEYS = {
|
|
13
|
+
// 分段管理基础操作 (Basic chunk management operations)
|
|
14
|
+
startingManager: keyGen('startingManager'),
|
|
15
|
+
creatingChunks: keyGen('creatingChunks'),
|
|
16
|
+
splittingContent: keyGen('splittingContent'),
|
|
17
|
+
creatingChunk: keyGen('creatingChunk'),
|
|
18
|
+
chunksCreated: keyGen('chunksCreated'),
|
|
19
|
+
calculatingChunks: keyGen('calculatingChunks'),
|
|
20
|
+
totalChunks: keyGen('totalChunks'),
|
|
21
|
+
storedChunks: keyGen('storedChunks'),
|
|
22
|
+
cleanedUpExpiredChunks: keyGen('cleanedUpExpiredChunks'),
|
|
23
|
+
// 分段相关错误 (Chunk related errors)
|
|
24
|
+
invalidChunkId: keyGen('invalidChunkId'),
|
|
25
|
+
chunkNotFound: keyGen('chunkNotFound'),
|
|
26
|
+
chunkIdNotFound: keyGen('chunkIdNotFound'),
|
|
27
|
+
invalidByteRange: keyGen('invalidByteRange'),
|
|
28
|
+
invalidStartCursor: keyGen('invalidStartCursor'),
|
|
29
|
+
splitError: keyGen('splitError'),
|
|
30
|
+
sizeInfoNotFound: keyGen('sizeInfoNotFound'),
|
|
31
|
+
// 分段内容控制 (Chunk content control)
|
|
32
|
+
gettingChunk: keyGen('gettingChunk'),
|
|
33
|
+
chunkRetrieved: keyGen('chunkRetrieved'),
|
|
34
|
+
retrievedChunkBySize: keyGen('retrievedChunkBySize'),
|
|
35
|
+
contentExceedsLimit: keyGen('contentExceedsLimit'),
|
|
36
|
+
byteRangeSplit: keyGen('byteRangeSplit'),
|
|
37
|
+
byteLimitApplied: keyGen('byteLimitApplied'),
|
|
38
|
+
// 进度和统计 (Progress and statistics)
|
|
39
|
+
processingChunk: keyGen('processingChunk'),
|
|
40
|
+
chunkStats: keyGen('chunkStats'),
|
|
41
|
+
remainingChunks: keyGen('remainingChunks'),
|
|
42
|
+
chunkProgress: keyGen('chunkProgress')
|
|
43
|
+
};
|
|
@@ -32,6 +32,46 @@ export const CLIENT_KEYS = (() => {
|
|
|
32
32
|
fetchingMarkdown: keyGen('fetchingMarkdown'),
|
|
33
33
|
fetchSuccessful: keyGen('fetchSuccessful'),
|
|
34
34
|
contentLength: keyGen('contentLength'),
|
|
35
|
+
// 分段内容相关 (Chunked content related)
|
|
36
|
+
hasMoreChunks: keyGen('hasMoreChunks'),
|
|
37
|
+
nextChunkCommand: keyGen('nextChunkCommand'),
|
|
38
|
+
chunkInfoSaved: keyGen('chunkInfoSaved'),
|
|
39
|
+
chunkInfoSaveError: keyGen('chunkInfoSaveError'),
|
|
40
|
+
chunkInfoLoaded: keyGen('chunkInfoLoaded'),
|
|
41
|
+
chunkInfoLoadError: keyGen('chunkInfoLoadError'),
|
|
42
|
+
noChunkInfo: keyGen('noChunkInfo'),
|
|
43
|
+
fetchingNextChunk: keyGen('fetchingNextChunk'),
|
|
44
|
+
chunkUsageInfo: keyGen('chunkUsageInfo'),
|
|
45
|
+
invalidChunkIndex: keyGen('invalidChunkIndex'),
|
|
46
|
+
missingMethodOrUrl: keyGen('missingMethodOrUrl'),
|
|
47
|
+
fetchingAllChunks: keyGen('fetchingAllChunks'),
|
|
48
|
+
fetchingChunk: keyGen('fetchingChunk'),
|
|
49
|
+
fetchChunkFailed: keyGen('fetchChunkFailed'),
|
|
50
|
+
allChunksFetched: keyGen('allChunksFetched'),
|
|
51
|
+
allChunksCommand: keyGen('allChunksCommand'),
|
|
52
|
+
allChunksUsageInfo: keyGen('allChunksUsageInfo'),
|
|
53
|
+
// 新增分块相关键 (New chunk related keys)
|
|
54
|
+
responseStructure: keyGen('responseStructure'),
|
|
55
|
+
parsedByteChunkInfo: keyGen('parsedByteChunkInfo'),
|
|
56
|
+
parsedChunkInfo: keyGen('parsedChunkInfo'),
|
|
57
|
+
chunkLimitNotice: keyGen('chunkLimitNotice'),
|
|
58
|
+
chunkLimitHint: keyGen('chunkLimitHint'),
|
|
59
|
+
fetchingChunkProgress: keyGen('fetchingChunkProgress'),
|
|
60
|
+
fetchChunkFailedError: keyGen('fetchChunkFailedError'),
|
|
61
|
+
chunkContent: keyGen('chunkContent'),
|
|
62
|
+
chunkSeparator: keyGen('chunkSeparator'),
|
|
63
|
+
partialChunksFetched: keyGen('partialChunksFetched'),
|
|
64
|
+
completeChunksFetched: keyGen('completeChunksFetched'),
|
|
65
|
+
fetchingChunksErrorMessage: keyGen('fetchingChunksErrorMessage'),
|
|
66
|
+
contentChunkedBytes: keyGen('contentChunkedBytes'),
|
|
67
|
+
contentChunkedCount: keyGen('contentChunkedCount'),
|
|
68
|
+
fetchAllChunksHint: keyGen('fetchAllChunksHint'),
|
|
69
|
+
fetchLimitedChunksHint: keyGen('fetchLimitedChunksHint'),
|
|
70
|
+
firstChunkCompleted: keyGen('firstChunkCompleted'),
|
|
71
|
+
chunkProgress: keyGen('chunkProgress'),
|
|
72
|
+
chunkCompleted: keyGen('chunkCompleted'),
|
|
73
|
+
recalculatedTotalChunks: keyGen('recalculatedTotalChunks'),
|
|
74
|
+
fetchingChunksError: keyGen('fetchingChunksError'),
|
|
35
75
|
// 调试相关 (Debug related)
|
|
36
76
|
debugMode: keyGen('debugMode'),
|
|
37
77
|
debugInfo: keyGen('debugInfo'),
|
|
@@ -63,6 +103,18 @@ export const CLIENT_KEYS = (() => {
|
|
|
63
103
|
browserModeFetchFailed: keyGen('browserModeFetchFailed'),
|
|
64
104
|
browserModeFetchSuccess: keyGen('browserModeFetchSuccess'),
|
|
65
105
|
serverClosed: keyGen('serverClosed'),
|
|
66
|
-
proxySet: keyGen('proxySet')
|
|
106
|
+
proxySet: keyGen('proxySet'),
|
|
107
|
+
// 模式相关 (Mode related)
|
|
108
|
+
usingStandardMode: keyGen('usingStandardMode'),
|
|
109
|
+
usingBrowserMode: keyGen('usingBrowserMode'),
|
|
110
|
+
// 参数相关 (Parameters related)
|
|
111
|
+
sendingParameters: keyGen('sendingParameters'),
|
|
112
|
+
limitingChunks: keyGen('limitingChunks'),
|
|
113
|
+
maxChunksUsageInfo: keyGen('maxChunksUsageInfo'),
|
|
114
|
+
alternateUsageInfo: keyGen('alternateUsageInfo'),
|
|
115
|
+
switchingToBrowserMode: keyGen('switchingToBrowserMode'),
|
|
116
|
+
invalidUrl: keyGen('invalidUrl'),
|
|
117
|
+
missingUrl: keyGen('missingUrl'),
|
|
118
|
+
missingStartCursor: keyGen('missingStartCursor')
|
|
67
119
|
};
|
|
68
120
|
})();
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Author: Martin <lmccc.dev@gmail.com>
|
|
3
|
+
* Co-Author: AI Assistant (Claude)
|
|
4
|
+
* Description: This code was collaboratively developed by Martin and AI Assistant.
|
|
5
|
+
*/
|
|
6
|
+
import { createKeyGenerator } from './base.js';
|
|
7
|
+
const PREFIX = 'contentSize';
|
|
8
|
+
const keyGen = createKeyGenerator(PREFIX);
|
|
9
|
+
/**
|
|
10
|
+
* 内容大小相关的国际化键 (Internationalization keys related to content size)
|
|
11
|
+
*/
|
|
12
|
+
export const CONTENT_SIZE_KEYS = {
|
|
13
|
+
// 内容大小检查相关 (Content size checking related)
|
|
14
|
+
checking: keyGen('checking'),
|
|
15
|
+
truncating: keyGen('truncating'),
|
|
16
|
+
truncated: keyGen('truncated'),
|
|
17
|
+
splitting: keyGen('splitting'),
|
|
18
|
+
splitComplete: keyGen('splitComplete'),
|
|
19
|
+
splitIntoChunks: keyGen('splitIntoChunks')
|
|
20
|
+
};
|