@lmcc-dev/mult-fetch-mcp-server 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -15
- package/README.zh.md +31 -0
- package/dist/src/client.js +118 -91
- package/dist/src/lib/fetchers/browser/BrowserFetcher.js +25 -10
- package/dist/src/lib/fetchers/browser/BrowserInstance.js +67 -64
- package/dist/src/lib/fetchers/common/BaseFetcher.js +106 -10
- package/dist/src/lib/fetchers/common/types.js +10 -1
- package/dist/src/lib/fetchers/common/utils.js +1 -1
- package/dist/src/lib/fetchers/node/HttpClient.js +41 -21
- package/dist/src/lib/fetchers/node/NodeFetcher.js +24 -8
- package/dist/src/lib/i18n/index.js +2 -2
- package/dist/src/lib/i18n/keys/client.js +1 -0
- package/dist/src/lib/i18n/keys/extractor.js +26 -0
- package/dist/src/lib/i18n/keys/fetcher.js +12 -1
- package/dist/src/lib/i18n/keys/index.js +1 -0
- package/dist/src/lib/i18n/keys/node.js +2 -0
- package/dist/src/lib/i18n/locales/en/client.js +1 -0
- package/dist/src/lib/i18n/locales/en/extractor.js +24 -0
- package/dist/src/lib/i18n/locales/en/fetcher.js +3 -1
- package/dist/src/lib/i18n/locales/en/index.js +2 -0
- package/dist/src/lib/i18n/locales/en/node.js +2 -0
- package/dist/src/lib/i18n/locales/zh/client.js +1 -0
- package/dist/src/lib/i18n/locales/zh/extractor.js +22 -0
- package/dist/src/lib/i18n/locales/zh/fetcher.js +13 -3
- package/dist/src/lib/i18n/locales/zh/index.js +2 -0
- package/dist/src/lib/i18n/locales/zh/node.js +2 -0
- package/dist/src/lib/i18n/logger.js +2 -2
- package/dist/src/lib/logger.js +38 -17
- package/dist/src/lib/server/browser.js +2 -2
- package/dist/src/lib/server/fetcher.js +0 -3
- package/dist/src/lib/server/index.js +2 -2
- package/dist/src/lib/server/prompts.js +4 -4
- package/dist/src/lib/server/tools.js +127 -354
- package/dist/src/lib/utils/ChunkManager.js +2 -2
- package/dist/src/lib/utils/ContentExtractor.js +141 -0
- package/dist/src/lib/utils/ContentProcessor.js +5 -11
- package/dist/src/lib/utils/ContentSizeManager.js +2 -2
- package/dist/src/lib/utils/ErrorHandler.js +1 -0
- package/dist/src/lib/utils/TemplateUtils.js +6 -2
- package/dist/tests/client.test.js +1 -1
- package/dist/tests/test-extract-single.js +389 -0
- package/dist/tests/utils/ContentExtractor.test.js +173 -0
- package/dist/tests/utils/ContentProcessor.test.js +136 -0
- package/dist/tests/utils/TemplateUtils.test.js +118 -0
- package/package.json +14 -3
- package/dist/i18n-test-report.json +0 -8
- package/dist/i18n-unused-keys-report.json +0 -8
- package/dist/src/lib/BrowserFetcher.js +0 -787
- package/dist/src/lib/NodeFetcher.js +0 -492
- package/dist/src/lib/i18n/keys.js +0 -529
- package/dist/src/test-i18n.js +0 -139
- package/dist/tests/BrowserFetcher.test.js +0 -951
- package/dist/tests/NodeFetcher.test.js +0 -263
- package/dist/tests/i18n-remove-unused-keys.js +0 -236
- package/dist/tests/i18n-test-report.json +0 -2004
- package/dist/tests/src/lib/i18n/index.js +0 -108
- package/dist/tests/src/lib/i18n/keys/base.js +0 -47
- package/dist/tests/src/lib/i18n/keys/browser.js +0 -93
- package/dist/tests/src/lib/i18n/keys/client.js +0 -70
- package/dist/tests/src/lib/i18n/keys/errors.js +0 -34
- package/dist/tests/src/lib/i18n/keys/fetcher.js +0 -84
- package/dist/tests/src/lib/i18n/keys/index.js +0 -31
- package/dist/tests/src/lib/i18n/keys/node.js +0 -56
- package/dist/tests/src/lib/i18n/keys/prompts.js +0 -82
- package/dist/tests/src/lib/i18n/keys/resources.js +0 -50
- package/dist/tests/src/lib/i18n/keys/server.js +0 -64
- package/dist/tests/src/lib/i18n/keys/tools.js +0 -34
- package/dist/tests/src/lib/i18n/locales/en/browser.js +0 -88
- package/dist/tests/src/lib/i18n/locales/en/client.js +0 -66
- package/dist/tests/src/lib/i18n/locales/en/errors.js +0 -28
- package/dist/tests/src/lib/i18n/locales/en/fetcher.js +0 -71
- package/dist/tests/src/lib/i18n/locales/en/index.js +0 -29
- package/dist/tests/src/lib/i18n/locales/en/node.js +0 -51
- package/dist/tests/src/lib/i18n/locales/en/prompts.js +0 -52
- package/dist/tests/src/lib/i18n/locales/en/resources.js +0 -50
- package/dist/tests/src/lib/i18n/locales/en/server.js +0 -56
- package/dist/tests/src/lib/i18n/locales/en/tools.js +0 -28
- package/dist/tests/src/lib/i18n/locales/zh/browser.js +0 -87
- package/dist/tests/src/lib/i18n/locales/zh/client.js +0 -66
- package/dist/tests/src/lib/i18n/locales/zh/errors.js +0 -28
- package/dist/tests/src/lib/i18n/locales/zh/fetcher.js +0 -71
- package/dist/tests/src/lib/i18n/locales/zh/index.js +0 -29
- package/dist/tests/src/lib/i18n/locales/zh/node.js +0 -51
- package/dist/tests/src/lib/i18n/locales/zh/prompts.js +0 -53
- package/dist/tests/src/lib/i18n/locales/zh/resources.js +0 -50
- package/dist/tests/src/lib/i18n/locales/zh/server.js +0 -57
- package/dist/tests/src/lib/i18n/locales/zh/tools.js +0 -28
- package/dist/tests/src/lib/i18n/logger.js +0 -114
- package/dist/tests/src/lib/logger.js +0 -181
- package/dist/tests/tests/test-i18n.js +0 -588
- package/dist/vitest.config.js +0 -29
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Author: Martin <lmccc.dev@gmail.com>
|
|
3
|
+
* Co-Author: AI Assistant (Claude)
|
|
4
|
+
* Description: This code was collaboratively developed by Martin and AI Assistant.
|
|
5
|
+
*/
|
|
6
|
+
import { createKeyGenerator } from './base.js';
|
|
7
|
+
const PREFIX = 'extractor';
|
|
8
|
+
const keyGen = createKeyGenerator(PREFIX);
|
|
9
|
+
/**
|
|
10
|
+
* 内容提取器国际化键值 (Content extractor internationalization keys)
|
|
11
|
+
*/
|
|
12
|
+
export const EXTRACTOR_KEYS = {
|
|
13
|
+
// 基础操作 (Basic operations)
|
|
14
|
+
creating_jsdom: keyGen('creating_jsdom'),
|
|
15
|
+
page_not_readerable: keyGen('page_not_readerable'),
|
|
16
|
+
creating_reader: keyGen('creating_reader'),
|
|
17
|
+
parsing_content: keyGen('parsing_content'),
|
|
18
|
+
parsing_failed: keyGen('parsing_failed'),
|
|
19
|
+
parsing_success: keyGen('parsing_success'),
|
|
20
|
+
extraction_error: keyGen('extraction_error'),
|
|
21
|
+
// 可读性检测 (Readability detection)
|
|
22
|
+
unlikely_page_type: keyGen('unlikely_page_type'),
|
|
23
|
+
has_article_elements: keyGen('has_article_elements'),
|
|
24
|
+
has_enough_paragraphs: keyGen('has_enough_paragraphs'),
|
|
25
|
+
not_enough_content: keyGen('not_enough_content')
|
|
26
|
+
};
|
|
@@ -92,5 +92,16 @@ export const FETCHER_KEYS = {
|
|
|
92
92
|
// 通用 (General)
|
|
93
93
|
notSet: keyGen('notSet'),
|
|
94
94
|
debug: keyGen('debug'),
|
|
95
|
-
none: keyGen('none')
|
|
95
|
+
none: keyGen('none'),
|
|
96
|
+
// 新增键
|
|
97
|
+
chunkInfo: keyGen('chunkInfo'),
|
|
98
|
+
lastChunkDetected: keyGen('lastChunkDetected'),
|
|
99
|
+
// 内容提取相关 (Content extraction related)
|
|
100
|
+
extractingContent: keyGen('extractingContent'),
|
|
101
|
+
extractionSuccess: keyGen('extractionSuccess'),
|
|
102
|
+
extractionFailed: keyGen('extractionFailed'),
|
|
103
|
+
extractionError: keyGen('extractionError'),
|
|
104
|
+
noFallback: keyGen('noFallback'),
|
|
105
|
+
usingOriginalContent: keyGen('usingOriginalContent'),
|
|
106
|
+
fallbackToOriginal: keyGen('fallbackToOriginal'),
|
|
96
107
|
};
|
|
@@ -36,9 +36,11 @@ export const NODE_KEYS = (() => {
|
|
|
36
36
|
requestOptions: keyGen('requestOptions'),
|
|
37
37
|
startingFetch: keyGen('startingFetch'),
|
|
38
38
|
fetchingUrl: keyGen('fetchingUrl'),
|
|
39
|
+
addingRandomDelay: keyGen('addingRandomDelay'),
|
|
39
40
|
// 响应相关 (Response related)
|
|
40
41
|
responseStatus: keyGen('responseStatus'),
|
|
41
42
|
redirectingTo: keyGen('redirectingTo'),
|
|
43
|
+
redirecting: keyGen('redirecting'),
|
|
42
44
|
constructedFullRedirectUrl: keyGen('constructedFullRedirectUrl'),
|
|
43
45
|
requestSuccess: keyGen('requestSuccess'),
|
|
44
46
|
// 错误相关 (Error related)
|
|
@@ -52,6 +52,7 @@ export default {
|
|
|
52
52
|
[CLIENT_KEYS.responseStructure]: '\n--- Response structure ---\n{{structure}}\n--- End of response structure ---\n',
|
|
53
53
|
[CLIENT_KEYS.parsedByteChunkInfo]: '\n--- Parsed byte-level chunk information ---\nchunkId: {{chunkId}}\nfetchedBytes: {{fetchedBytes}}\ntotalBytes: {{totalBytes}}\nremainingBytes: {{remainingBytes}}\nestimatedRemainingRequests: {{estimatedRemainingRequests}}\n--- End of parsed byte-level chunk information ---\n',
|
|
54
54
|
[CLIENT_KEYS.parsedChunkInfo]: '\n--- Parsed chunk information ---\nchunkId: {{chunkId}}\ncurrentChunk: {{currentChunk}}\ntotalChunks: {{totalChunks}}\nhasMoreChunks: {{hasMoreChunks}}\n--- End of parsed chunk information ---\n',
|
|
55
|
+
[CLIENT_KEYS.chunkInfoParsed]: 'Parsed chunk information: isChunked={{isChunked}}, hasMoreChunks={{hasMoreChunks}}, isLastChunk={{isLastChunk}}, chunkId={{chunkId}}, currentChunk={{currentChunk}}, totalChunks={{totalChunks}}, fetchedBytes={{fetchedBytes}}, totalBytes={{totalBytes}}, remainingBytes={{remainingBytes}}',
|
|
55
56
|
[CLIENT_KEYS.chunkLimitNotice]: '\nNote: Content has {{total}} chunks, but only fetching first {{fetching}} chunks',
|
|
56
57
|
[CLIENT_KEYS.chunkLimitHint]: 'To fetch more chunks, use --max-chunks=N parameter\n',
|
|
57
58
|
[CLIENT_KEYS.fetchingChunkProgress]: '\nFetching chunk {{current}}/{{total}}...',
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Author: Martin <lmccc.dev@gmail.com>
|
|
3
|
+
* Co-Author: AI Assistant (Claude)
|
|
4
|
+
* Description: This code was collaboratively developed by Martin and AI Assistant.
|
|
5
|
+
*/
|
|
6
|
+
import { EXTRACTOR_KEYS } from '../../keys/extractor.js';
|
|
7
|
+
/**
|
|
8
|
+
* 内容提取器的英文翻译 (English translations for content extractor)
|
|
9
|
+
*/
|
|
10
|
+
export default {
|
|
11
|
+
// Basic operations
|
|
12
|
+
[EXTRACTOR_KEYS.creating_jsdom]: 'Creating JSDOM document for content extraction',
|
|
13
|
+
[EXTRACTOR_KEYS.page_not_readerable]: 'Page is not suitable for Readability parsing',
|
|
14
|
+
[EXTRACTOR_KEYS.creating_reader]: 'Creating Readability parser',
|
|
15
|
+
[EXTRACTOR_KEYS.parsing_content]: 'Parsing content with Readability',
|
|
16
|
+
[EXTRACTOR_KEYS.parsing_failed]: 'Failed to parse content with Readability',
|
|
17
|
+
[EXTRACTOR_KEYS.parsing_success]: 'Successfully parsed content with Readability',
|
|
18
|
+
[EXTRACTOR_KEYS.extraction_error]: 'Error occurred during content extraction',
|
|
19
|
+
// Readability detection
|
|
20
|
+
[EXTRACTOR_KEYS.unlikely_page_type]: 'Page type is unlikely to contain readable content',
|
|
21
|
+
[EXTRACTOR_KEYS.has_article_elements]: 'Page contains article elements',
|
|
22
|
+
[EXTRACTOR_KEYS.has_enough_paragraphs]: 'Page contains enough paragraphs for extraction',
|
|
23
|
+
[EXTRACTOR_KEYS.not_enough_content]: 'Page does not contain enough content for extraction'
|
|
24
|
+
};
|
|
@@ -14,8 +14,10 @@ export default {
|
|
|
14
14
|
[FETCHER_KEYS.contentTruncated]: "Content truncated from {{originalSize}} to {{truncatedSize}} bytes",
|
|
15
15
|
// 分块处理相关 (Chunk processing related)
|
|
16
16
|
[FETCHER_KEYS.chunkRetrievalFailed]: "Failed to retrieve chunk: {{error}}",
|
|
17
|
-
[FETCHER_KEYS.gettingChunkBySize]: "Getting
|
|
17
|
+
[FETCHER_KEYS.gettingChunkBySize]: "Getting chunk by size, start: {{start}}, end: {{end}}",
|
|
18
18
|
[FETCHER_KEYS.chunkNotFound]: "Chunk not found: {{chunkId}}",
|
|
19
|
+
[FETCHER_KEYS.chunkInfo]: "Chunk info: Retrieved {{fetchedBytes}} bytes, total size {{totalBytes}} bytes, remaining {{remainingBytes}} bytes, completion {{percentage}}, is last chunk: {{isLastChunk}}",
|
|
20
|
+
[FETCHER_KEYS.lastChunkDetected]: "Last chunk detected, fetched bytes: {{fetchedBytes}}, total bytes: {{totalBytes}}, remaining bytes: {{remainingBytes}}",
|
|
19
21
|
// 获取类型相关 (Fetch type related)
|
|
20
22
|
[FETCHER_KEYS.fetchingPlainText]: "Fetching plain text content",
|
|
21
23
|
// 延迟相关 (Delay related)
|
|
@@ -14,6 +14,7 @@ import errors from './errors.js';
|
|
|
14
14
|
import resources from './resources.js';
|
|
15
15
|
import prompts from './prompts.js';
|
|
16
16
|
import url from './url.js';
|
|
17
|
+
import extractor from './extractor.js';
|
|
17
18
|
import { contentSizeEn as contentSize } from './contentSize.js';
|
|
18
19
|
import { chunkManagerEn as chunkManager } from './chunkManager.js';
|
|
19
20
|
import { processorEn as processor } from './processor.js';
|
|
@@ -30,6 +31,7 @@ export const enTranslation = {
|
|
|
30
31
|
resources,
|
|
31
32
|
prompts,
|
|
32
33
|
url,
|
|
34
|
+
extractor,
|
|
33
35
|
contentSize,
|
|
34
36
|
chunkManager,
|
|
35
37
|
processor
|
|
@@ -32,9 +32,11 @@ export default {
|
|
|
32
32
|
[NODE_KEYS.requestOptions]: "Request options: {{options}}",
|
|
33
33
|
[NODE_KEYS.startingFetch]: "Starting fetch: {{method}} {{url}}",
|
|
34
34
|
[NODE_KEYS.fetchingUrl]: "Fetching URL: {{url}}",
|
|
35
|
+
[NODE_KEYS.addingRandomDelay]: "Adding random delay between requests",
|
|
35
36
|
// 响应相关 (Response related)
|
|
36
37
|
[NODE_KEYS.responseStatus]: "Response status: {{status}}",
|
|
37
38
|
[NODE_KEYS.redirectingTo]: "Redirecting to: {{url}}",
|
|
39
|
+
[NODE_KEYS.redirecting]: "Redirecting to {{to}} (redirect count: {{redirectCount}})",
|
|
38
40
|
[NODE_KEYS.constructedFullRedirectUrl]: "Constructed full redirect URL: {{url}}",
|
|
39
41
|
[NODE_KEYS.requestSuccess]: "Request successful",
|
|
40
42
|
// 错误相关 (Error related)
|
|
@@ -52,6 +52,7 @@ export default {
|
|
|
52
52
|
[CLIENT_KEYS.responseStructure]: '\n--- 响应结构 (Response structure) ---\n{{structure}}\n--- 响应结构结束 (End of response structure) ---\n',
|
|
53
53
|
[CLIENT_KEYS.parsedByteChunkInfo]: '\n--- 解析后的字节级分段信息 (Parsed byte-level chunk information) ---\nchunkId: {{chunkId}}\nfetchedBytes: {{fetchedBytes}}\ntotalBytes: {{totalBytes}}\nremainingBytes: {{remainingBytes}}\nestimatedRemainingRequests: {{estimatedRemainingRequests}}\n--- 解析后的字节级分段信息结束 (End of parsed byte-level chunk information) ---\n',
|
|
54
54
|
[CLIENT_KEYS.parsedChunkInfo]: '\n--- 解析后的分段信息 (Parsed chunk information) ---\nchunkId: {{chunkId}}\ncurrentChunk: {{currentChunk}}\ntotalChunks: {{totalChunks}}\nhasMoreChunks: {{hasMoreChunks}}\n--- 解析后的分段信息结束 (End of parsed chunk information) ---\n',
|
|
55
|
+
[CLIENT_KEYS.chunkInfoParsed]: '已解析分段信息: 是否分段={{isChunked}}, 是否有更多分段={{hasMoreChunks}}, 是否为最后一段={{isLastChunk}}, 分段ID={{chunkId}}, 当前分段={{currentChunk}}, 总分段数={{totalChunks}}, 已获取字节数={{fetchedBytes}}, 总字节数={{totalBytes}}, 剩余字节数={{remainingBytes}}',
|
|
55
56
|
[CLIENT_KEYS.chunkLimitNotice]: '\n注意:内容共有 {{total}} 段,但只获取前 {{fetching}} 段',
|
|
56
57
|
[CLIENT_KEYS.chunkLimitHint]: '要获取更多分段,请使用 --max-chunks=N 参数 (To fetch more chunks, use --max-chunks=N parameter)\n',
|
|
57
58
|
[CLIENT_KEYS.fetchingChunkProgress]: '\n正在获取第 {{current}}/{{total}} 段内容...',
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Author: Martin <lmccc.dev@gmail.com>
|
|
3
|
+
* Co-Author: AI Assistant (Claude)
|
|
4
|
+
* Description: This code was collaboratively developed by Martin and AI Assistant.
|
|
5
|
+
*/
|
|
6
|
+
import { EXTRACTOR_KEYS } from '../../keys/extractor.js';
|
|
7
|
+
// 内容提取器的中文翻译 (Chinese translations for content extractor)
|
|
8
|
+
export default {
|
|
9
|
+
// 基础操作 (Basic operations)
|
|
10
|
+
[EXTRACTOR_KEYS.creating_jsdom]: '正在为内容提取创建JSDOM文档',
|
|
11
|
+
[EXTRACTOR_KEYS.page_not_readerable]: '页面不适合使用Readability解析',
|
|
12
|
+
[EXTRACTOR_KEYS.creating_reader]: '正在创建Readability解析器',
|
|
13
|
+
[EXTRACTOR_KEYS.parsing_content]: '正在使用Readability解析内容',
|
|
14
|
+
[EXTRACTOR_KEYS.parsing_failed]: '使用Readability解析内容失败',
|
|
15
|
+
[EXTRACTOR_KEYS.parsing_success]: '使用Readability成功解析内容',
|
|
16
|
+
[EXTRACTOR_KEYS.extraction_error]: '内容提取过程中发生错误',
|
|
17
|
+
// 可读性检测 (Readability detection)
|
|
18
|
+
[EXTRACTOR_KEYS.unlikely_page_type]: '页面类型不太可能包含可读内容',
|
|
19
|
+
[EXTRACTOR_KEYS.has_article_elements]: '页面包含文章元素',
|
|
20
|
+
[EXTRACTOR_KEYS.has_enough_paragraphs]: '页面包含足够的段落可供提取',
|
|
21
|
+
[EXTRACTOR_KEYS.not_enough_content]: '页面不包含足够的内容用于提取'
|
|
22
|
+
};
|
|
@@ -14,8 +14,10 @@ export default {
|
|
|
14
14
|
[FETCHER_KEYS.contentTruncated]: "内容已从 {{originalSize}} 截断至 {{truncatedSize}} 字节",
|
|
15
15
|
// 分块处理相关 (Chunk processing related)
|
|
16
16
|
[FETCHER_KEYS.chunkRetrievalFailed]: "获取分块失败: {{error}}",
|
|
17
|
-
[FETCHER_KEYS.gettingChunkBySize]: "
|
|
18
|
-
[FETCHER_KEYS.chunkNotFound]: "
|
|
17
|
+
[FETCHER_KEYS.gettingChunkBySize]: "正在获取分块内容,chunkId:{{chunkId}},起始位置:{{startCursor}},大小:{{sizeLimit}}",
|
|
18
|
+
[FETCHER_KEYS.chunkNotFound]: "未找到指定分块内容,chunkId:{{chunkId}},起始位置:{{startCursor}}",
|
|
19
|
+
[FETCHER_KEYS.chunkInfo]: "分块信息:获取了 {{fetchedBytes}} 字节,总大小 {{totalBytes}} 字节,剩余 {{remainingBytes}} 字节,完成度 {{percentage}},是否为最后分块:{{isLastChunk}}",
|
|
20
|
+
[FETCHER_KEYS.lastChunkDetected]: "检测到这是最后一个分块,已获取字节:{{fetchedBytes}},总字节:{{totalBytes}},剩余字节:{{remainingBytes}}",
|
|
19
21
|
// 获取类型相关 (Fetch type related)
|
|
20
22
|
[FETCHER_KEYS.fetchingPlainText]: "获取纯文本内容",
|
|
21
23
|
// 延迟相关 (Delay related)
|
|
@@ -90,5 +92,13 @@ export default {
|
|
|
90
92
|
// 通用 (General)
|
|
91
93
|
[FETCHER_KEYS.notSet]: "未设置",
|
|
92
94
|
[FETCHER_KEYS.debug]: "调试: {{message}}",
|
|
93
|
-
[FETCHER_KEYS.none]: "无"
|
|
95
|
+
[FETCHER_KEYS.none]: "无",
|
|
96
|
+
// 内容提取相关 (Content extraction related)
|
|
97
|
+
[FETCHER_KEYS.extractingContent]: "正在提取页面主要内容,URL: {{url}}",
|
|
98
|
+
[FETCHER_KEYS.extractionSuccess]: "内容提取成功,内容长度: {{contentLength}},标题: {{title}}",
|
|
99
|
+
[FETCHER_KEYS.extractionFailed]: "内容提取失败,URL: {{url}}",
|
|
100
|
+
[FETCHER_KEYS.extractionError]: "内容提取过程中出错: {{error}}, URL: {{url}}",
|
|
101
|
+
[FETCHER_KEYS.noFallback]: "内容提取失败,且禁用了回退选项",
|
|
102
|
+
[FETCHER_KEYS.usingOriginalContent]: "使用原始HTML内容作为结果",
|
|
103
|
+
[FETCHER_KEYS.fallbackToOriginal]: "回退到原始HTML内容"
|
|
94
104
|
};
|
|
@@ -14,6 +14,7 @@ import errors from './errors.js';
|
|
|
14
14
|
import resources from './resources.js';
|
|
15
15
|
import prompts from './prompts.js';
|
|
16
16
|
import url from './url.js';
|
|
17
|
+
import extractor from './extractor.js';
|
|
17
18
|
import { contentSizeZh as contentSize } from './contentSize.js';
|
|
18
19
|
import { chunkManagerZh as chunkManager } from './chunkManager.js';
|
|
19
20
|
import { processorZh as processor } from './processor.js';
|
|
@@ -30,6 +31,7 @@ export const zhTranslation = {
|
|
|
30
31
|
resources,
|
|
31
32
|
prompts,
|
|
32
33
|
url,
|
|
34
|
+
extractor,
|
|
33
35
|
contentSize,
|
|
34
36
|
chunkManager,
|
|
35
37
|
processor
|
|
@@ -32,8 +32,10 @@ export default {
|
|
|
32
32
|
[NODE_KEYS.requestOptions]: '请求选项: {{options}}',
|
|
33
33
|
[NODE_KEYS.startingFetch]: '开始获取: {{method}} {{url}}',
|
|
34
34
|
[NODE_KEYS.fetchingUrl]: '获取URL: {{url}}',
|
|
35
|
+
[NODE_KEYS.addingRandomDelay]: '添加请求间随机延迟',
|
|
35
36
|
[NODE_KEYS.responseStatus]: '响应状态: {{status}}',
|
|
36
37
|
[NODE_KEYS.redirectingTo]: '重定向到: {{url}}',
|
|
38
|
+
[NODE_KEYS.redirecting]: '重定向到 {{to}}(重定向计数:{{redirectCount}})',
|
|
37
39
|
[NODE_KEYS.constructedFullRedirectUrl]: '构建完整重定向 URL: {{url}}',
|
|
38
40
|
// 响应相关 (Response related)
|
|
39
41
|
[NODE_KEYS.requestSuccess]: '请求成功',
|
|
@@ -19,7 +19,7 @@ export var LogLevel;
|
|
|
19
19
|
* 获取当前日志级别 (Get current log level)
|
|
20
20
|
* @returns 日志级别 (Log level)
|
|
21
21
|
*/
|
|
22
|
-
function
|
|
22
|
+
function _getCurrentLogLevel() {
|
|
23
23
|
const debugEnv = process.env.DEBUG;
|
|
24
24
|
if (debugEnv === 'none')
|
|
25
25
|
return LogLevel.NONE;
|
|
@@ -58,7 +58,7 @@ function translateKey(key, params) {
|
|
|
58
58
|
}
|
|
59
59
|
return translatedStr;
|
|
60
60
|
}
|
|
61
|
-
catch (
|
|
61
|
+
catch (_error) {
|
|
62
62
|
// 如果翻译失败,返回原始键名 (If translation fails, return the original key)
|
|
63
63
|
return key;
|
|
64
64
|
}
|
package/dist/src/lib/logger.js
CHANGED
|
@@ -22,6 +22,7 @@ import promptsZh from './i18n/locales/zh/prompts.js';
|
|
|
22
22
|
import { contentSizeZh } from './i18n/locales/zh/contentSize.js';
|
|
23
23
|
import { chunkManagerZh } from './i18n/locales/zh/chunkManager.js';
|
|
24
24
|
import { processorZh } from './i18n/locales/zh/processor.js';
|
|
25
|
+
import extractorZh from './i18n/locales/zh/extractor.js';
|
|
25
26
|
// 创建翻译映射 (Create translation mapping)
|
|
26
27
|
const translationMap = {
|
|
27
28
|
server: serverZh,
|
|
@@ -36,7 +37,8 @@ const translationMap = {
|
|
|
36
37
|
prompts: promptsZh,
|
|
37
38
|
contentSize: contentSizeZh,
|
|
38
39
|
chunkManager: chunkManagerZh,
|
|
39
|
-
processor: processorZh
|
|
40
|
+
processor: processorZh,
|
|
41
|
+
extractor: extractorZh
|
|
40
42
|
};
|
|
41
43
|
// 组件名称常量 (Component name constants)
|
|
42
44
|
export const COMPONENTS = {
|
|
@@ -47,10 +49,13 @@ export const COMPONENTS = {
|
|
|
47
49
|
RESOURCES: 'MCP-RESOURCES',
|
|
48
50
|
PROMPTS: 'MCP-PROMPTS',
|
|
49
51
|
TOOLS: 'MCP-TOOLS',
|
|
52
|
+
FETCHER: 'FETCHER',
|
|
50
53
|
FETCHER_FACTORY: 'FETCHER-FACTORY',
|
|
51
54
|
CONTENT_SIZE: 'CONTENT-SIZE',
|
|
55
|
+
PROCESSOR: 'CONTENT-PROCESSOR',
|
|
52
56
|
CHUNK_MANAGER: 'CHUNK-MANAGER',
|
|
53
|
-
|
|
57
|
+
CONTENT_SIZE_MANAGER: 'CONTENT-SIZE-MANAGER',
|
|
58
|
+
EXTRACTOR: 'CONTENT-EXTRACTOR'
|
|
54
59
|
};
|
|
55
60
|
// 日志文件路径 (Log file path)
|
|
56
61
|
const LOG_DIR = path.join(os.homedir(), '.mult-fetch-mcp-server');
|
|
@@ -61,8 +66,8 @@ try {
|
|
|
61
66
|
fs.mkdirSync(LOG_DIR, { recursive: true });
|
|
62
67
|
}
|
|
63
68
|
}
|
|
64
|
-
catch (
|
|
65
|
-
console.error(`[${COMPONENTS.SERVER}] Failed to create log directory: ${
|
|
69
|
+
catch (_error) {
|
|
70
|
+
console.error(`[${COMPONENTS.SERVER}] Failed to create log directory: ${_error}`);
|
|
66
71
|
}
|
|
67
72
|
// 组件日志记录器缓存 (Component logger cache)
|
|
68
73
|
const loggerCache = {};
|
|
@@ -107,8 +112,8 @@ function writeToLogFile(component, message) {
|
|
|
107
112
|
const logEntry = `${timestamp} [${component}] ${message}\n`;
|
|
108
113
|
fs.appendFileSync(LOG_FILE, logEntry);
|
|
109
114
|
}
|
|
110
|
-
catch (
|
|
111
|
-
console.error(`[${COMPONENTS.SERVER}] Failed to write to log file: ${
|
|
115
|
+
catch (_error) {
|
|
116
|
+
console.error(`[${COMPONENTS.SERVER}] Failed to write to log file: ${_error}`);
|
|
112
117
|
}
|
|
113
118
|
}
|
|
114
119
|
/**
|
|
@@ -118,8 +123,8 @@ export function clearLogFile() {
|
|
|
118
123
|
try {
|
|
119
124
|
fs.writeFileSync(LOG_FILE, '');
|
|
120
125
|
}
|
|
121
|
-
catch (
|
|
122
|
-
console.error(`[${COMPONENTS.SERVER}] Failed to clear log file: ${
|
|
126
|
+
catch (_error) {
|
|
127
|
+
console.error(`[${COMPONENTS.SERVER}] Failed to clear log file: ${_error}`);
|
|
123
128
|
}
|
|
124
129
|
}
|
|
125
130
|
/**
|
|
@@ -133,10 +138,10 @@ export function getLogFilePath() {
|
|
|
133
138
|
* 尝试翻译键名 (Try to translate a key)
|
|
134
139
|
* @param key 翻译键 (Translation key)
|
|
135
140
|
* @param options 翻译选项 (Translation options)
|
|
136
|
-
* @param
|
|
141
|
+
* @param _component 组件名称 (Component name)
|
|
137
142
|
* @returns 翻译后的文本 (Translated text)
|
|
138
143
|
*/
|
|
139
|
-
function translateKey(key, options,
|
|
144
|
+
function translateKey(key, options, _component = COMPONENTS.SERVER) {
|
|
140
145
|
// 如果没有选项,尝试使用缓存 (If no options, try to use cache)
|
|
141
146
|
if (!options || Object.keys(options).length === 0) {
|
|
142
147
|
// 生成缓存键 (Generate cache key)
|
|
@@ -149,7 +154,17 @@ function translateKey(key, options, component = COMPONENTS.SERVER) {
|
|
|
149
154
|
// 尝试翻译键名 (Try to translate the key)
|
|
150
155
|
const translated = t(key, options);
|
|
151
156
|
// 确保结果是字符串 (Ensure the result is a string)
|
|
152
|
-
|
|
157
|
+
let translatedStr = typeof translated === 'string' ? translated : JSON.stringify(translated);
|
|
158
|
+
// 检查翻译后的字符串是否仍然包含占位符,如果包含,手动进行插值替换
|
|
159
|
+
// (Check if the translated string still contains placeholders, if so, manually perform interpolation)
|
|
160
|
+
if (options && typeof translatedStr === 'string') {
|
|
161
|
+
for (const key in options) {
|
|
162
|
+
const placeholder = `{{${key}}}`;
|
|
163
|
+
if (translatedStr.includes(placeholder)) {
|
|
164
|
+
translatedStr = translatedStr.replace(new RegExp(placeholder, 'g'), String(options[key]));
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
153
168
|
// 如果翻译结果与键名相同,可能是嵌套键未被正确处理 (If translation result is the same as key, nested key may not be handled correctly)
|
|
154
169
|
if (translatedStr === key) {
|
|
155
170
|
// 尝试手动处理嵌套键 (Try to manually handle nested keys)
|
|
@@ -161,9 +176,15 @@ function translateKey(key, options, component = COMPONENTS.SERVER) {
|
|
|
161
176
|
const translation = translationMap[namespace];
|
|
162
177
|
// 如果找到了翻译对象,尝试获取子键的翻译 (If translation object is found, try to get translation of subkey)
|
|
163
178
|
if (translation && translation[subKey]) {
|
|
164
|
-
//
|
|
165
|
-
|
|
166
|
-
|
|
179
|
+
// 如果有选项,进行插值 (If there are options, do interpolation)
|
|
180
|
+
let result = translation[subKey];
|
|
181
|
+
for (const key in options) {
|
|
182
|
+
const placeholder = `{{${key}}}`;
|
|
183
|
+
if (result.includes(placeholder)) {
|
|
184
|
+
result = result.replace(new RegExp(placeholder, 'g'), String(options[key]));
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return result;
|
|
167
188
|
}
|
|
168
189
|
}
|
|
169
190
|
}
|
|
@@ -171,7 +192,7 @@ function translateKey(key, options, component = COMPONENTS.SERVER) {
|
|
|
171
192
|
translationCache[cacheKey] = translatedStr;
|
|
172
193
|
return translatedStr;
|
|
173
194
|
}
|
|
174
|
-
catch (
|
|
195
|
+
catch (_error) {
|
|
175
196
|
// 如果翻译失败,返回原始键名 (If translation fails, return the original key)
|
|
176
197
|
return key;
|
|
177
198
|
}
|
|
@@ -218,7 +239,7 @@ function translateKey(key, options, component = COMPONENTS.SERVER) {
|
|
|
218
239
|
}
|
|
219
240
|
return translatedStr;
|
|
220
241
|
}
|
|
221
|
-
catch (
|
|
242
|
+
catch (_error) {
|
|
222
243
|
// 如果翻译失败,返回原始键名 (If translation fails, return the original key)
|
|
223
244
|
return key;
|
|
224
245
|
}
|
|
@@ -238,7 +259,7 @@ export function log(key, debug = false, options, component = COMPONENTS.SERVER)
|
|
|
238
259
|
return;
|
|
239
260
|
}
|
|
240
261
|
// 获取组件的日志记录器 (Get logger for component)
|
|
241
|
-
const
|
|
262
|
+
const _logger = getLogger(component);
|
|
242
263
|
// 翻译键名 (Translate the key)
|
|
243
264
|
const translatedMessage = translateKey(key, options, component);
|
|
244
265
|
// 在 MCP 环境中,所有日志都应该使用 error 级别输出 (In MCP environment, all logs should be output at error level)
|
|
@@ -92,10 +92,10 @@ export function shouldSwitchToBrowser(error) {
|
|
|
92
92
|
* 检查是否应该使用浏览器模式 (Check if browser mode should be used)
|
|
93
93
|
* 根据响应内容判断是否需要浏览器模式 (Determine if browser mode is needed based on response content)
|
|
94
94
|
* @param response 响应对象 (Response object)
|
|
95
|
-
* @param
|
|
95
|
+
* @param _url 请求URL (Request URL)
|
|
96
96
|
* @returns 是否应该使用浏览器模式 (Whether to use browser mode)
|
|
97
97
|
*/
|
|
98
|
-
export function shouldUseBrowser(response,
|
|
98
|
+
export function shouldUseBrowser(response, _url) {
|
|
99
99
|
// 检查响应是否表明需要浏览器模式 (Check if response indicates browser mode is needed)
|
|
100
100
|
if (response.isError) {
|
|
101
101
|
const errorText = response.content[0].text.toLowerCase();
|
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
import { Fetcher } from '../fetch.js';
|
|
7
7
|
import { log, COMPONENTS } from '../logger.js';
|
|
8
8
|
import { initializeBrowser, closeBrowserInstance, shouldSwitchToBrowser } from './browser.js';
|
|
9
|
-
import { BaseFetcher } from '../fetchers/common/BaseFetcher.js';
|
|
10
9
|
/**
|
|
11
10
|
* 辅助函数:根据类型和参数自动选择合适的获取方法 (Helper function: automatically select appropriate fetching method based on type and parameters)
|
|
12
11
|
* 支持自动在标准模式和浏览器模式之间切换 (Supports automatic switching between standard mode and browser mode)
|
|
@@ -185,6 +184,4 @@ export async function fetchWithAutoDetect(params, type) {
|
|
|
185
184
|
}
|
|
186
185
|
throw error;
|
|
187
186
|
}
|
|
188
|
-
// 如果没有匹配的类型,返回错误
|
|
189
|
-
return BaseFetcher.createErrorResponse(`Unsupported content type: ${type}`);
|
|
190
187
|
}
|
|
@@ -65,8 +65,8 @@ export async function startServer() {
|
|
|
65
65
|
await server.connect(transport);
|
|
66
66
|
log('server.started', true, {}, COMPONENTS.SERVER);
|
|
67
67
|
}
|
|
68
|
-
catch (
|
|
69
|
-
log('server.connectionError', true, { error:
|
|
68
|
+
catch (_error) {
|
|
69
|
+
log('server.connectionError', true, { error: String(_error) }, COMPONENTS.SERVER);
|
|
70
70
|
await closeBrowserInstance(false);
|
|
71
71
|
process.exit(1);
|
|
72
72
|
}
|
|
@@ -116,13 +116,13 @@ export function registerPrompts(server) {
|
|
|
116
116
|
}
|
|
117
117
|
}
|
|
118
118
|
log('prompts.useBrowserValue', debug, { original: args.useBrowser, parsed: useBrowser }, COMPONENTS.PROMPTS);
|
|
119
|
-
let
|
|
119
|
+
let _toolName = "fetch_html";
|
|
120
120
|
if (format === "json")
|
|
121
|
-
|
|
121
|
+
_toolName = "fetch_json";
|
|
122
122
|
if (format === "text")
|
|
123
|
-
|
|
123
|
+
_toolName = "fetch_txt";
|
|
124
124
|
if (format === "markdown")
|
|
125
|
-
|
|
125
|
+
_toolName = "fetch_markdown";
|
|
126
126
|
return {
|
|
127
127
|
description: t('prompts.fetchWebsite.result'),
|
|
128
128
|
messages: [
|