@lmcc-dev/mult-fetch-mcp-server 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -15
- package/README.zh.md +31 -0
- package/dist/src/client.js +118 -91
- package/dist/src/lib/fetchers/browser/BrowserFetcher.js +25 -10
- package/dist/src/lib/fetchers/browser/BrowserInstance.js +67 -64
- package/dist/src/lib/fetchers/common/BaseFetcher.js +106 -10
- package/dist/src/lib/fetchers/common/types.js +10 -1
- package/dist/src/lib/fetchers/common/utils.js +1 -1
- package/dist/src/lib/fetchers/node/HttpClient.js +41 -21
- package/dist/src/lib/fetchers/node/NodeFetcher.js +24 -8
- package/dist/src/lib/i18n/index.js +2 -2
- package/dist/src/lib/i18n/keys/client.js +1 -0
- package/dist/src/lib/i18n/keys/extractor.js +26 -0
- package/dist/src/lib/i18n/keys/fetcher.js +12 -1
- package/dist/src/lib/i18n/keys/index.js +1 -0
- package/dist/src/lib/i18n/keys/node.js +2 -0
- package/dist/src/lib/i18n/locales/en/client.js +1 -0
- package/dist/src/lib/i18n/locales/en/extractor.js +24 -0
- package/dist/src/lib/i18n/locales/en/fetcher.js +3 -1
- package/dist/src/lib/i18n/locales/en/index.js +2 -0
- package/dist/src/lib/i18n/locales/en/node.js +2 -0
- package/dist/src/lib/i18n/locales/zh/client.js +1 -0
- package/dist/src/lib/i18n/locales/zh/extractor.js +22 -0
- package/dist/src/lib/i18n/locales/zh/fetcher.js +13 -3
- package/dist/src/lib/i18n/locales/zh/index.js +2 -0
- package/dist/src/lib/i18n/locales/zh/node.js +2 -0
- package/dist/src/lib/i18n/logger.js +2 -2
- package/dist/src/lib/logger.js +38 -17
- package/dist/src/lib/server/browser.js +2 -2
- package/dist/src/lib/server/fetcher.js +0 -3
- package/dist/src/lib/server/index.js +2 -2
- package/dist/src/lib/server/prompts.js +4 -4
- package/dist/src/lib/server/tools.js +127 -354
- package/dist/src/lib/utils/ChunkManager.js +2 -2
- package/dist/src/lib/utils/ContentExtractor.js +141 -0
- package/dist/src/lib/utils/ContentProcessor.js +5 -11
- package/dist/src/lib/utils/ContentSizeManager.js +2 -2
- package/dist/src/lib/utils/ErrorHandler.js +1 -0
- package/dist/src/lib/utils/TemplateUtils.js +6 -2
- package/dist/tests/client.test.js +1 -1
- package/dist/tests/test-extract-single.js +389 -0
- package/dist/tests/utils/ContentExtractor.test.js +173 -0
- package/dist/tests/utils/ContentProcessor.test.js +136 -0
- package/dist/tests/utils/TemplateUtils.test.js +118 -0
- package/package.json +14 -3
- package/dist/i18n-test-report.json +0 -8
- package/dist/i18n-unused-keys-report.json +0 -8
- package/dist/src/lib/BrowserFetcher.js +0 -787
- package/dist/src/lib/NodeFetcher.js +0 -492
- package/dist/src/lib/i18n/keys.js +0 -529
- package/dist/src/test-i18n.js +0 -139
- package/dist/tests/BrowserFetcher.test.js +0 -951
- package/dist/tests/NodeFetcher.test.js +0 -263
- package/dist/tests/i18n-remove-unused-keys.js +0 -236
- package/dist/tests/i18n-test-report.json +0 -2004
- package/dist/tests/src/lib/i18n/index.js +0 -108
- package/dist/tests/src/lib/i18n/keys/base.js +0 -47
- package/dist/tests/src/lib/i18n/keys/browser.js +0 -93
- package/dist/tests/src/lib/i18n/keys/client.js +0 -70
- package/dist/tests/src/lib/i18n/keys/errors.js +0 -34
- package/dist/tests/src/lib/i18n/keys/fetcher.js +0 -84
- package/dist/tests/src/lib/i18n/keys/index.js +0 -31
- package/dist/tests/src/lib/i18n/keys/node.js +0 -56
- package/dist/tests/src/lib/i18n/keys/prompts.js +0 -82
- package/dist/tests/src/lib/i18n/keys/resources.js +0 -50
- package/dist/tests/src/lib/i18n/keys/server.js +0 -64
- package/dist/tests/src/lib/i18n/keys/tools.js +0 -34
- package/dist/tests/src/lib/i18n/locales/en/browser.js +0 -88
- package/dist/tests/src/lib/i18n/locales/en/client.js +0 -66
- package/dist/tests/src/lib/i18n/locales/en/errors.js +0 -28
- package/dist/tests/src/lib/i18n/locales/en/fetcher.js +0 -71
- package/dist/tests/src/lib/i18n/locales/en/index.js +0 -29
- package/dist/tests/src/lib/i18n/locales/en/node.js +0 -51
- package/dist/tests/src/lib/i18n/locales/en/prompts.js +0 -52
- package/dist/tests/src/lib/i18n/locales/en/resources.js +0 -50
- package/dist/tests/src/lib/i18n/locales/en/server.js +0 -56
- package/dist/tests/src/lib/i18n/locales/en/tools.js +0 -28
- package/dist/tests/src/lib/i18n/locales/zh/browser.js +0 -87
- package/dist/tests/src/lib/i18n/locales/zh/client.js +0 -66
- package/dist/tests/src/lib/i18n/locales/zh/errors.js +0 -28
- package/dist/tests/src/lib/i18n/locales/zh/fetcher.js +0 -71
- package/dist/tests/src/lib/i18n/locales/zh/index.js +0 -29
- package/dist/tests/src/lib/i18n/locales/zh/node.js +0 -51
- package/dist/tests/src/lib/i18n/locales/zh/prompts.js +0 -53
- package/dist/tests/src/lib/i18n/locales/zh/resources.js +0 -50
- package/dist/tests/src/lib/i18n/locales/zh/server.js +0 -57
- package/dist/tests/src/lib/i18n/locales/zh/tools.js +0 -28
- package/dist/tests/src/lib/i18n/logger.js +0 -114
- package/dist/tests/src/lib/logger.js +0 -181
- package/dist/tests/tests/test-i18n.js +0 -588
- package/dist/vitest.config.js +0 -29
package/README.md
CHANGED
|
@@ -33,20 +33,21 @@ This project implements an MCP-compliant client and server for communication bet
|
|
|
33
33
|
fetch-mcp/
|
|
34
34
|
├── src/ # Source code directory
|
|
35
35
|
│ ├── lib/ # Library files
|
|
36
|
-
│ │ ├── fetchers/ # Web fetching
|
|
36
|
+
│ │ ├── fetchers/ # Web fetching implementation
|
|
37
37
|
│ │ │ ├── browser/ # Browser-based fetching
|
|
38
|
-
│ │ │ │ ├── BrowserFetcher.ts # Browser
|
|
38
|
+
│ │ │ │ ├── BrowserFetcher.ts # Browser fetcher implementation
|
|
39
39
|
│ │ │ │ ├── BrowserInstance.ts # Browser instance management
|
|
40
40
|
│ │ │ │ └── PageOperations.ts # Page interaction operations
|
|
41
41
|
│ │ │ ├── node/ # Node.js-based fetching
|
|
42
42
|
│ │ │ └── common/ # Shared fetching utilities
|
|
43
43
|
│ │ ├── utils/ # Utility modules
|
|
44
|
-
│ │ │ ├── ChunkManager.ts # Content chunking
|
|
44
|
+
│ │ │ ├── ChunkManager.ts # Content chunking
|
|
45
45
|
│ │ │ ├── ContentProcessor.ts # HTML to text conversion
|
|
46
|
+
│ │ │ ├── ContentExtractor.ts # Intelligent content extraction
|
|
46
47
|
│ │ │ ├── ContentSizeManager.ts # Content size limiting
|
|
47
|
-
│ │ │ └── ErrorHandler.ts # Error
|
|
48
|
+
│ │ │ └── ErrorHandler.ts # Error handling
|
|
48
49
|
│ │ ├── server/ # Server-related modules
|
|
49
|
-
│ │ │ ├── index.ts # Server entry
|
|
50
|
+
│ │ │ ├── index.ts # Server entry
|
|
50
51
|
│ │ │ ├── browser.ts # Browser management
|
|
51
52
|
│ │ │ ├── fetcher.ts # Web fetching logic
|
|
52
53
|
│ │ │ ├── tools.ts # Tool registration and handling
|
|
@@ -76,12 +77,16 @@ This project implements the Standard Input/Output (Stdio) transport method.
|
|
|
76
77
|
- Implementation based on the official MCP SDK
|
|
77
78
|
- Support for Standard Input/Output (Stdio) transport
|
|
78
79
|
- Multiple web scraping methods (HTML, JSON, text, Markdown, plain text conversion)
|
|
79
|
-
- Intelligent mode switching:
|
|
80
|
-
- Content size management: automatically splits large content into manageable chunks to
|
|
80
|
+
- Intelligent mode switching: automatic switching between standard requests and browser mode
|
|
81
|
+
- Content size management: automatically splits large content into manageable chunks to solve AI model context size limitations
|
|
81
82
|
- Chunked content retrieval: ability to request specific chunks of large content while maintaining context continuity
|
|
82
|
-
- Detailed debug
|
|
83
|
-
-
|
|
83
|
+
- Detailed debug logging to stderr
|
|
84
|
+
- Bilingual internationalization (English and Chinese)
|
|
84
85
|
- Modular design for easy maintenance and extension
|
|
86
|
+
- **Intelligent Content Extraction**: Based on Mozilla's Readability library, capable of extracting meaningful content from web pages while filtering out advertisements and navigation elements
|
|
87
|
+
- **Metadata Support**: Ability to extract webpage metadata such as title, author, publication date, and site information
|
|
88
|
+
- **Smart Content Detection**: Automatically detects if a page contains meaningful content, filtering out login pages, error pages, and other pages without substantial content
|
|
89
|
+
- **Browser Automation Enhancements**: Support for page scrolling, cookie management, selector waiting, and other advanced browser interactions
|
|
85
90
|
|
|
86
91
|
## Installation
|
|
87
92
|
|
|
@@ -493,8 +498,8 @@ Each tool supports the following parameters:
|
|
|
493
498
|
|
|
494
499
|
#### Basic Parameters
|
|
495
500
|
- `url`: URL to fetch (required)
|
|
496
|
-
- `headers`: Custom request headers (optional, default
|
|
497
|
-
- `proxy`: Proxy server URL in format http://host:port or https://host:port (optional)
|
|
501
|
+
- `headers`: Custom request headers (optional, default {})
|
|
502
|
+
- `proxy`: Proxy server URL in the format http://host:port or https://host:port (optional)
|
|
498
503
|
|
|
499
504
|
#### Network Control Parameters
|
|
500
505
|
- `timeout`: Timeout in milliseconds (optional, default is 30000)
|
|
@@ -526,11 +531,58 @@ When content is split into chunks, the response includes metadata that allows th
|
|
|
526
531
|
- `saveCookies`: Whether to save cookies in browser mode (optional, default is true)
|
|
527
532
|
- `closeBrowser`: Whether to close the browser instance (optional, default is false)
|
|
528
533
|
|
|
534
|
+
#### Content Extraction Parameters
|
|
535
|
+
- `extractContent`: Whether to use the Readability algorithm to extract main content (optional, default false)
|
|
536
|
+
- `includeMetadata`: Whether to include metadata in the extracted content (optional, default false, only works when `extractContent` is true)
|
|
537
|
+
- `fallbackToOriginal`: Whether to fall back to the original content when extraction fails (optional, default true, only works when `extractContent` is true)
|
|
538
|
+
|
|
529
539
|
#### Debug Parameters
|
|
530
|
-
- `debug`: Whether to enable debug output (optional, default
|
|
540
|
+
- `debug`: Whether to enable debug output (optional, default false)
|
|
541
|
+
|
|
542
|
+
### Content Extraction Feature
|
|
543
|
+
|
|
544
|
+
Use the content extraction feature to get the core content of a webpage, filtering out navigation bars, advertisements, sidebars, and other distracting elements:
|
|
545
|
+
|
|
546
|
+
```json
|
|
547
|
+
{
|
|
548
|
+
"url": "https://example.com/article",
|
|
549
|
+
"extractContent": true,
|
|
550
|
+
"includeMetadata": true
|
|
551
|
+
}
|
|
552
|
+
```
|
|
553
|
+
|
|
554
|
+
The extracted content will include the following metadata (if available):
|
|
555
|
+
- Title
|
|
556
|
+
- Byline (author)
|
|
557
|
+
- Site name
|
|
558
|
+
- Excerpt
|
|
559
|
+
- Content length
|
|
560
|
+
- Readability flag (isReaderable)
|
|
531
561
|
|
|
532
562
|
### Special Usage
|
|
533
563
|
|
|
564
|
+
#### Content Extraction Examples
|
|
565
|
+
|
|
566
|
+
To extract only the meaningful content from an article webpage:
|
|
567
|
+
|
|
568
|
+
```json
|
|
569
|
+
{
|
|
570
|
+
"url": "https://example.com/news/article",
|
|
571
|
+
"extractContent": true,
|
|
572
|
+
"includeMetadata": true
|
|
573
|
+
}
|
|
574
|
+
```
|
|
575
|
+
|
|
576
|
+
For websites where content extraction might fail, you can use `fallbackToOriginal` to ensure you get some content:
|
|
577
|
+
|
|
578
|
+
```json
|
|
579
|
+
{
|
|
580
|
+
"url": "https://example.com/complex-layout",
|
|
581
|
+
"extractContent": true,
|
|
582
|
+
"fallbackToOriginal": true
|
|
583
|
+
}
|
|
584
|
+
```
|
|
585
|
+
|
|
534
586
|
#### Closing Browser Without Fetching
|
|
535
587
|
To close the browser instance without performing any fetch operation:
|
|
536
588
|
```json
|
|
@@ -551,19 +603,20 @@ If `proxy` is set, `useSystemProxy` will be automatically set to false.
|
|
|
551
603
|
|
|
552
604
|
### Debug Output
|
|
553
605
|
|
|
554
|
-
When `debug: true` is set,
|
|
606
|
+
When `debug: true` is set, logs will be output to stderr with the following prefixes:
|
|
555
607
|
- `[MCP-SERVER]`: Logs from the MCP server
|
|
556
608
|
- `[NODE-FETCH]`: Logs from the Node.js fetcher
|
|
557
609
|
- `[BROWSER-FETCH]`: Logs from the browser fetcher
|
|
558
610
|
- `[CLIENT]`: Logs from the client
|
|
559
|
-
- `[TOOLS]`: Logs from the
|
|
611
|
+
- `[TOOLS]`: Logs from the tool implementation
|
|
560
612
|
- `[FETCHER]`: Logs from the main fetcher interface
|
|
561
|
-
- `[CONTENT]`: Logs related to content
|
|
613
|
+
- `[CONTENT]`: Logs related to content handling
|
|
562
614
|
- `[CONTENT-PROCESSOR]`: Logs from the HTML content processor
|
|
563
615
|
- `[CONTENT-SIZE]`: Logs related to content size management
|
|
564
616
|
- `[CHUNK-MANAGER]`: Logs related to content chunking operations
|
|
565
617
|
- `[ERROR-HANDLER]`: Logs related to error handling
|
|
566
618
|
- `[BROWSER-MANAGER]`: Logs from the browser instance manager
|
|
619
|
+
- `[CONTENT-EXTRACTOR]`: Logs from the content extractor
|
|
567
620
|
|
|
568
621
|
|
|
569
622
|
## License
|
package/README.zh.md
CHANGED
|
@@ -40,6 +40,7 @@ fetch-mcp/
|
|
|
40
40
|
│ │ ├── utils/ # 工具模块
|
|
41
41
|
│ │ │ ├── ChunkManager.ts # 内容分块功能
|
|
42
42
|
│ │ │ ├── ContentProcessor.ts # HTML到文本转换
|
|
43
|
+
│ │ │ ├── ContentExtractor.ts # 内容智能提取
|
|
43
44
|
│ │ │ ├── ContentSizeManager.ts # 内容大小限制
|
|
44
45
|
│ │ │ └── ErrorHandler.ts # 错误处理
|
|
45
46
|
│ │ ├── server/ # 服务器相关模块
|
|
@@ -79,6 +80,10 @@ fetch-mcp/
|
|
|
79
80
|
- 详细的调试日志输出到标准错误流
|
|
80
81
|
- 支持中英文双语国际化
|
|
81
82
|
- 模块化设计,便于维护和扩展
|
|
83
|
+
- **智能内容提取**:基于Mozilla Readability库,能够从网页中提取有意义的主要内容,过滤广告和导航元素
|
|
84
|
+
- **元数据支持**:能够提取网页元数据,如标题、作者、发布日期、站点信息等
|
|
85
|
+
- **智能内容分析**:自动检测页面是否包含有意义的内容,过滤登录页、错误页等无实质内容的页面
|
|
86
|
+
- **浏览器自动化增强**:支持页面滚动、Cookie管理、选择器等待等高级浏览器交互
|
|
82
87
|
|
|
83
88
|
## 安装
|
|
84
89
|
|
|
@@ -525,9 +530,34 @@ console.log('调试获取提示:', debugPrompt);
|
|
|
525
530
|
- `saveCookies`: 浏览器模式下是否保存cookies(可选,默认为true)
|
|
526
531
|
- `closeBrowser`: 是否关闭浏览器实例(可选,默认为false)
|
|
527
532
|
|
|
533
|
+
#### 内容提取参数
|
|
534
|
+
- `extractContent`: 是否使用Readability算法提取主要内容(可选,默认为false)
|
|
535
|
+
- `includeMetadata`: 是否在提取的内容中包含元数据(可选,默认为false,仅当`extractContent`为true时有效)
|
|
536
|
+
- `fallbackToOriginal`: 当提取失败时是否回退到原始内容(可选,默认为true,仅当`extractContent`为true时有效)
|
|
537
|
+
|
|
528
538
|
#### 调试参数
|
|
529
539
|
- `debug`: 是否启用调试输出(可选,默认为false)
|
|
530
540
|
|
|
541
|
+
### 内容提取特性
|
|
542
|
+
|
|
543
|
+
使用内容提取功能可以获取网页的核心内容,过滤掉导航栏、广告、侧边栏等干扰元素:
|
|
544
|
+
|
|
545
|
+
```json
|
|
546
|
+
{
|
|
547
|
+
"url": "https://example.com/article",
|
|
548
|
+
"extractContent": true,
|
|
549
|
+
"includeMetadata": true
|
|
550
|
+
}
|
|
551
|
+
```
|
|
552
|
+
|
|
553
|
+
提取的内容将包括以下元数据(如果可用):
|
|
554
|
+
- 标题 (title)
|
|
555
|
+
- 作者 (byline)
|
|
556
|
+
- 站点名称 (siteName)
|
|
557
|
+
- 摘要 (excerpt)
|
|
558
|
+
- 内容长度 (length)
|
|
559
|
+
- 可读性标志 (isReaderable)
|
|
560
|
+
|
|
531
561
|
### 特殊用法
|
|
532
562
|
|
|
533
563
|
#### 不获取内容直接关闭浏览器
|
|
@@ -563,6 +593,7 @@ console.log('调试获取提示:', debugPrompt);
|
|
|
563
593
|
- `[CHUNK-MANAGER]`: 内容分块操作相关的日志
|
|
564
594
|
- `[ERROR-HANDLER]`: 错误处理相关的日志
|
|
565
595
|
- `[BROWSER-MANAGER]`: 浏览器实例管理器的日志
|
|
596
|
+
- `[CONTENT-EXTRACTOR]`: 内容提取器的日志
|
|
566
597
|
|
|
567
598
|
|
|
568
599
|
## 许可证
|
package/dist/src/client.js
CHANGED
|
@@ -81,7 +81,7 @@ function responseRequiresBrowser(response, debug = false) {
|
|
|
81
81
|
* @param params 请求参数 (Request parameters)
|
|
82
82
|
* @param debug 是否为调试模式 (Whether in debug mode)
|
|
83
83
|
*/
|
|
84
|
-
function
|
|
84
|
+
function _saveChunkInfo(result, params, debug = false) {
|
|
85
85
|
if (result.isChunked && result.hasMoreChunks) {
|
|
86
86
|
try {
|
|
87
87
|
const chunkInfo = {
|
|
@@ -101,9 +101,9 @@ function saveChunkInfo(result, params, debug = false) {
|
|
|
101
101
|
}, COMPONENTS.CLIENT);
|
|
102
102
|
}
|
|
103
103
|
}
|
|
104
|
-
catch (
|
|
104
|
+
catch (_error) {
|
|
105
105
|
if (debug) {
|
|
106
|
-
log('client.chunkInfoSaveError', debug, { error: String(
|
|
106
|
+
log('client.chunkInfoSaveError', debug, { error: String(_error) }, COMPONENTS.CLIENT);
|
|
107
107
|
}
|
|
108
108
|
}
|
|
109
109
|
}
|
|
@@ -113,7 +113,7 @@ function saveChunkInfo(result, params, debug = false) {
|
|
|
113
113
|
* @param debug 是否为调试模式 (Whether in debug mode)
|
|
114
114
|
* @returns 分段信息 (Chunk information)
|
|
115
115
|
*/
|
|
116
|
-
function
|
|
116
|
+
function _loadChunkInfo(debug = false) {
|
|
117
117
|
try {
|
|
118
118
|
if (fs.existsSync(CHUNK_INFO_FILE)) {
|
|
119
119
|
const chunkInfoStr = fs.readFileSync(CHUNK_INFO_FILE, 'utf8');
|
|
@@ -128,9 +128,9 @@ function loadChunkInfo(debug = false) {
|
|
|
128
128
|
return chunkInfo;
|
|
129
129
|
}
|
|
130
130
|
}
|
|
131
|
-
catch (
|
|
131
|
+
catch (_error) {
|
|
132
132
|
if (debug) {
|
|
133
|
-
log('client.chunkInfoLoadError', debug, { error: String(
|
|
133
|
+
log('client.chunkInfoLoadError', debug, { error: String(_error) }, COMPONENTS.CLIENT);
|
|
134
134
|
}
|
|
135
135
|
}
|
|
136
136
|
return null;
|
|
@@ -140,17 +140,19 @@ function loadChunkInfo(debug = false) {
|
|
|
140
140
|
* @param result 响应结果 (Response result)
|
|
141
141
|
* @returns 包含分段信息的结果 (Result with chunk information)
|
|
142
142
|
*/
|
|
143
|
-
function
|
|
143
|
+
function _parseChunkInfo(result) {
|
|
144
144
|
if (result.isError || !result.content || !result.content[0] || !result.content[0].text) {
|
|
145
145
|
return result;
|
|
146
146
|
}
|
|
147
147
|
const content = result.content[0].text;
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
148
|
+
// 首先检查API直接返回的isLastChunk标识
|
|
149
|
+
const isLastChunk = result.isLastChunk === true;
|
|
150
|
+
// 支持多种分块ID匹配模式 (Support multiple chunking ID matching patterns)
|
|
151
|
+
const systemNoteMatch = content.match(/=== SYSTEM NOTE ===\s*([\s\S]*?)\s*={19}/);
|
|
152
|
+
const chunkIdMatch = systemNoteMatch ?
|
|
153
|
+
systemNoteMatch[1].match(/chunkId(?:=|\s*[:=]\s*|\s*)?["']?([^"',\s]+)["']?/i) :
|
|
154
|
+
content.match(/chunkId="([^"]+)"/);
|
|
155
|
+
// 如果没有找到分块ID,返回原始结果 (If no chunk ID found, return original result)
|
|
154
156
|
if (!chunkIdMatch) {
|
|
155
157
|
return result;
|
|
156
158
|
}
|
|
@@ -161,19 +163,37 @@ function parseChunkInfo(result) {
|
|
|
161
163
|
let fetchedBytes = 0;
|
|
162
164
|
let totalBytes = 0;
|
|
163
165
|
let remainingBytes = 0;
|
|
166
|
+
// 查找字节级分块信息 (Look for byte-level chunking information)
|
|
167
|
+
const byteInfoMatch = systemNoteMatch ?
|
|
168
|
+
systemNoteMatch[1].match(/(\d+(?:,\d+)*)\s*bytes\s*(?:retrieved|fetched).*?(\d+)%\s*of\s*(?:the\s*)?total\s*(\d+(?:,\d+)*)\s*bytes/) :
|
|
169
|
+
content.match(/retrieved ([,\d]+) bytes \((\d+)% of total ([,\d]+) bytes\)/);
|
|
170
|
+
const remainingBytesMatch = systemNoteMatch ?
|
|
171
|
+
systemNoteMatch[1].match(/(\d+(?:,\d+)*)\s*bytes\s*(?:are\s*)?remaining/) :
|
|
172
|
+
content.match(/([,\d]+) bytes remaining/);
|
|
173
|
+
// 查找旧版分块信息 (Look for old version chunking information)
|
|
174
|
+
const partMatch = systemNoteMatch ?
|
|
175
|
+
systemNoteMatch[1].match(/This is part (\d+) of (\d+)/) :
|
|
176
|
+
content.match(/This is part (\d+) of (\d+)/);
|
|
177
|
+
// 检查是否有"最后一部分"的提示 (Check if there's a "last part" hint)
|
|
178
|
+
const lastPartMatch = systemNoteMatch ?
|
|
179
|
+
systemNoteMatch[1].match(/(?:this|the) (?:is|last|final) (?:the|last|final)? part/i) :
|
|
180
|
+
content.match(/(?:this|the) (?:is|last|final) (?:the|last|final)? part/i);
|
|
181
|
+
// 如果有直接指示这是最后一部分的消息,设置isLastChunk为true (If there's a message directly indicating this is the last part, set isLastChunk to true)
|
|
182
|
+
const contentIndicatesLastChunk = !!lastPartMatch || content.includes("No further requests needed");
|
|
164
183
|
// 检查是否包含字节级分块信息 (Check if contains byte-level chunking info)
|
|
165
184
|
if (byteInfoMatch) {
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
185
|
+
// 移除千位分隔符逗号 (Remove thousands separator commas)
|
|
186
|
+
fetchedBytes = parseInt(byteInfoMatch[1].replace(/,/g, ''), 10);
|
|
187
|
+
const _percentage = parseInt(byteInfoMatch[2], 10);
|
|
188
|
+
totalBytes = parseInt(byteInfoMatch[3].replace(/,/g, ''), 10);
|
|
169
189
|
if (remainingBytesMatch) {
|
|
170
|
-
remainingBytes = parseInt(remainingBytesMatch[1], 10);
|
|
190
|
+
remainingBytes = parseInt(remainingBytesMatch[1].replace(/,/g, ''), 10);
|
|
171
191
|
}
|
|
172
192
|
else {
|
|
173
193
|
remainingBytes = totalBytes - fetchedBytes;
|
|
174
194
|
}
|
|
175
|
-
//
|
|
176
|
-
hasMoreChunks = remainingBytes
|
|
195
|
+
// 如果有明确的isLastChunk标识或内容中有指示,或者剩余字节为0,表示没有更多分块
|
|
196
|
+
hasMoreChunks = !(isLastChunk || contentIndicatesLastChunk || remainingBytes <= 0);
|
|
177
197
|
// 估算总分块数 (Estimate total chunks)
|
|
178
198
|
if (fetchedBytes > 0) {
|
|
179
199
|
// 估算总块数,基于已获取的字节数 (Estimate total chunks based on fetched bytes)
|
|
@@ -186,7 +206,8 @@ function parseChunkInfo(result) {
|
|
|
186
206
|
else if (partMatch) {
|
|
187
207
|
currentChunk = parseInt(partMatch[1], 10) - 1; // 转换为0索引 (Convert to 0-based index)
|
|
188
208
|
totalChunks = parseInt(partMatch[2], 10);
|
|
189
|
-
|
|
209
|
+
// 如果有明确的isLastChunk标识或内容中有指示,或者当前是最后一块,表示没有更多分块
|
|
210
|
+
hasMoreChunks = !(isLastChunk || contentIndicatesLastChunk || currentChunk >= totalChunks - 1);
|
|
190
211
|
}
|
|
191
212
|
return {
|
|
192
213
|
...result,
|
|
@@ -197,7 +218,9 @@ function parseChunkInfo(result) {
|
|
|
197
218
|
totalChunks,
|
|
198
219
|
fetchedBytes,
|
|
199
220
|
totalBytes,
|
|
200
|
-
remainingBytes
|
|
221
|
+
remainingBytes,
|
|
222
|
+
// 合并所有最后分块的判断条件
|
|
223
|
+
isLastChunk: isLastChunk || contentIndicatesLastChunk || remainingBytes <= 0 || (partMatch && currentChunk >= totalChunks - 1)
|
|
201
224
|
};
|
|
202
225
|
}
|
|
203
226
|
/**
|
|
@@ -265,7 +288,17 @@ async function smartFetch(params, client) {
|
|
|
265
288
|
async function main() {
|
|
266
289
|
// 检查是否请求所有分段内容 (Check if requesting all chunks)
|
|
267
290
|
const allChunksFlag = process.argv.includes('--all-chunks');
|
|
268
|
-
|
|
291
|
+
// 获取方法 (Get method)
|
|
292
|
+
if (process.argv.length < 3) {
|
|
293
|
+
log('client.usageInfo', true, { info: 'node src/client.js <method> <params_json> [--debug] [--all-chunks] [--max-chunks=N]' }, COMPONENTS.CLIENT);
|
|
294
|
+
log('client.exampleUsage', true, { example: 'node src/client.js fetch_html {"url":"https://example.com"}' }, COMPONENTS.CLIENT);
|
|
295
|
+
log('client.chunkUsageInfo', true, {}, COMPONENTS.CLIENT);
|
|
296
|
+
log('client.allChunksUsageInfo', true, {}, COMPONENTS.CLIENT);
|
|
297
|
+
log('client.maxChunksUsageInfo', true, { default: DEFAULT_MAX_CHUNKS }, COMPONENTS.CLIENT);
|
|
298
|
+
log('client.alternateUsageInfo', true, { example: 'node src/client.js fetch_html --url=https://example.com --startCursor=0' }, COMPONENTS.CLIENT);
|
|
299
|
+
process.exit(1);
|
|
300
|
+
}
|
|
301
|
+
const method = process.argv[2];
|
|
269
302
|
let params = {};
|
|
270
303
|
let paramsJson = "{}";
|
|
271
304
|
// 获取最大分块数限制 (Get maximum chunk limit)
|
|
@@ -278,17 +311,6 @@ async function main() {
|
|
|
278
311
|
}
|
|
279
312
|
}
|
|
280
313
|
// 解析命令行参数 (Parse command line arguments)
|
|
281
|
-
if (process.argv.length < 3) {
|
|
282
|
-
log('client.usageInfo', true, { info: 'node src/client.js <method> <params_json> [--debug] [--all-chunks] [--max-chunks=N]' }, COMPONENTS.CLIENT);
|
|
283
|
-
log('client.exampleUsage', true, { example: 'node src/client.js fetch_html {"url":"https://example.com"}' }, COMPONENTS.CLIENT);
|
|
284
|
-
log('client.chunkUsageInfo', true, {}, COMPONENTS.CLIENT);
|
|
285
|
-
log('client.allChunksUsageInfo', true, {}, COMPONENTS.CLIENT);
|
|
286
|
-
log('client.maxChunksUsageInfo', true, { default: DEFAULT_MAX_CHUNKS }, COMPONENTS.CLIENT);
|
|
287
|
-
log('client.alternateUsageInfo', true, { example: 'node src/client.js fetch_html --url=https://example.com --startCursor=0' }, COMPONENTS.CLIENT);
|
|
288
|
-
process.exit(1);
|
|
289
|
-
}
|
|
290
|
-
method = process.argv[2];
|
|
291
|
-
// 检查参数格式:JSON 或 命令行参数 (Check parameter format: JSON or command line arguments)
|
|
292
314
|
const thirdArg = process.argv[3];
|
|
293
315
|
const isJsonParams = thirdArg && thirdArg.startsWith('{') && thirdArg.endsWith('}');
|
|
294
316
|
if (isJsonParams) {
|
|
@@ -297,8 +319,8 @@ async function main() {
|
|
|
297
319
|
try {
|
|
298
320
|
params = JSON.parse(paramsJson);
|
|
299
321
|
}
|
|
300
|
-
catch (
|
|
301
|
-
log('client.invalidJson', true, { error: String(
|
|
322
|
+
catch (_error) {
|
|
323
|
+
log('client.invalidJson', true, { error: String(_error) }, COMPONENTS.CLIENT);
|
|
302
324
|
process.exit(1);
|
|
303
325
|
}
|
|
304
326
|
}
|
|
@@ -366,7 +388,7 @@ async function main() {
|
|
|
366
388
|
proxyUrl = proxyMatch[1].trim();
|
|
367
389
|
}
|
|
368
390
|
}
|
|
369
|
-
catch (
|
|
391
|
+
catch (_error) {
|
|
370
392
|
// 忽略错误 (Ignore errors)
|
|
371
393
|
}
|
|
372
394
|
}
|
|
@@ -379,7 +401,7 @@ async function main() {
|
|
|
379
401
|
proxyUrl = proxyMatch[1].trim();
|
|
380
402
|
}
|
|
381
403
|
}
|
|
382
|
-
catch (
|
|
404
|
+
catch (_error) {
|
|
383
405
|
// 忽略错误 (Ignore errors)
|
|
384
406
|
}
|
|
385
407
|
}
|
|
@@ -391,7 +413,7 @@ async function main() {
|
|
|
391
413
|
log('client.noSystemProxy', debug, {}, COMPONENTS.CLIENT);
|
|
392
414
|
}
|
|
393
415
|
}
|
|
394
|
-
catch (
|
|
416
|
+
catch (_error) {
|
|
395
417
|
// 忽略错误 (Ignore errors)
|
|
396
418
|
}
|
|
397
419
|
}
|
|
@@ -431,33 +453,28 @@ async function main() {
|
|
|
431
453
|
method
|
|
432
454
|
};
|
|
433
455
|
// 获取第一段内容 (Fetch first chunk)
|
|
434
|
-
|
|
456
|
+
const result = await smartFetch(initialParams, client);
|
|
435
457
|
// 调试输出,查看响应结构 (Debug output, check response structure)
|
|
436
458
|
if (debug) {
|
|
437
459
|
log('client.responseStructure', debug, { structure: JSON.stringify(result, null, 2) }, COMPONENTS.CLIENT);
|
|
438
460
|
}
|
|
439
461
|
// 从响应内容中解析分段信息 (Parse chunk information from response content)
|
|
440
|
-
let chunkId;
|
|
441
|
-
let currentChunk = 0;
|
|
442
|
-
let totalChunks = 1;
|
|
443
462
|
let hasMoreChunks = false;
|
|
463
|
+
let chunkId = '';
|
|
464
|
+
let totalChunks = 1;
|
|
465
|
+
let currentChunk = 0;
|
|
466
|
+
// 检查是否有明确的最后分块标识
|
|
467
|
+
const isLastChunk = result.isLastChunk === true;
|
|
468
|
+
// 解析响应内容中的分段信息 (Parse chunk information from response content)
|
|
444
469
|
if (result.content && result.content[0] && result.content[0].text) {
|
|
445
470
|
const content = result.content[0].text;
|
|
446
|
-
|
|
447
|
-
const
|
|
448
|
-
const
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
const
|
|
452
|
-
|
|
453
|
-
// 尝试兼容新旧两种格式 (Try to be compatible with both new and old formats)
|
|
454
|
-
const remainingBytesMatch = systemNoteMatch ?
|
|
455
|
-
systemNoteMatch[1].match(/(\d+(?:,\d+)*)\s*bytes\s*(?:are\s*)?remaining/) : null;
|
|
456
|
-
const moreRequestsMatch = systemNoteMatch ?
|
|
457
|
-
systemNoteMatch[1].match(/(?:a|ap)proximately\s*(\d+)\s*more\s*requests?\s*(?:are\s*)?needed/) : null;
|
|
458
|
-
// 旧版分块信息匹配 (Old version chunking information matching)
|
|
459
|
-
const partMatch = systemNoteMatch ?
|
|
460
|
-
systemNoteMatch[1].match(/This is part (\d+) of (\d+)/) : null;
|
|
471
|
+
const chunkIdMatch = content.match(/chunkId="([^"]+)"/);
|
|
472
|
+
const startCursorMatch = content.match(/startCursor=(\d+)/);
|
|
473
|
+
const bytesRetrievedMatch = content.match(/retrieved ([,\d]+) bytes \((\d+)% of total ([,\d]+) bytes\)/);
|
|
474
|
+
const remainingBytesMatch = content.match(/([,\d]+) bytes remaining/);
|
|
475
|
+
const moreRequestsMatch = content.match(/approximately (\d+) more requests needed/);
|
|
476
|
+
const partMatch = content.match(/This is part (\d+) of (\d+)/);
|
|
477
|
+
// 如果找到了分块ID匹配 (If chunk ID match found)
|
|
461
478
|
if (chunkIdMatch) {
|
|
462
479
|
chunkId = chunkIdMatch[1];
|
|
463
480
|
// 判断是否有字节级分块信息 (Check if there's byte-level chunking info)
|
|
@@ -466,7 +483,7 @@ async function main() {
|
|
|
466
483
|
const fetchedBytesStr = bytesRetrievedMatch[1].replace(/,/g, '');
|
|
467
484
|
const totalBytesStr = bytesRetrievedMatch[3].replace(/,/g, '');
|
|
468
485
|
const fetchedBytes = parseInt(fetchedBytesStr, 10);
|
|
469
|
-
const
|
|
486
|
+
const _percentage = parseInt(bytesRetrievedMatch[2], 10);
|
|
470
487
|
const totalBytes = parseInt(totalBytesStr, 10);
|
|
471
488
|
let remainingBytes = 0;
|
|
472
489
|
if (remainingBytesMatch) {
|
|
@@ -475,8 +492,8 @@ async function main() {
|
|
|
475
492
|
else {
|
|
476
493
|
remainingBytes = totalBytes - fetchedBytes;
|
|
477
494
|
}
|
|
478
|
-
//
|
|
479
|
-
hasMoreChunks = remainingBytes > 0;
|
|
495
|
+
// 如果使用isLastChunk标识,优先使用它;否则基于remainingBytes计算
|
|
496
|
+
hasMoreChunks = isLastChunk ? false : remainingBytes > 0;
|
|
480
497
|
// 估算剩余的请求次数 (Estimate remaining requests)
|
|
481
498
|
let estimatedRemainingRequests = 0;
|
|
482
499
|
if (moreRequestsMatch) {
|
|
@@ -493,38 +510,48 @@ async function main() {
|
|
|
493
510
|
result.fetchedBytes = fetchedBytes;
|
|
494
511
|
result.totalBytes = totalBytes;
|
|
495
512
|
result.remainingBytes = remainingBytes;
|
|
496
|
-
//
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
chunkId,
|
|
500
|
-
fetchedBytes,
|
|
501
|
-
totalBytes,
|
|
502
|
-
remainingBytes,
|
|
503
|
-
estimatedRemainingRequests
|
|
504
|
-
}, COMPONENTS.CLIENT);
|
|
505
|
-
}
|
|
513
|
+
// 如果服务器返回了isLastChunk属性,使用它;否则基于remainingBytes计算
|
|
514
|
+
result.isLastChunk = isLastChunk || remainingBytes <= 0;
|
|
515
|
+
result.hasMoreChunks = hasMoreChunks;
|
|
506
516
|
}
|
|
507
|
-
//
|
|
517
|
+
// 如果没有字节级信息,则使用分块匹配 (If no byte-level info, use chunk matching)
|
|
508
518
|
else if (partMatch) {
|
|
509
|
-
currentChunk = parseInt(partMatch[1], 10) - 1;
|
|
519
|
+
currentChunk = parseInt(partMatch[1], 10) - 1;
|
|
510
520
|
totalChunks = parseInt(partMatch[2], 10);
|
|
511
|
-
hasMoreChunks = currentChunk < totalChunks - 1;
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
currentChunk,
|
|
517
|
-
totalChunks,
|
|
518
|
-
hasMoreChunks
|
|
519
|
-
}, COMPONENTS.CLIENT);
|
|
520
|
-
}
|
|
521
|
+
hasMoreChunks = isLastChunk ? false : currentChunk < totalChunks - 1;
|
|
522
|
+
}
|
|
523
|
+
// 如果有启动游标信息,这表明还有更多分块 (If start cursor info exists, indicates more chunks)
|
|
524
|
+
else if (startCursorMatch) {
|
|
525
|
+
hasMoreChunks = !isLastChunk && true;
|
|
521
526
|
}
|
|
527
|
+
// 使用解析出的分段信息 (Use parsed chunk information)
|
|
528
|
+
result.isChunked = hasMoreChunks;
|
|
529
|
+
result.hasMoreChunks = hasMoreChunks;
|
|
530
|
+
result.chunkId = chunkId;
|
|
531
|
+
result.currentChunk = currentChunk;
|
|
532
|
+
result.totalChunks = totalChunks;
|
|
533
|
+
// 保存分段信息到临时文件 (Save chunk information to temporary file)
|
|
534
|
+
_saveChunkInfo(result, params, debug);
|
|
522
535
|
}
|
|
523
536
|
}
|
|
524
|
-
//
|
|
537
|
+
// 调试输出,查看解析后的分段信息 (Debug output, check parsed chunk information)
|
|
538
|
+
if (debug) {
|
|
539
|
+
log('client.chunkInfoParsed', debug, {
|
|
540
|
+
isChunked: result.isChunked,
|
|
541
|
+
hasMoreChunks,
|
|
542
|
+
isLastChunk,
|
|
543
|
+
chunkId,
|
|
544
|
+
currentChunk,
|
|
545
|
+
totalChunks,
|
|
546
|
+
fetchedBytes: result.fetchedBytes,
|
|
547
|
+
totalBytes: result.totalBytes,
|
|
548
|
+
remainingBytes: result.remainingBytes
|
|
549
|
+
}, COMPONENTS.CLIENT);
|
|
550
|
+
}
|
|
551
|
+
// 输出结果 (Output result)
|
|
525
552
|
console.log(JSON.stringify(result, null, 2));
|
|
526
553
|
// 如果启用了获取所有分段内容,并且结果包含分段信息 (If all chunks mode is enabled and result contains chunk information)
|
|
527
|
-
if (allChunksFlag && hasMoreChunks && chunkId) {
|
|
554
|
+
if (allChunksFlag && hasMoreChunks && chunkId && !isLastChunk) {
|
|
528
555
|
try {
|
|
529
556
|
// 确保totalChunks的计算正确 (Ensure totalChunks calculation is correct)
|
|
530
557
|
if (result.totalBytes && result.fetchedBytes) {
|
|
@@ -640,14 +667,14 @@ async function main() {
|
|
|
640
667
|
log('client.completeChunksFetched', true, { total: totalChunks }, COMPONENTS.CLIENT);
|
|
641
668
|
}
|
|
642
669
|
}
|
|
643
|
-
catch (
|
|
670
|
+
catch (_error) {
|
|
644
671
|
// 处理获取分块内容过程中的错误 (Handle errors during chunk fetching)
|
|
645
|
-
log('client.fetchingChunksError', true, { error: String(
|
|
646
|
-
log('client.fetchingChunksErrorMessage', true, { error: String(
|
|
672
|
+
log('client.fetchingChunksError', true, { error: String(_error) }, COMPONENTS.CLIENT);
|
|
673
|
+
log('client.fetchingChunksErrorMessage', true, { error: String(_error) }, COMPONENTS.CLIENT);
|
|
647
674
|
}
|
|
648
675
|
}
|
|
649
676
|
// 如果结果包含分段信息,但没有启用获取所有分段内容 (If result contains chunk information but all chunks mode is not enabled)
|
|
650
|
-
else if (hasMoreChunks && chunkId) {
|
|
677
|
+
else if (hasMoreChunks && chunkId && !isLastChunk) {
|
|
651
678
|
log('client.hasMoreChunks', debug, {
|
|
652
679
|
current: currentChunk,
|
|
653
680
|
total: totalChunks,
|
|
@@ -672,16 +699,16 @@ async function main() {
|
|
|
672
699
|
}
|
|
673
700
|
}
|
|
674
701
|
}
|
|
675
|
-
catch (
|
|
702
|
+
catch (_error) {
|
|
676
703
|
// 处理错误 (Handle error)
|
|
677
|
-
log('client.fatalError', true, { error: String(
|
|
704
|
+
log('client.fatalError', true, { error: String(_error) }, COMPONENTS.CLIENT);
|
|
678
705
|
// 返回标准错误结构体 (Return standard error structure)
|
|
679
706
|
const errorResult = {
|
|
680
707
|
isError: true,
|
|
681
708
|
content: [
|
|
682
709
|
{
|
|
683
710
|
type: "text",
|
|
684
|
-
text: `Fatal error: ${String(
|
|
711
|
+
text: `Fatal error: ${String(_error)}`
|
|
685
712
|
}
|
|
686
713
|
]
|
|
687
714
|
};
|