@lmcc-dev/mult-fetch-mcp-server 1.3.1 → 1.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/README.md +70 -15
  2. package/README.zh.md +31 -0
  3. package/dist/src/client.js +118 -91
  4. package/dist/src/index.js +0 -0
  5. package/dist/src/lib/fetchers/browser/BrowserFetcher.js +27 -10
  6. package/dist/src/lib/fetchers/browser/BrowserInstance.js +49 -43
  7. package/dist/src/lib/fetchers/common/BaseFetcher.js +106 -10
  8. package/dist/src/lib/fetchers/common/types.js +10 -1
  9. package/dist/src/lib/fetchers/common/utils.js +1 -1
  10. package/dist/src/lib/fetchers/node/HttpClient.js +47 -21
  11. package/dist/src/lib/fetchers/node/NodeFetcher.js +24 -8
  12. package/dist/src/lib/i18n/index.js +2 -2
  13. package/dist/src/lib/i18n/keys/client.js +1 -0
  14. package/dist/src/lib/i18n/keys/extractor.js +26 -0
  15. package/dist/src/lib/i18n/keys/fetcher.js +12 -1
  16. package/dist/src/lib/i18n/keys/index.js +1 -0
  17. package/dist/src/lib/i18n/keys/node.js +2 -0
  18. package/dist/src/lib/i18n/locales/en/client.js +1 -0
  19. package/dist/src/lib/i18n/locales/en/extractor.js +24 -0
  20. package/dist/src/lib/i18n/locales/en/fetcher.js +3 -1
  21. package/dist/src/lib/i18n/locales/en/index.js +2 -0
  22. package/dist/src/lib/i18n/locales/en/node.js +2 -0
  23. package/dist/src/lib/i18n/locales/zh/client.js +1 -0
  24. package/dist/src/lib/i18n/locales/zh/extractor.js +22 -0
  25. package/dist/src/lib/i18n/locales/zh/fetcher.js +13 -3
  26. package/dist/src/lib/i18n/locales/zh/index.js +2 -0
  27. package/dist/src/lib/i18n/locales/zh/node.js +2 -0
  28. package/dist/src/lib/i18n/logger.js +2 -2
  29. package/dist/src/lib/logger.js +38 -17
  30. package/dist/src/lib/server/browser.js +2 -2
  31. package/dist/src/lib/server/fetcher.js +0 -3
  32. package/dist/src/lib/server/index.js +2 -2
  33. package/dist/src/lib/server/prompts.js +4 -4
  34. package/dist/src/lib/server/tools.js +127 -354
  35. package/dist/src/lib/utils/ChunkManager.js +2 -2
  36. package/dist/src/lib/utils/ContentExtractor.js +141 -0
  37. package/dist/src/lib/utils/ContentProcessor.js +5 -11
  38. package/dist/src/lib/utils/ContentSizeManager.js +2 -2
  39. package/dist/src/lib/utils/ErrorHandler.js +1 -0
  40. package/dist/src/lib/utils/TemplateUtils.js +6 -2
  41. package/dist/src/lib/utils/UrlValidator.js +205 -0
  42. package/dist/src/mcp-server.js +0 -0
  43. package/dist/tests/client.test.js +1 -1
  44. package/dist/tests/fetch.test.js +0 -0
  45. package/dist/tests/fetchers/node/HttpClient.test.js +50 -0
  46. package/dist/tests/i18n-missing-keys.js +0 -0
  47. package/dist/tests/i18n-unused-keys.js +0 -0
  48. package/dist/tests/i18n.test.js +0 -0
  49. package/dist/tests/logger.test.js +0 -0
  50. package/dist/tests/mcp-server.test.js +0 -0
  51. package/dist/tests/setup.js +0 -0
  52. package/dist/tests/test-direct-client.js +0 -0
  53. package/dist/tests/test-extract-single.js +389 -0
  54. package/dist/tests/test-i18n.js +0 -0
  55. package/dist/tests/test-mcp-methods.js +0 -0
  56. package/dist/tests/test-mcp.js +0 -0
  57. package/dist/tests/test-mini4k.js +0 -0
  58. package/dist/tests/types.test.js +0 -0
  59. package/dist/tests/utils/ContentExtractor.test.js +173 -0
  60. package/dist/tests/utils/ContentProcessor.test.js +136 -0
  61. package/dist/tests/utils/TemplateUtils.test.js +118 -0
  62. package/dist/tests/utils/UrlValidator.test.js +58 -0
  63. package/package.json +33 -25
  64. package/dist/i18n-test-report.json +0 -8
  65. package/dist/i18n-unused-keys-report.json +0 -8
  66. package/dist/src/lib/BrowserFetcher.js +0 -787
  67. package/dist/src/lib/NodeFetcher.js +0 -492
  68. package/dist/src/lib/i18n/keys.js +0 -529
  69. package/dist/src/test-i18n.js +0 -139
  70. package/dist/tests/BrowserFetcher.test.js +0 -951
  71. package/dist/tests/NodeFetcher.test.js +0 -263
  72. package/dist/tests/i18n-remove-unused-keys.js +0 -236
  73. package/dist/tests/i18n-test-report.json +0 -2004
  74. package/dist/tests/src/lib/i18n/index.js +0 -108
  75. package/dist/tests/src/lib/i18n/keys/base.js +0 -47
  76. package/dist/tests/src/lib/i18n/keys/browser.js +0 -93
  77. package/dist/tests/src/lib/i18n/keys/client.js +0 -70
  78. package/dist/tests/src/lib/i18n/keys/errors.js +0 -34
  79. package/dist/tests/src/lib/i18n/keys/fetcher.js +0 -84
  80. package/dist/tests/src/lib/i18n/keys/index.js +0 -31
  81. package/dist/tests/src/lib/i18n/keys/node.js +0 -56
  82. package/dist/tests/src/lib/i18n/keys/prompts.js +0 -82
  83. package/dist/tests/src/lib/i18n/keys/resources.js +0 -50
  84. package/dist/tests/src/lib/i18n/keys/server.js +0 -64
  85. package/dist/tests/src/lib/i18n/keys/tools.js +0 -34
  86. package/dist/tests/src/lib/i18n/locales/en/browser.js +0 -88
  87. package/dist/tests/src/lib/i18n/locales/en/client.js +0 -66
  88. package/dist/tests/src/lib/i18n/locales/en/errors.js +0 -28
  89. package/dist/tests/src/lib/i18n/locales/en/fetcher.js +0 -71
  90. package/dist/tests/src/lib/i18n/locales/en/index.js +0 -29
  91. package/dist/tests/src/lib/i18n/locales/en/node.js +0 -51
  92. package/dist/tests/src/lib/i18n/locales/en/prompts.js +0 -52
  93. package/dist/tests/src/lib/i18n/locales/en/resources.js +0 -50
  94. package/dist/tests/src/lib/i18n/locales/en/server.js +0 -56
  95. package/dist/tests/src/lib/i18n/locales/en/tools.js +0 -28
  96. package/dist/tests/src/lib/i18n/locales/zh/browser.js +0 -87
  97. package/dist/tests/src/lib/i18n/locales/zh/client.js +0 -66
  98. package/dist/tests/src/lib/i18n/locales/zh/errors.js +0 -28
  99. package/dist/tests/src/lib/i18n/locales/zh/fetcher.js +0 -71
  100. package/dist/tests/src/lib/i18n/locales/zh/index.js +0 -29
  101. package/dist/tests/src/lib/i18n/locales/zh/node.js +0 -51
  102. package/dist/tests/src/lib/i18n/locales/zh/prompts.js +0 -53
  103. package/dist/tests/src/lib/i18n/locales/zh/resources.js +0 -50
  104. package/dist/tests/src/lib/i18n/locales/zh/server.js +0 -57
  105. package/dist/tests/src/lib/i18n/locales/zh/tools.js +0 -28
  106. package/dist/tests/src/lib/i18n/logger.js +0 -114
  107. package/dist/tests/src/lib/logger.js +0 -181
  108. package/dist/tests/tests/test-i18n.js +0 -588
  109. package/dist/vitest.config.js +0 -29
package/README.md CHANGED
@@ -1,3 +1,5 @@
1
+ [![MseeP.ai Security Assessment Badge](https://mseep.net/mseep-audited.png)](https://mseep.ai/app/lmcc-dev-mult-fetch-mcp-server)
2
+
1
3
  # mult-fetch-mcp-server
2
4
 
3
5
  [![npm version](https://img.shields.io/npm/v/@lmcc-dev/mult-fetch-mcp-server.svg)](https://www.npmjs.com/package/@lmcc-dev/mult-fetch-mcp-server)
@@ -33,20 +35,21 @@ This project implements an MCP-compliant client and server for communication bet
33
35
  fetch-mcp/
34
36
  ├── src/ # Source code directory
35
37
  │ ├── lib/ # Library files
36
- │ │ ├── fetchers/ # Web fetching implementations
38
+ │ │ ├── fetchers/ # Web fetching implementation
37
39
  │ │ │ ├── browser/ # Browser-based fetching
38
- │ │ │ │ ├── BrowserFetcher.ts # Browser fetching implementation
40
+ │ │ │ │ ├── BrowserFetcher.ts # Browser fetcher implementation
39
41
  │ │ │ │ ├── BrowserInstance.ts # Browser instance management
40
42
  │ │ │ │ └── PageOperations.ts # Page interaction operations
41
43
  │ │ │ ├── node/ # Node.js-based fetching
42
44
  │ │ │ └── common/ # Shared fetching utilities
43
45
  │ │ ├── utils/ # Utility modules
44
- │ │ │ ├── ChunkManager.ts # Content chunking functionality
46
+ │ │ │ ├── ChunkManager.ts # Content chunking
45
47
  │ │ │ ├── ContentProcessor.ts # HTML to text conversion
48
+ │ │ │ ├── ContentExtractor.ts # Intelligent content extraction
46
49
  │ │ │ ├── ContentSizeManager.ts # Content size limiting
47
- │ │ │ └── ErrorHandler.ts # Error management
50
+ │ │ │ └── ErrorHandler.ts # Error handling
48
51
  │ │ ├── server/ # Server-related modules
49
- │ │ │ ├── index.ts # Server entry point
52
+ │ │ │ ├── index.ts # Server entry
50
53
  │ │ │ ├── browser.ts # Browser management
51
54
  │ │ │ ├── fetcher.ts # Web fetching logic
52
55
  │ │ │ ├── tools.ts # Tool registration and handling
@@ -76,12 +79,16 @@ This project implements the Standard Input/Output (Stdio) transport method.
76
79
  - Implementation based on the official MCP SDK
77
80
  - Support for Standard Input/Output (Stdio) transport
78
81
  - Multiple web scraping methods (HTML, JSON, text, Markdown, plain text conversion)
79
- - Intelligent mode switching: automatically switches between standard requests and browser mode
80
- - Content size management: automatically splits large content into manageable chunks to overcome AI context size limitations
82
+ - Intelligent mode switching: automatic switching between standard requests and browser mode
83
+ - Content size management: automatically splits large content into manageable chunks to solve AI model context size limitations
81
84
  - Chunked content retrieval: ability to request specific chunks of large content while maintaining context continuity
82
- - Detailed debug logs output to standard error stream
83
- - Support for Chinese and English bilingual internationalization
85
+ - Detailed debug logging to stderr
86
+ - Bilingual internationalization (English and Chinese)
84
87
  - Modular design for easy maintenance and extension
88
+ - **Intelligent Content Extraction**: Based on Mozilla's Readability library, capable of extracting meaningful content from web pages while filtering out advertisements and navigation elements
89
+ - **Metadata Support**: Ability to extract webpage metadata such as title, author, publication date, and site information
90
+ - **Smart Content Detection**: Automatically detects if a page contains meaningful content, filtering out login pages, error pages, and other pages without substantial content
91
+ - **Browser Automation Enhancements**: Support for page scrolling, cookie management, selector waiting, and other advanced browser interactions
85
92
 
86
93
  ## Installation
87
94
 
@@ -493,8 +500,8 @@ Each tool supports the following parameters:
493
500
 
494
501
  #### Basic Parameters
495
502
  - `url`: URL to fetch (required)
496
- - `headers`: Custom request headers (optional, default is {})
497
- - `proxy`: Proxy server URL in format http://host:port or https://host:port (optional)
503
+ - `headers`: Custom request headers (optional, default {})
504
+ - `proxy`: Proxy server URL in the format http://host:port or https://host:port (optional)
498
505
 
499
506
  #### Network Control Parameters
500
507
  - `timeout`: Timeout in milliseconds (optional, default is 30000)
@@ -526,11 +533,58 @@ When content is split into chunks, the response includes metadata that allows th
526
533
  - `saveCookies`: Whether to save cookies in browser mode (optional, default is true)
527
534
  - `closeBrowser`: Whether to close the browser instance (optional, default is false)
528
535
 
536
+ #### Content Extraction Parameters
537
+ - `extractContent`: Whether to use the Readability algorithm to extract main content (optional, default false)
538
+ - `includeMetadata`: Whether to include metadata in the extracted content (optional, default false, only works when `extractContent` is true)
539
+ - `fallbackToOriginal`: Whether to fall back to the original content when extraction fails (optional, default true, only works when `extractContent` is true)
540
+
529
541
  #### Debug Parameters
530
- - `debug`: Whether to enable debug output (optional, default is false)
542
+ - `debug`: Whether to enable debug output (optional, default false)
543
+
544
+ ### Content Extraction Feature
545
+
546
+ Use the content extraction feature to get the core content of a webpage, filtering out navigation bars, advertisements, sidebars, and other distracting elements:
547
+
548
+ ```json
549
+ {
550
+ "url": "https://example.com/article",
551
+ "extractContent": true,
552
+ "includeMetadata": true
553
+ }
554
+ ```
555
+
556
+ The extracted content will include the following metadata (if available):
557
+ - Title
558
+ - Byline (author)
559
+ - Site name
560
+ - Excerpt
561
+ - Content length
562
+ - Readability flag (isReaderable)
531
563
 
532
564
  ### Special Usage
533
565
 
566
+ #### Content Extraction Examples
567
+
568
+ To extract only the meaningful content from an article webpage:
569
+
570
+ ```json
571
+ {
572
+ "url": "https://example.com/news/article",
573
+ "extractContent": true,
574
+ "includeMetadata": true
575
+ }
576
+ ```
577
+
578
+ For websites where content extraction might fail, you can use `fallbackToOriginal` to ensure you get some content:
579
+
580
+ ```json
581
+ {
582
+ "url": "https://example.com/complex-layout",
583
+ "extractContent": true,
584
+ "fallbackToOriginal": true
585
+ }
586
+ ```
587
+
534
588
  #### Closing Browser Without Fetching
535
589
  To close the browser instance without performing any fetch operation:
536
590
  ```json
@@ -551,19 +605,20 @@ If `proxy` is set, `useSystemProxy` will be automatically set to false.
551
605
 
552
606
  ### Debug Output
553
607
 
554
- When `debug: true` is set, the logs will be output to stderr with the following prefixes:
608
+ When `debug: true` is set, logs will be output to stderr with the following prefixes:
555
609
  - `[MCP-SERVER]`: Logs from the MCP server
556
610
  - `[NODE-FETCH]`: Logs from the Node.js fetcher
557
611
  - `[BROWSER-FETCH]`: Logs from the browser fetcher
558
612
  - `[CLIENT]`: Logs from the client
559
- - `[TOOLS]`: Logs from the tools implementation
613
+ - `[TOOLS]`: Logs from the tool implementation
560
614
  - `[FETCHER]`: Logs from the main fetcher interface
561
- - `[CONTENT]`: Logs related to content processing
615
+ - `[CONTENT]`: Logs related to content handling
562
616
  - `[CONTENT-PROCESSOR]`: Logs from the HTML content processor
563
617
  - `[CONTENT-SIZE]`: Logs related to content size management
564
618
  - `[CHUNK-MANAGER]`: Logs related to content chunking operations
565
619
  - `[ERROR-HANDLER]`: Logs related to error handling
566
620
  - `[BROWSER-MANAGER]`: Logs from the browser instance manager
621
+ - `[CONTENT-EXTRACTOR]`: Logs from the content extractor
567
622
 
568
623
 
569
624
  ## License
package/README.zh.md CHANGED
@@ -40,6 +40,7 @@ fetch-mcp/
40
40
  │ │ ├── utils/ # 工具模块
41
41
  │ │ │ ├── ChunkManager.ts # 内容分块功能
42
42
  │ │ │ ├── ContentProcessor.ts # HTML到文本转换
43
+ │ │ │ ├── ContentExtractor.ts # 内容智能提取
43
44
  │ │ │ ├── ContentSizeManager.ts # 内容大小限制
44
45
  │ │ │ └── ErrorHandler.ts # 错误处理
45
46
  │ │ ├── server/ # 服务器相关模块
@@ -79,6 +80,10 @@ fetch-mcp/
79
80
  - 详细的调试日志输出到标准错误流
80
81
  - 支持中英文双语国际化
81
82
  - 模块化设计,便于维护和扩展
83
+ - **智能内容提取**:基于Mozilla Readability库,能够从网页中提取有意义的主要内容,过滤广告和导航元素
84
+ - **元数据支持**:能够提取网页元数据,如标题、作者、发布日期、站点信息等
85
+ - **智能内容分析**:自动检测页面是否包含有意义的内容,过滤登录页、错误页等无实质内容的页面
86
+ - **浏览器自动化增强**:支持页面滚动、Cookie管理、选择器等待等高级浏览器交互
82
87
 
83
88
  ## 安装
84
89
 
@@ -525,9 +530,34 @@ console.log('调试获取提示:', debugPrompt);
525
530
  - `saveCookies`: 浏览器模式下是否保存cookies(可选,默认为true)
526
531
  - `closeBrowser`: 是否关闭浏览器实例(可选,默认为false)
527
532
 
533
+ #### 内容提取参数
534
+ - `extractContent`: 是否使用Readability算法提取主要内容(可选,默认为false)
535
+ - `includeMetadata`: 是否在提取的内容中包含元数据(可选,默认为false,仅当`extractContent`为true时有效)
536
+ - `fallbackToOriginal`: 当提取失败时是否回退到原始内容(可选,默认为true,仅当`extractContent`为true时有效)
537
+
528
538
  #### 调试参数
529
539
  - `debug`: 是否启用调试输出(可选,默认为false)
530
540
 
541
+ ### 内容提取特性
542
+
543
+ 使用内容提取功能可以获取网页的核心内容,过滤掉导航栏、广告、侧边栏等干扰元素:
544
+
545
+ ```json
546
+ {
547
+ "url": "https://example.com/article",
548
+ "extractContent": true,
549
+ "includeMetadata": true
550
+ }
551
+ ```
552
+
553
+ 提取的内容将包括以下元数据(如果可用):
554
+ - 标题 (title)
555
+ - 作者 (byline)
556
+ - 站点名称 (siteName)
557
+ - 摘要 (excerpt)
558
+ - 内容长度 (length)
559
+ - 可读性标志 (isReaderable)
560
+
531
561
  ### 特殊用法
532
562
 
533
563
  #### 不获取内容直接关闭浏览器
@@ -563,6 +593,7 @@ console.log('调试获取提示:', debugPrompt);
563
593
  - `[CHUNK-MANAGER]`: 内容分块操作相关的日志
564
594
  - `[ERROR-HANDLER]`: 错误处理相关的日志
565
595
  - `[BROWSER-MANAGER]`: 浏览器实例管理器的日志
596
+ - `[CONTENT-EXTRACTOR]`: 内容提取器的日志
566
597
 
567
598
 
568
599
  ## 许可证
@@ -81,7 +81,7 @@ function responseRequiresBrowser(response, debug = false) {
81
81
  * @param params 请求参数 (Request parameters)
82
82
  * @param debug 是否为调试模式 (Whether in debug mode)
83
83
  */
84
- function saveChunkInfo(result, params, debug = false) {
84
+ function _saveChunkInfo(result, params, debug = false) {
85
85
  if (result.isChunked && result.hasMoreChunks) {
86
86
  try {
87
87
  const chunkInfo = {
@@ -101,9 +101,9 @@ function saveChunkInfo(result, params, debug = false) {
101
101
  }, COMPONENTS.CLIENT);
102
102
  }
103
103
  }
104
- catch (error) {
104
+ catch (_error) {
105
105
  if (debug) {
106
- log('client.chunkInfoSaveError', debug, { error: String(error) }, COMPONENTS.CLIENT);
106
+ log('client.chunkInfoSaveError', debug, { error: String(_error) }, COMPONENTS.CLIENT);
107
107
  }
108
108
  }
109
109
  }
@@ -113,7 +113,7 @@ function saveChunkInfo(result, params, debug = false) {
113
113
  * @param debug 是否为调试模式 (Whether in debug mode)
114
114
  * @returns 分段信息 (Chunk information)
115
115
  */
116
- function loadChunkInfo(debug = false) {
116
+ function _loadChunkInfo(debug = false) {
117
117
  try {
118
118
  if (fs.existsSync(CHUNK_INFO_FILE)) {
119
119
  const chunkInfoStr = fs.readFileSync(CHUNK_INFO_FILE, 'utf8');
@@ -128,9 +128,9 @@ function loadChunkInfo(debug = false) {
128
128
  return chunkInfo;
129
129
  }
130
130
  }
131
- catch (error) {
131
+ catch (_error) {
132
132
  if (debug) {
133
- log('client.chunkInfoLoadError', debug, { error: String(error) }, COMPONENTS.CLIENT);
133
+ log('client.chunkInfoLoadError', debug, { error: String(_error) }, COMPONENTS.CLIENT);
134
134
  }
135
135
  }
136
136
  return null;
@@ -140,17 +140,19 @@ function loadChunkInfo(debug = false) {
140
140
  * @param result 响应结果 (Response result)
141
141
  * @returns 包含分段信息的结果 (Result with chunk information)
142
142
  */
143
- function parseChunkInfo(result) {
143
+ function _parseChunkInfo(result) {
144
144
  if (result.isError || !result.content || !result.content[0] || !result.content[0].text) {
145
145
  return result;
146
146
  }
147
147
  const content = result.content[0].text;
148
- const chunkIdMatch = content.match(/chunkId="([^"]+)"/);
149
- // 更新正则表达式,识别新的基于字节的分块信息 (Update regex to identify new byte-based chunking info)
150
- const byteInfoMatch = content.match(/(\d+) bytes retrieved \((\d+)% of total (\d+) bytes\)/);
151
- const remainingBytesMatch = content.match(/(\d+) bytes remaining/);
152
- // 存在chunkId但找不到分块信息,可能是老的格式 (ChunkId exists but no chunking info, might be old format)
153
- const partMatch = content.match(/This is part (\d+) of (\d+)/);
148
+ // 首先检查API直接返回的isLastChunk标识
149
+ const isLastChunk = result.isLastChunk === true;
150
+ // 支持多种分块ID匹配模式 (Support multiple chunking ID matching patterns)
151
+ const systemNoteMatch = content.match(/=== SYSTEM NOTE ===\s*([\s\S]*?)\s*={19}/);
152
+ const chunkIdMatch = systemNoteMatch ?
153
+ systemNoteMatch[1].match(/chunkId(?:=|\s*[:=]\s*|\s*)?["']?([^"',\s]+)["']?/i) :
154
+ content.match(/chunkId="([^"]+)"/);
155
+ // 如果没有找到分块ID,返回原始结果 (If no chunk ID found, return original result)
154
156
  if (!chunkIdMatch) {
155
157
  return result;
156
158
  }
@@ -161,19 +163,37 @@ function parseChunkInfo(result) {
161
163
  let fetchedBytes = 0;
162
164
  let totalBytes = 0;
163
165
  let remainingBytes = 0;
166
+ // 查找字节级分块信息 (Look for byte-level chunking information)
167
+ const byteInfoMatch = systemNoteMatch ?
168
+ systemNoteMatch[1].match(/(\d+(?:,\d+)*)\s*bytes\s*(?:retrieved|fetched).*?(\d+)%\s*of\s*(?:the\s*)?total\s*(\d+(?:,\d+)*)\s*bytes/) :
169
+ content.match(/retrieved ([,\d]+) bytes \((\d+)% of total ([,\d]+) bytes\)/);
170
+ const remainingBytesMatch = systemNoteMatch ?
171
+ systemNoteMatch[1].match(/(\d+(?:,\d+)*)\s*bytes\s*(?:are\s*)?remaining/) :
172
+ content.match(/([,\d]+) bytes remaining/);
173
+ // 查找旧版分块信息 (Look for old version chunking information)
174
+ const partMatch = systemNoteMatch ?
175
+ systemNoteMatch[1].match(/This is part (\d+) of (\d+)/) :
176
+ content.match(/This is part (\d+) of (\d+)/);
177
+ // 检查是否有"最后一部分"的提示 (Check if there's a "last part" hint)
178
+ const lastPartMatch = systemNoteMatch ?
179
+ systemNoteMatch[1].match(/(?:this|the) (?:is|last|final) (?:the|last|final)? part/i) :
180
+ content.match(/(?:this|the) (?:is|last|final) (?:the|last|final)? part/i);
181
+ // 如果有直接指示这是最后一部分的消息,设置isLastChunk为true (If there's a message directly indicating this is the last part, set isLastChunk to true)
182
+ const contentIndicatesLastChunk = !!lastPartMatch || content.includes("No further requests needed");
164
183
  // 检查是否包含字节级分块信息 (Check if contains byte-level chunking info)
165
184
  if (byteInfoMatch) {
166
- fetchedBytes = parseInt(byteInfoMatch[1], 10);
167
- const percentage = parseInt(byteInfoMatch[2], 10);
168
- totalBytes = parseInt(byteInfoMatch[3], 10);
185
+ // 移除千位分隔符逗号 (Remove thousands separator commas)
186
+ fetchedBytes = parseInt(byteInfoMatch[1].replace(/,/g, ''), 10);
187
+ const _percentage = parseInt(byteInfoMatch[2], 10);
188
+ totalBytes = parseInt(byteInfoMatch[3].replace(/,/g, ''), 10);
169
189
  if (remainingBytesMatch) {
170
- remainingBytes = parseInt(remainingBytesMatch[1], 10);
190
+ remainingBytes = parseInt(remainingBytesMatch[1].replace(/,/g, ''), 10);
171
191
  }
172
192
  else {
173
193
  remainingBytes = totalBytes - fetchedBytes;
174
194
  }
175
- // 如果还有剩余字节,表示有更多分块 (If there are remaining bytes, indicates more chunks)
176
- hasMoreChunks = remainingBytes > 0;
195
+ // 如果有明确的isLastChunk标识或内容中有指示,或者剩余字节为0,表示没有更多分块
196
+ hasMoreChunks = !(isLastChunk || contentIndicatesLastChunk || remainingBytes <= 0);
177
197
  // 估算总分块数 (Estimate total chunks)
178
198
  if (fetchedBytes > 0) {
179
199
  // 估算总块数,基于已获取的字节数 (Estimate total chunks based on fetched bytes)
@@ -186,7 +206,8 @@ function parseChunkInfo(result) {
186
206
  else if (partMatch) {
187
207
  currentChunk = parseInt(partMatch[1], 10) - 1; // 转换为0索引 (Convert to 0-based index)
188
208
  totalChunks = parseInt(partMatch[2], 10);
189
- hasMoreChunks = currentChunk < totalChunks - 1;
209
+ // 如果有明确的isLastChunk标识或内容中有指示,或者当前是最后一块,表示没有更多分块
210
+ hasMoreChunks = !(isLastChunk || contentIndicatesLastChunk || currentChunk >= totalChunks - 1);
190
211
  }
191
212
  return {
192
213
  ...result,
@@ -197,7 +218,9 @@ function parseChunkInfo(result) {
197
218
  totalChunks,
198
219
  fetchedBytes,
199
220
  totalBytes,
200
- remainingBytes
221
+ remainingBytes,
222
+ // 合并所有最后分块的判断条件
223
+ isLastChunk: isLastChunk || contentIndicatesLastChunk || remainingBytes <= 0 || (partMatch && currentChunk >= totalChunks - 1)
201
224
  };
202
225
  }
203
226
  /**
@@ -265,7 +288,17 @@ async function smartFetch(params, client) {
265
288
  async function main() {
266
289
  // 检查是否请求所有分段内容 (Check if requesting all chunks)
267
290
  const allChunksFlag = process.argv.includes('--all-chunks');
268
- let method;
291
+ // 获取方法 (Get method)
292
+ if (process.argv.length < 3) {
293
+ log('client.usageInfo', true, { info: 'node src/client.js <method> <params_json> [--debug] [--all-chunks] [--max-chunks=N]' }, COMPONENTS.CLIENT);
294
+ log('client.exampleUsage', true, { example: 'node src/client.js fetch_html {"url":"https://example.com"}' }, COMPONENTS.CLIENT);
295
+ log('client.chunkUsageInfo', true, {}, COMPONENTS.CLIENT);
296
+ log('client.allChunksUsageInfo', true, {}, COMPONENTS.CLIENT);
297
+ log('client.maxChunksUsageInfo', true, { default: DEFAULT_MAX_CHUNKS }, COMPONENTS.CLIENT);
298
+ log('client.alternateUsageInfo', true, { example: 'node src/client.js fetch_html --url=https://example.com --startCursor=0' }, COMPONENTS.CLIENT);
299
+ process.exit(1);
300
+ }
301
+ const method = process.argv[2];
269
302
  let params = {};
270
303
  let paramsJson = "{}";
271
304
  // 获取最大分块数限制 (Get maximum chunk limit)
@@ -278,17 +311,6 @@ async function main() {
278
311
  }
279
312
  }
280
313
  // 解析命令行参数 (Parse command line arguments)
281
- if (process.argv.length < 3) {
282
- log('client.usageInfo', true, { info: 'node src/client.js <method> <params_json> [--debug] [--all-chunks] [--max-chunks=N]' }, COMPONENTS.CLIENT);
283
- log('client.exampleUsage', true, { example: 'node src/client.js fetch_html {"url":"https://example.com"}' }, COMPONENTS.CLIENT);
284
- log('client.chunkUsageInfo', true, {}, COMPONENTS.CLIENT);
285
- log('client.allChunksUsageInfo', true, {}, COMPONENTS.CLIENT);
286
- log('client.maxChunksUsageInfo', true, { default: DEFAULT_MAX_CHUNKS }, COMPONENTS.CLIENT);
287
- log('client.alternateUsageInfo', true, { example: 'node src/client.js fetch_html --url=https://example.com --startCursor=0' }, COMPONENTS.CLIENT);
288
- process.exit(1);
289
- }
290
- method = process.argv[2];
291
- // 检查参数格式:JSON 或 命令行参数 (Check parameter format: JSON or command line arguments)
292
314
  const thirdArg = process.argv[3];
293
315
  const isJsonParams = thirdArg && thirdArg.startsWith('{') && thirdArg.endsWith('}');
294
316
  if (isJsonParams) {
@@ -297,8 +319,8 @@ async function main() {
297
319
  try {
298
320
  params = JSON.parse(paramsJson);
299
321
  }
300
- catch (e) {
301
- log('client.invalidJson', true, { error: String(e) }, COMPONENTS.CLIENT);
322
+ catch (_error) {
323
+ log('client.invalidJson', true, { error: String(_error) }, COMPONENTS.CLIENT);
302
324
  process.exit(1);
303
325
  }
304
326
  }
@@ -366,7 +388,7 @@ async function main() {
366
388
  proxyUrl = proxyMatch[1].trim();
367
389
  }
368
390
  }
369
- catch (error) {
391
+ catch (_error) {
370
392
  // 忽略错误 (Ignore errors)
371
393
  }
372
394
  }
@@ -379,7 +401,7 @@ async function main() {
379
401
  proxyUrl = proxyMatch[1].trim();
380
402
  }
381
403
  }
382
- catch (error) {
404
+ catch (_error) {
383
405
  // 忽略错误 (Ignore errors)
384
406
  }
385
407
  }
@@ -391,7 +413,7 @@ async function main() {
391
413
  log('client.noSystemProxy', debug, {}, COMPONENTS.CLIENT);
392
414
  }
393
415
  }
394
- catch (error) {
416
+ catch (_error) {
395
417
  // 忽略错误 (Ignore errors)
396
418
  }
397
419
  }
@@ -431,33 +453,28 @@ async function main() {
431
453
  method
432
454
  };
433
455
  // 获取第一段内容 (Fetch first chunk)
434
- let result = await smartFetch(initialParams, client);
456
+ const result = await smartFetch(initialParams, client);
435
457
  // 调试输出,查看响应结构 (Debug output, check response structure)
436
458
  if (debug) {
437
459
  log('client.responseStructure', debug, { structure: JSON.stringify(result, null, 2) }, COMPONENTS.CLIENT);
438
460
  }
439
461
  // 从响应内容中解析分段信息 (Parse chunk information from response content)
440
- let chunkId;
441
- let currentChunk = 0;
442
- let totalChunks = 1;
443
462
  let hasMoreChunks = false;
463
+ let chunkId = '';
464
+ let totalChunks = 1;
465
+ let currentChunk = 0;
466
+ // 检查是否有明确的最后分块标识
467
+ const isLastChunk = result.isLastChunk === true;
468
+ // 解析响应内容中的分段信息 (Parse chunk information from response content)
444
469
  if (result.content && result.content[0] && result.content[0].text) {
445
470
  const content = result.content[0].text;
446
- // Update regex to match chunk information in system notes
447
- const systemNoteMatch = content.match(/=== SYSTEM NOTE ===\s*([\s\S]*?)\s*={19}/);
448
- const chunkIdMatch = systemNoteMatch ? systemNoteMatch[1].match(/chunkId(?:=|\s*[:=]\s*|\s*)?["']?([^"',\s]+)["']?/i) : null;
449
- // 匹配字节级分块信息 (Match byte-level chunking information)
450
- // 更新正则表达式来适应新的系统注释格式 (Update regex to adapt to new system note format)
451
- const bytesRetrievedMatch = systemNoteMatch ?
452
- systemNoteMatch[1].match(/(?:You'?(?:ve|have))?\s*retrieved\s*(\d+(?:,\d+)*)\s*bytes\s*\((\d+)%\s*of\s*(?:the\s*)?total\s*(\d+(?:,\d+)*)\s*bytes\)/) : null;
453
- // 尝试兼容新旧两种格式 (Try to be compatible with both new and old formats)
454
- const remainingBytesMatch = systemNoteMatch ?
455
- systemNoteMatch[1].match(/(\d+(?:,\d+)*)\s*bytes\s*(?:are\s*)?remaining/) : null;
456
- const moreRequestsMatch = systemNoteMatch ?
457
- systemNoteMatch[1].match(/(?:a|ap)proximately\s*(\d+)\s*more\s*requests?\s*(?:are\s*)?needed/) : null;
458
- // 旧版分块信息匹配 (Old version chunking information matching)
459
- const partMatch = systemNoteMatch ?
460
- systemNoteMatch[1].match(/This is part (\d+) of (\d+)/) : null;
471
+ const chunkIdMatch = content.match(/chunkId="([^"]+)"/);
472
+ const startCursorMatch = content.match(/startCursor=(\d+)/);
473
+ const bytesRetrievedMatch = content.match(/retrieved ([,\d]+) bytes \((\d+)% of total ([,\d]+) bytes\)/);
474
+ const remainingBytesMatch = content.match(/([,\d]+) bytes remaining/);
475
+ const moreRequestsMatch = content.match(/approximately (\d+) more requests needed/);
476
+ const partMatch = content.match(/This is part (\d+) of (\d+)/);
477
+ // 如果找到了分块ID匹配 (If chunk ID match found)
461
478
  if (chunkIdMatch) {
462
479
  chunkId = chunkIdMatch[1];
463
480
  // 判断是否有字节级分块信息 (Check if there's byte-level chunking info)
@@ -466,7 +483,7 @@ async function main() {
466
483
  const fetchedBytesStr = bytesRetrievedMatch[1].replace(/,/g, '');
467
484
  const totalBytesStr = bytesRetrievedMatch[3].replace(/,/g, '');
468
485
  const fetchedBytes = parseInt(fetchedBytesStr, 10);
469
- const percentage = parseInt(bytesRetrievedMatch[2], 10);
486
+ const _percentage = parseInt(bytesRetrievedMatch[2], 10);
470
487
  const totalBytes = parseInt(totalBytesStr, 10);
471
488
  let remainingBytes = 0;
472
489
  if (remainingBytesMatch) {
@@ -475,8 +492,8 @@ async function main() {
475
492
  else {
476
493
  remainingBytes = totalBytes - fetchedBytes;
477
494
  }
478
- // 如果还有剩余字节,表示有更多分块 (If there are remaining bytes, indicates more chunks)
479
- hasMoreChunks = remainingBytes > 0;
495
+ // 如果使用isLastChunk标识,优先使用它;否则基于remainingBytes计算
496
+ hasMoreChunks = isLastChunk ? false : remainingBytes > 0;
480
497
  // 估算剩余的请求次数 (Estimate remaining requests)
481
498
  let estimatedRemainingRequests = 0;
482
499
  if (moreRequestsMatch) {
@@ -493,38 +510,48 @@ async function main() {
493
510
  result.fetchedBytes = fetchedBytes;
494
511
  result.totalBytes = totalBytes;
495
512
  result.remainingBytes = remainingBytes;
496
- // 调试输出,查看解析后的分段信息 (Debug output, check parsed chunk information)
497
- if (debug) {
498
- log('client.parsedByteChunkInfo', debug, {
499
- chunkId,
500
- fetchedBytes,
501
- totalBytes,
502
- remainingBytes,
503
- estimatedRemainingRequests
504
- }, COMPONENTS.CLIENT);
505
- }
513
+ // 如果服务器返回了isLastChunk属性,使用它;否则基于remainingBytes计算
514
+ result.isLastChunk = isLastChunk || remainingBytes <= 0;
515
+ result.hasMoreChunks = hasMoreChunks;
506
516
  }
507
- // 尝试匹配旧版的分块格式 (Try to match old version chunking format)
517
+ // 如果没有字节级信息,则使用分块匹配 (If no byte-level info, use chunk matching)
508
518
  else if (partMatch) {
509
- currentChunk = parseInt(partMatch[1], 10) - 1; // 转换为0索引 (Convert to 0-based index)
519
+ currentChunk = parseInt(partMatch[1], 10) - 1;
510
520
  totalChunks = parseInt(partMatch[2], 10);
511
- hasMoreChunks = currentChunk < totalChunks - 1;
512
- // 调试输出,查看解析后的分段信息 (Debug output, check parsed chunk information)
513
- if (debug) {
514
- log('client.parsedChunkInfo', debug, {
515
- chunkId,
516
- currentChunk,
517
- totalChunks,
518
- hasMoreChunks
519
- }, COMPONENTS.CLIENT);
520
- }
521
+ hasMoreChunks = isLastChunk ? false : currentChunk < totalChunks - 1;
522
+ }
523
+ // 如果有启动游标信息,这表明还有更多分块 (If start cursor info exists, indicates more chunks)
524
+ else if (startCursorMatch) {
525
+ hasMoreChunks = !isLastChunk && true;
521
526
  }
527
+ // 使用解析出的分段信息 (Use parsed chunk information)
528
+ result.isChunked = hasMoreChunks;
529
+ result.hasMoreChunks = hasMoreChunks;
530
+ result.chunkId = chunkId;
531
+ result.currentChunk = currentChunk;
532
+ result.totalChunks = totalChunks;
533
+ // 保存分段信息到临时文件 (Save chunk information to temporary file)
534
+ _saveChunkInfo(result, params, debug);
522
535
  }
523
536
  }
524
- // 输出第一段内容 (Output first chunk)
537
+ // 调试输出,查看解析后的分段信息 (Debug output, check parsed chunk information)
538
+ if (debug) {
539
+ log('client.chunkInfoParsed', debug, {
540
+ isChunked: result.isChunked,
541
+ hasMoreChunks,
542
+ isLastChunk,
543
+ chunkId,
544
+ currentChunk,
545
+ totalChunks,
546
+ fetchedBytes: result.fetchedBytes,
547
+ totalBytes: result.totalBytes,
548
+ remainingBytes: result.remainingBytes
549
+ }, COMPONENTS.CLIENT);
550
+ }
551
+ // 输出结果 (Output result)
525
552
  console.log(JSON.stringify(result, null, 2));
526
553
  // 如果启用了获取所有分段内容,并且结果包含分段信息 (If all chunks mode is enabled and result contains chunk information)
527
- if (allChunksFlag && hasMoreChunks && chunkId) {
554
+ if (allChunksFlag && hasMoreChunks && chunkId && !isLastChunk) {
528
555
  try {
529
556
  // 确保totalChunks的计算正确 (Ensure totalChunks calculation is correct)
530
557
  if (result.totalBytes && result.fetchedBytes) {
@@ -640,14 +667,14 @@ async function main() {
640
667
  log('client.completeChunksFetched', true, { total: totalChunks }, COMPONENTS.CLIENT);
641
668
  }
642
669
  }
643
- catch (error) {
670
+ catch (_error) {
644
671
  // 处理获取分块内容过程中的错误 (Handle errors during chunk fetching)
645
- log('client.fetchingChunksError', true, { error: String(error) }, COMPONENTS.CLIENT);
646
- log('client.fetchingChunksErrorMessage', true, { error: String(error) }, COMPONENTS.CLIENT);
672
+ log('client.fetchingChunksError', true, { error: String(_error) }, COMPONENTS.CLIENT);
673
+ log('client.fetchingChunksErrorMessage', true, { error: String(_error) }, COMPONENTS.CLIENT);
647
674
  }
648
675
  }
649
676
  // 如果结果包含分段信息,但没有启用获取所有分段内容 (If result contains chunk information but all chunks mode is not enabled)
650
- else if (hasMoreChunks && chunkId) {
677
+ else if (hasMoreChunks && chunkId && !isLastChunk) {
651
678
  log('client.hasMoreChunks', debug, {
652
679
  current: currentChunk,
653
680
  total: totalChunks,
@@ -672,16 +699,16 @@ async function main() {
672
699
  }
673
700
  }
674
701
  }
675
- catch (error) {
702
+ catch (_error) {
676
703
  // 处理错误 (Handle error)
677
- log('client.fatalError', true, { error: String(error) }, COMPONENTS.CLIENT);
704
+ log('client.fatalError', true, { error: String(_error) }, COMPONENTS.CLIENT);
678
705
  // 返回标准错误结构体 (Return standard error structure)
679
706
  const errorResult = {
680
707
  isError: true,
681
708
  content: [
682
709
  {
683
710
  type: "text",
684
- text: `Fatal error: ${String(error)}`
711
+ text: `Fatal error: ${String(_error)}`
685
712
  }
686
713
  ]
687
714
  };
package/dist/src/index.js CHANGED
File without changes