@lmcc-dev/mult-fetch-mcp-server 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/README.zh.md +16 -8
- package/dist/i18n-test-report.json +8 -0
- package/dist/i18n-unused-keys-report.json +8 -0
- package/dist/src/client.js +50 -35
- package/dist/src/index.js +0 -0
- package/dist/src/lib/BrowserFetcher.js +12 -12
- package/dist/src/lib/NodeFetcher.js +22 -22
- package/dist/src/lib/fetch.js +40 -0
- package/dist/src/lib/fetchers/FetcherFactory.js +43 -0
- package/dist/src/lib/fetchers/browser/BrowserFetcher.js +381 -0
- package/dist/src/lib/fetchers/browser/BrowserInstance.js +180 -0
- package/dist/src/lib/fetchers/browser/PageOperations.js +168 -0
- package/dist/src/lib/fetchers/common/types.js +41 -0
- package/dist/src/lib/fetchers/common/utils.js +141 -0
- package/dist/src/lib/fetchers/index.js +62 -0
- package/dist/src/lib/fetchers/node/HttpClient.js +239 -0
- package/dist/src/lib/fetchers/node/NodeFetcher.js +180 -0
- package/dist/src/lib/i18n/index.js +105 -17
- package/dist/src/lib/i18n/keys/base.js +41 -0
- package/dist/src/lib/i18n/keys/browser.js +91 -0
- package/dist/src/lib/i18n/keys/client.js +68 -0
- package/dist/src/lib/i18n/keys/errors.js +49 -0
- package/dist/src/lib/i18n/keys/factory.js +46 -0
- package/dist/src/lib/i18n/keys/fetcher.js +87 -0
- package/dist/src/lib/i18n/keys/index.js +16 -0
- package/dist/src/lib/i18n/keys/node.js +69 -0
- package/dist/src/lib/i18n/keys/prompts.js +79 -0
- package/dist/src/lib/i18n/keys/resources.js +53 -0
- package/dist/src/lib/i18n/keys/server.js +67 -0
- package/dist/src/lib/i18n/keys/tools.js +39 -0
- package/dist/src/lib/i18n/keys.js +488 -119
- package/dist/src/lib/i18n/locales/en/browser.js +67 -57
- package/dist/src/lib/i18n/locales/en/client.js +57 -33
- package/dist/src/lib/i18n/locales/en/errors.js +36 -12
- package/dist/src/lib/i18n/locales/en/factory.js +38 -0
- package/dist/src/lib/i18n/locales/en/fetcher.js +8 -2
- package/dist/src/lib/i18n/locales/en/index.js +10 -8
- package/dist/src/lib/i18n/locales/en/node.js +57 -35
- package/dist/src/lib/i18n/locales/en/prompts.js +42 -65
- package/dist/src/lib/i18n/locales/en/resources.js +52 -55
- package/dist/src/lib/i18n/locales/en/server.js +49 -24
- package/dist/src/lib/i18n/locales/en/tools.js +27 -19
- package/dist/src/lib/i18n/locales/zh/browser.js +71 -62
- package/dist/src/lib/i18n/locales/zh/client.js +57 -32
- package/dist/src/lib/i18n/locales/zh/errors.js +36 -12
- package/dist/src/lib/i18n/locales/zh/factory.js +38 -0
- package/dist/src/lib/i18n/locales/zh/fetcher.js +36 -30
- package/dist/src/lib/i18n/locales/zh/index.js +10 -8
- package/dist/src/lib/i18n/locales/zh/node.js +61 -36
- package/dist/src/lib/i18n/locales/zh/prompts.js +35 -57
- package/dist/src/lib/i18n/locales/zh/resources.js +52 -55
- package/dist/src/lib/i18n/locales/zh/server.js +51 -25
- package/dist/src/lib/i18n/locales/zh/tools.js +27 -19
- package/dist/src/lib/i18n/logger.js +90 -19
- package/dist/src/lib/logger.js +135 -14
- package/dist/src/lib/server/browser.js +2 -2
- package/dist/src/lib/server/fetcher.js +13 -34
- package/dist/src/lib/server/index.js +2 -2
- package/dist/src/lib/server/prompts.js +1 -1
- package/dist/src/lib/server/resources.js +1 -1
- package/dist/src/lib/server/tools.js +46 -10
- package/dist/src/lib/utils/ErrorHandler.js +125 -0
- package/dist/src/mcp-server.js +0 -0
- package/dist/src/test-i18n.js +139 -0
- package/dist/tests/BrowserFetcher.test.js +0 -0
- package/dist/tests/NodeFetcher.test.js +0 -0
- package/dist/tests/client.test.js +42 -81
- package/dist/tests/fetch.test.js +70 -0
- package/dist/tests/fetchers/Fetcher.test.js +108 -0
- package/dist/tests/fetchers/browser/BrowserFetcher.test.js +195 -0
- package/dist/tests/fetchers/browser/BrowserInstance.test.js +163 -0
- package/dist/tests/fetchers/node/NodeFetcher.test.js +211 -0
- package/dist/tests/i18n-missing-keys.js +364 -0
- package/dist/tests/i18n-remove-unused-keys.js +236 -0
- package/dist/tests/i18n-test-report.json +2004 -0
- package/dist/tests/i18n-unused-keys.js +425 -0
- package/dist/tests/i18n.test.js +6 -7
- package/dist/tests/logger.test.js +14 -13
- package/dist/tests/mcp-server.test.js +7 -7
- package/dist/tests/server/browser.test.js +25 -21
- package/dist/tests/server/fetcher.test.js +58 -101
- package/dist/tests/server/index.test.js +30 -30
- package/dist/tests/server/prompts.test.js +36 -33
- package/dist/tests/server/resources.test.js +120 -84
- package/dist/tests/server/tools.test.js +7 -6
- package/dist/tests/setup.js +36 -0
- package/dist/tests/src/lib/i18n/index.js +108 -0
- package/dist/tests/src/lib/i18n/keys/base.js +47 -0
- package/dist/tests/src/lib/i18n/keys/browser.js +93 -0
- package/dist/tests/src/lib/i18n/keys/client.js +70 -0
- package/dist/tests/src/lib/i18n/keys/errors.js +34 -0
- package/dist/tests/src/lib/i18n/keys/fetcher.js +84 -0
- package/dist/tests/src/lib/i18n/keys/index.js +31 -0
- package/dist/tests/src/lib/i18n/keys/node.js +56 -0
- package/dist/tests/src/lib/i18n/keys/prompts.js +82 -0
- package/dist/tests/src/lib/i18n/keys/resources.js +50 -0
- package/dist/tests/src/lib/i18n/keys/server.js +64 -0
- package/dist/tests/src/lib/i18n/keys/tools.js +34 -0
- package/dist/tests/src/lib/i18n/locales/en/browser.js +88 -0
- package/dist/tests/src/lib/i18n/locales/en/client.js +66 -0
- package/dist/tests/src/lib/i18n/locales/en/errors.js +28 -0
- package/dist/tests/src/lib/i18n/locales/en/fetcher.js +71 -0
- package/dist/tests/src/lib/i18n/locales/en/index.js +29 -0
- package/dist/tests/src/lib/i18n/locales/en/node.js +51 -0
- package/dist/tests/src/lib/i18n/locales/en/prompts.js +52 -0
- package/dist/tests/src/lib/i18n/locales/en/resources.js +50 -0
- package/dist/tests/src/lib/i18n/locales/en/server.js +56 -0
- package/dist/tests/src/lib/i18n/locales/en/tools.js +28 -0
- package/dist/tests/src/lib/i18n/locales/zh/browser.js +87 -0
- package/dist/tests/src/lib/i18n/locales/zh/client.js +66 -0
- package/dist/tests/src/lib/i18n/locales/zh/errors.js +28 -0
- package/dist/tests/src/lib/i18n/locales/zh/fetcher.js +71 -0
- package/dist/tests/src/lib/i18n/locales/zh/index.js +29 -0
- package/dist/tests/src/lib/i18n/locales/zh/node.js +51 -0
- package/dist/tests/src/lib/i18n/locales/zh/prompts.js +53 -0
- package/dist/tests/src/lib/i18n/locales/zh/resources.js +50 -0
- package/dist/tests/src/lib/i18n/locales/zh/server.js +57 -0
- package/dist/tests/src/lib/i18n/locales/zh/tools.js +28 -0
- package/dist/tests/src/lib/i18n/logger.js +114 -0
- package/dist/tests/src/lib/logger.js +181 -0
- package/dist/tests/test-direct-client.js +0 -0
- package/dist/tests/test-i18n.js +530 -0
- package/dist/tests/test-mcp-methods.js +1 -1
- package/dist/tests/test-mcp.js +0 -0
- package/dist/tests/test-mini4k.js +0 -0
- package/dist/tests/tests/test-i18n.js +588 -0
- package/dist/tests/types.test.js +1 -1
- package/dist/tests/utils/ErrorHandler.test.js +144 -0
- package/dist/vitest.config.js +29 -0
- package/package.json +31 -27
- package/dist/src/cli-language.js +0 -78
- package/dist/src/client.d.ts +0 -6
- package/dist/src/client.js.map +0 -1
- package/dist/src/index.d.ts +0 -15
- package/dist/src/index.js.map +0 -1
- package/dist/src/lib/BrowserFetcher.d.ts +0 -175
- package/dist/src/lib/BrowserFetcher.js.map +0 -1
- package/dist/src/lib/NodeFetcher.d.ts +0 -92
- package/dist/src/lib/NodeFetcher.js.map +0 -1
- package/dist/src/lib/example.js +0 -93
- package/dist/src/lib/i18n/index.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/browser.d.ts +0 -61
- package/dist/src/lib/i18n/locales/en/browser.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/client.d.ts +0 -39
- package/dist/src/lib/i18n/locales/en/client.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/errors.d.ts +0 -17
- package/dist/src/lib/i18n/locales/en/errors.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/fetcher.d.ts +0 -66
- package/dist/src/lib/i18n/locales/en/fetcher.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/index.d.ts +0 -344
- package/dist/src/lib/i18n/locales/en/index.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/node.d.ts +0 -41
- package/dist/src/lib/i18n/locales/en/node.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/prompts.d.ts +0 -65
- package/dist/src/lib/i18n/locales/en/prompts.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/resources.d.ts +0 -41
- package/dist/src/lib/i18n/locales/en/resources.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/server.d.ts +0 -30
- package/dist/src/lib/i18n/locales/en/server.js.map +0 -1
- package/dist/src/lib/i18n/locales/en/tools.d.ts +0 -24
- package/dist/src/lib/i18n/locales/en/tools.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/browser.d.ts +0 -61
- package/dist/src/lib/i18n/locales/zh/browser.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/client.d.ts +0 -38
- package/dist/src/lib/i18n/locales/zh/client.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/errors.d.ts +0 -17
- package/dist/src/lib/i18n/locales/zh/errors.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/fetcher.d.ts +0 -66
- package/dist/src/lib/i18n/locales/zh/fetcher.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/index.d.ts +0 -343
- package/dist/src/lib/i18n/locales/zh/index.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/node.d.ts +0 -41
- package/dist/src/lib/i18n/locales/zh/node.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/prompts.d.ts +0 -65
- package/dist/src/lib/i18n/locales/zh/prompts.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/resources.d.ts +0 -41
- package/dist/src/lib/i18n/locales/zh/resources.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/server.d.ts +0 -30
- package/dist/src/lib/i18n/locales/zh/server.js.map +0 -1
- package/dist/src/lib/i18n/locales/zh/tools.d.ts +0 -24
- package/dist/src/lib/i18n/locales/zh/tools.js.map +0 -1
- package/dist/src/lib/i18n/logger.d.ts +0 -38
- package/dist/src/lib/i18n/logger.js.map +0 -1
- package/dist/src/lib/logger.d.ts +0 -32
- package/dist/src/lib/logger.js.map +0 -1
- package/dist/src/lib/server/browser.d.ts +0 -34
- package/dist/src/lib/server/browser.js.map +0 -1
- package/dist/src/lib/server/fetcher.d.ts +0 -20
- package/dist/src/lib/server/fetcher.js.map +0 -1
- package/dist/src/lib/server/index.d.ts +0 -10
- package/dist/src/lib/server/index.js.map +0 -1
- package/dist/src/lib/server/logger.js +0 -23
- package/dist/src/lib/server/prompts.d.ts +0 -11
- package/dist/src/lib/server/prompts.js.map +0 -1
- package/dist/src/lib/server/resources.d.ts +0 -11
- package/dist/src/lib/server/resources.js.map +0 -1
- package/dist/src/lib/server/tools.d.ts +0 -12
- package/dist/src/lib/server/tools.js.map +0 -1
- package/dist/src/lib/server/types.d.ts +0 -71
- package/dist/src/lib/server/types.js.map +0 -1
- package/dist/src/lib/types.d.ts +0 -119
- package/dist/src/lib/types.js.map +0 -1
- package/dist/src/mcp-server.d.ts +0 -6
- package/dist/src/mcp-server.js.map +0 -1
- package/dist/tests/jest.setup.d.ts +0 -6
- package/dist/tests/jest.setup.js +0 -30
- package/dist/tests/jest.setup.js.map +0 -1
- package/dist/tests/test-direct-client.d.ts +0 -6
- package/dist/tests/test-direct-client.js.map +0 -1
- package/dist/tests/test-mcp-methods.d.ts +0 -6
- package/dist/tests/test-mcp-methods.js.map +0 -1
- package/dist/tests/test-mcp.d.ts +0 -6
- package/dist/tests/test-mcp.js.map +0 -1
- package/dist/tests/test-mini4k.d.ts +0 -6
- package/dist/tests/test-mini4k.js.map +0 -1
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
[](https://opensource.org/licenses/MIT)
|
|
5
5
|
[](https://nodejs.org/)
|
|
6
6
|
[](https://www.typescriptlang.org/)
|
|
7
|
-
[](https://github.com/modelcontextprotocol/typescript-sdk)
|
|
8
8
|
[](https://github.com/lmcc-dev/mult-fetch-mcp-server/stargazers)
|
|
9
9
|
[](https://github.com/lmcc-dev/mult-fetch-mcp-server/network/members)
|
|
10
10
|
[](https://github.com/lmcc-dev/mult-fetch-mcp-server/issues)
|
|
@@ -91,16 +91,16 @@ npx -y @smithery/cli install @lmcc-dev/mult-fetch-mcp-server --client claude
|
|
|
91
91
|
### Local Installation
|
|
92
92
|
|
|
93
93
|
```bash
|
|
94
|
-
|
|
94
|
+
pnpm install
|
|
95
95
|
```
|
|
96
96
|
|
|
97
97
|
### Global Installation
|
|
98
98
|
|
|
99
99
|
```bash
|
|
100
|
-
|
|
100
|
+
pnpm add -g @lmcc-dev/mult-fetch-mcp-server
|
|
101
101
|
```
|
|
102
102
|
|
|
103
|
-
Or
|
|
103
|
+
Or run directly with npx (no installation required):
|
|
104
104
|
|
|
105
105
|
```bash
|
|
106
106
|
npx @lmcc-dev/mult-fetch-mcp-server
|
|
@@ -175,13 +175,13 @@ After configuration, restart Claude desktop, and you can use the following tools
|
|
|
175
175
|
## Build
|
|
176
176
|
|
|
177
177
|
```bash
|
|
178
|
-
|
|
178
|
+
pnpm run build
|
|
179
179
|
```
|
|
180
180
|
|
|
181
181
|
## Run Server
|
|
182
182
|
|
|
183
183
|
```bash
|
|
184
|
-
|
|
184
|
+
pnpm run server
|
|
185
185
|
# or
|
|
186
186
|
node dist/index.js
|
|
187
187
|
# if globally installed, you can run directly
|
|
@@ -193,9 +193,9 @@ npx @lmcc-dev/mult-fetch-mcp-server
|
|
|
193
193
|
## Run Client
|
|
194
194
|
|
|
195
195
|
```bash
|
|
196
|
-
|
|
196
|
+
pnpm run client <method> <params_json>
|
|
197
197
|
# example
|
|
198
|
-
|
|
198
|
+
pnpm run client fetch_html '{"url": "https://example.com", "debug": true}'
|
|
199
199
|
```
|
|
200
200
|
|
|
201
201
|
## Run Tests
|
package/README.zh.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
[](https://opensource.org/licenses/MIT)
|
|
5
5
|
[](https://nodejs.org/)
|
|
6
6
|
[](https://www.typescriptlang.org/)
|
|
7
|
-
[](https://github.com/modelcontextprotocol/typescript-sdk)
|
|
8
8
|
[](https://github.com/lmcc-dev/mult-fetch-mcp-server/stargazers)
|
|
9
9
|
[](https://github.com/lmcc-dev/mult-fetch-mcp-server/network/members)
|
|
10
10
|
[](https://github.com/lmcc-dev/mult-fetch-mcp-server/issues)
|
|
@@ -77,22 +77,30 @@ fetch-mcp/
|
|
|
77
77
|
|
|
78
78
|
## 安装
|
|
79
79
|
|
|
80
|
+
### 通过 Smithery 安装
|
|
81
|
+
|
|
82
|
+
要通过 [Smithery](https://smithery.ai/server/@lmcc-dev/mult-fetch-mcp-server) 自动为 Claude 桌面版安装 Mult Fetch MCP Server:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
npx -y @smithery/cli install @lmcc-dev/mult-fetch-mcp-server --client claude
|
|
86
|
+
```
|
|
87
|
+
|
|
80
88
|
### 本地安装
|
|
81
89
|
|
|
82
90
|
```bash
|
|
83
|
-
|
|
91
|
+
pnpm install
|
|
84
92
|
```
|
|
85
93
|
|
|
86
94
|
### 全局安装
|
|
87
95
|
|
|
88
96
|
```bash
|
|
89
|
-
|
|
97
|
+
pnpm add -g @lmcc-dev/mult-fetch-mcp-server
|
|
90
98
|
```
|
|
91
99
|
|
|
92
100
|
或者直接使用 npx 运行(无需安装):
|
|
93
101
|
|
|
94
102
|
```bash
|
|
95
|
-
npx mult-fetch-mcp-server
|
|
103
|
+
npx @lmcc-dev/mult-fetch-mcp-server
|
|
96
104
|
```
|
|
97
105
|
|
|
98
106
|
## 与 Claude 集成
|
|
@@ -164,13 +172,13 @@ npx mult-fetch-mcp-server
|
|
|
164
172
|
## 构建
|
|
165
173
|
|
|
166
174
|
```bash
|
|
167
|
-
|
|
175
|
+
pnpm run build
|
|
168
176
|
```
|
|
169
177
|
|
|
170
178
|
## 运行服务器
|
|
171
179
|
|
|
172
180
|
```bash
|
|
173
|
-
|
|
181
|
+
pnpm run server
|
|
174
182
|
# 或者
|
|
175
183
|
node dist/index.js
|
|
176
184
|
# 如果全局安装,可以直接运行
|
|
@@ -182,9 +190,9 @@ npx mult-fetch-mcp-server
|
|
|
182
190
|
## 运行客户端
|
|
183
191
|
|
|
184
192
|
```bash
|
|
185
|
-
|
|
193
|
+
pnpm run client <method> <params_json>
|
|
186
194
|
# 例如
|
|
187
|
-
|
|
195
|
+
pnpm run client fetch_html '{"url": "https://example.com", "debug": true}'
|
|
188
196
|
```
|
|
189
197
|
|
|
190
198
|
## 运行测试
|
package/dist/src/client.js
CHANGED
|
@@ -28,7 +28,7 @@ function responseRequiresBrowser(response, debug = false) {
|
|
|
28
28
|
if (response.isError) {
|
|
29
29
|
const errorText = response.content[0].text;
|
|
30
30
|
if (debug) {
|
|
31
|
-
log('
|
|
31
|
+
log('client.fetchError', debug, { url: '', error: errorText }, COMPONENTS.CLIENT);
|
|
32
32
|
// 尝试从错误信息中提取HTTP状态码 (Try to extract HTTP status code from error message)
|
|
33
33
|
const statusCodeMatch = errorText.match(/(\b[45]\d\d\b)/);
|
|
34
34
|
if (statusCodeMatch) {
|
|
@@ -104,67 +104,60 @@ async function smartFetch(params) {
|
|
|
104
104
|
// 在调试模式下打印响应结果的摘要 (Print response result summary in debug mode)
|
|
105
105
|
if (debug) {
|
|
106
106
|
if (result.isError) {
|
|
107
|
-
log('client.fetchFailed', debug, { error: result.content[0].text
|
|
107
|
+
log('client.fetchFailed', debug, { url, error: result.content[0].text }, COMPONENTS.CLIENT);
|
|
108
108
|
}
|
|
109
109
|
else {
|
|
110
|
-
|
|
111
|
-
log('client.fetchSuccess', debug, { length: contentLength }, COMPONENTS.CLIENT);
|
|
110
|
+
log('client.fetchSuccess', debug, { url }, COMPONENTS.CLIENT);
|
|
112
111
|
}
|
|
113
112
|
}
|
|
114
|
-
//
|
|
115
|
-
if (
|
|
113
|
+
// 检查是否需要切换到浏览器模式 (Check if need to switch to browser mode)
|
|
114
|
+
if (result.isError && responseRequiresBrowser(result, debug) && !params.useBrowser) {
|
|
116
115
|
log('client.browserModeNeeded', debug, { url }, COMPONENTS.CLIENT);
|
|
117
|
-
// 切换到浏览器模式重试 (Switch to browser mode and retry)
|
|
118
|
-
params.useBrowser = true;
|
|
119
|
-
// 设置关闭浏览器选项,确保资源被释放 (Set close browser option to ensure resources are released)
|
|
120
|
-
params.closeBrowser = true;
|
|
121
116
|
log('client.retryingWithBrowser', debug, { url }, COMPONENTS.CLIENT);
|
|
117
|
+
// 切换到浏览器模式重试 (Switch to browser mode and retry)
|
|
122
118
|
const browserResult = await client.callTool({
|
|
123
119
|
name: method,
|
|
124
|
-
arguments:
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
if (debug) {
|
|
128
|
-
if (browserResult.isError) {
|
|
129
|
-
log('client.browserModeFetchFailed', debug, { error: browserResult.content[0].text.substring(0, 150) + (browserResult.content[0].text.length > 150 ? '...' : '') }, COMPONENTS.CLIENT);
|
|
130
|
-
}
|
|
131
|
-
else {
|
|
132
|
-
const contentLength = browserResult.content[0].text.length;
|
|
133
|
-
log('client.browserModeFetchSuccess', debug, { length: contentLength }, COMPONENTS.CLIENT);
|
|
120
|
+
arguments: {
|
|
121
|
+
...params,
|
|
122
|
+
useBrowser: true
|
|
134
123
|
}
|
|
124
|
+
});
|
|
125
|
+
if (browserResult.isError) {
|
|
126
|
+
log('client.browserModeFetchFailed', debug, { url, error: browserResult.content[0].text }, COMPONENTS.CLIENT);
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
log('client.browserModeFetchSuccess', debug, { url }, COMPONENTS.CLIENT);
|
|
135
130
|
}
|
|
136
131
|
return browserResult;
|
|
137
132
|
}
|
|
133
|
+
if (result.isError) {
|
|
134
|
+
log('client.fetchFailed', debug, { url, error: result.content[0].text }, COMPONENTS.CLIENT);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
log('client.fetchSuccess', debug, { url }, COMPONENTS.CLIENT);
|
|
138
|
+
}
|
|
138
139
|
return result;
|
|
139
140
|
}
|
|
140
141
|
else {
|
|
141
|
-
//
|
|
142
|
+
// 直接使用指定的模式 (Directly use the specified mode)
|
|
142
143
|
log('client.usingMode', debug, { mode: params.useBrowser ? 'browser' : 'standard', url }, COMPONENTS.CLIENT);
|
|
143
|
-
// 如果使用浏览器模式,设置关闭浏览器选项
|
|
144
|
-
if (params.useBrowser) {
|
|
145
|
-
params.closeBrowser = true;
|
|
146
|
-
}
|
|
147
144
|
const result = await client.callTool({
|
|
148
145
|
name: method,
|
|
149
146
|
arguments: params
|
|
150
147
|
});
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
}
|
|
156
|
-
else {
|
|
157
|
-
const contentLength = result.content[0].text.length;
|
|
158
|
-
log('client.fetchSuccess', debug, { length: contentLength }, COMPONENTS.CLIENT);
|
|
159
|
-
}
|
|
148
|
+
if (result.isError) {
|
|
149
|
+
log('client.fetchFailed', debug, { url, error: result.content[0].text }, COMPONENTS.CLIENT);
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
log('client.fetchSuccess', debug, { url }, COMPONENTS.CLIENT);
|
|
160
153
|
}
|
|
161
154
|
return result;
|
|
162
155
|
}
|
|
163
156
|
}
|
|
164
157
|
finally {
|
|
165
158
|
// 关闭客户端连接 (Close client connection)
|
|
166
|
-
await client.close();
|
|
167
159
|
log('client.serverClosed', debug, {}, COMPONENTS.CLIENT);
|
|
160
|
+
await client.close();
|
|
168
161
|
}
|
|
169
162
|
}
|
|
170
163
|
/**
|
|
@@ -186,6 +179,17 @@ async function main() {
|
|
|
186
179
|
}
|
|
187
180
|
catch (error) {
|
|
188
181
|
log('client.invalidJson', true, {}, COMPONENTS.CLIENT);
|
|
182
|
+
// 返回标准错误结构体 (Return standard error structure)
|
|
183
|
+
const errorResult = {
|
|
184
|
+
isError: true,
|
|
185
|
+
content: [
|
|
186
|
+
{
|
|
187
|
+
type: "text",
|
|
188
|
+
text: `Invalid JSON parameter: ${paramsJson}`
|
|
189
|
+
}
|
|
190
|
+
]
|
|
191
|
+
};
|
|
192
|
+
process.stdout.write(JSON.stringify(errorResult, null, 2));
|
|
189
193
|
process.exit(1);
|
|
190
194
|
}
|
|
191
195
|
const debug = params.debug === true;
|
|
@@ -270,6 +274,17 @@ async function main() {
|
|
|
270
274
|
}
|
|
271
275
|
catch (error) {
|
|
272
276
|
log('client.requestFailed', debug, { error: error.message }, COMPONENTS.CLIENT);
|
|
277
|
+
// 即使在出错的情况下也输出标准结构体 (Output standard structure even in case of error)
|
|
278
|
+
const errorResult = {
|
|
279
|
+
isError: true,
|
|
280
|
+
content: [
|
|
281
|
+
{
|
|
282
|
+
type: "text",
|
|
283
|
+
text: `Client error: ${error.message}`
|
|
284
|
+
}
|
|
285
|
+
]
|
|
286
|
+
};
|
|
287
|
+
process.stdout.write(JSON.stringify(errorResult, null, 2));
|
|
273
288
|
process.exit(1);
|
|
274
289
|
}
|
|
275
290
|
}
|
package/dist/src/index.js
CHANGED
|
File without changes
|
|
@@ -88,7 +88,7 @@ export class BrowserFetcher {
|
|
|
88
88
|
const heapTotal = Math.round(memoryUsage.heapTotal / 1024 / 1024);
|
|
89
89
|
const rss = Math.round(memoryUsage.rss / 1024 / 1024);
|
|
90
90
|
// 记录内存使用情况
|
|
91
|
-
log('browser.memoryUsage', debug, { heapUsed
|
|
91
|
+
log('browser.memoryUsage', debug, { usage: heapUsed }, COMPONENTS.BROWSER_FETCH);
|
|
92
92
|
// 如果内存使用过高,记录警告
|
|
93
93
|
if (heapUsed > 500 || rss > 1000) {
|
|
94
94
|
log('browser.memoryTooHigh', debug, { heapUsed, heapTotal, rss }, COMPONENTS.BROWSER_FETCH);
|
|
@@ -473,7 +473,7 @@ export class BrowserFetcher {
|
|
|
473
473
|
const domain = this.getDomain(url);
|
|
474
474
|
const storedCookies = this.getCookies(domain);
|
|
475
475
|
if (storedCookies && saveCookies) {
|
|
476
|
-
log('browser.usingStoredCookies', debug, { domain
|
|
476
|
+
log('browser.usingStoredCookies', debug, { domain }, COMPONENTS.BROWSER_FETCH);
|
|
477
477
|
await page.setExtraHTTPHeaders({
|
|
478
478
|
'Cookie': storedCookies
|
|
479
479
|
});
|
|
@@ -483,7 +483,7 @@ export class BrowserFetcher {
|
|
|
483
483
|
log('browser.usingProxy', debug, { proxy: effectiveProxy }, COMPONENTS.BROWSER_FETCH);
|
|
484
484
|
}
|
|
485
485
|
// 导航到URL
|
|
486
|
-
log('browser.navigating', debug, {
|
|
486
|
+
log('browser.navigating', debug, { url }, COMPONENTS.BROWSER_FETCH);
|
|
487
487
|
await page.goto(url, {
|
|
488
488
|
waitUntil: 'networkidle2',
|
|
489
489
|
timeout: timeout
|
|
@@ -491,27 +491,27 @@ export class BrowserFetcher {
|
|
|
491
491
|
// 尝试处理Cloudflare保护
|
|
492
492
|
const bypassedCloudflare = await this.handleCloudflareProtection(page, url, debug);
|
|
493
493
|
if (!bypassedCloudflare) {
|
|
494
|
-
log('browser.continuingWithoutBypass', debug, {
|
|
494
|
+
log('browser.continuingWithoutBypass', debug, {}, COMPONENTS.BROWSER_FETCH);
|
|
495
495
|
}
|
|
496
496
|
// 等待选择器
|
|
497
|
-
log('browser.waitingForSelector', debug, { selector: waitForSelector
|
|
497
|
+
log('browser.waitingForSelector', debug, { selector: waitForSelector }, COMPONENTS.BROWSER_FETCH);
|
|
498
498
|
await page.waitForSelector(waitForSelector, {
|
|
499
499
|
timeout: timeout
|
|
500
500
|
}).catch(() => { });
|
|
501
501
|
// 等待额外时间
|
|
502
|
-
log('browser.waitingForTimeout', debug, { timeout: waitForTimeout
|
|
502
|
+
log('browser.waitingForTimeout', debug, { timeout: waitForTimeout }, COMPONENTS.BROWSER_FETCH);
|
|
503
503
|
await new Promise(resolve => setTimeout(resolve, waitForTimeout));
|
|
504
504
|
// 如果需要滚动到底部
|
|
505
505
|
if (scrollToBottom) {
|
|
506
|
-
log('browser.scrolling', debug, {
|
|
506
|
+
log('browser.scrolling', debug, {}, COMPONENTS.BROWSER_FETCH);
|
|
507
507
|
await this.autoScroll(page, debug);
|
|
508
508
|
}
|
|
509
509
|
// 获取页面内容
|
|
510
|
-
log('browser.gettingContent', debug, {
|
|
510
|
+
log('browser.gettingContent', debug, {}, COMPONENTS.BROWSER_FETCH);
|
|
511
511
|
const content = await page.content();
|
|
512
512
|
// 如果需要保存cookies
|
|
513
513
|
if (saveCookies) {
|
|
514
|
-
log('browser.savingCookies', debug, {
|
|
514
|
+
log('browser.savingCookies', debug, { domain }, COMPONENTS.BROWSER_FETCH);
|
|
515
515
|
const cookies = await page.evaluate(() => document.cookie);
|
|
516
516
|
if (cookies) {
|
|
517
517
|
this.saveCookies(url, cookies);
|
|
@@ -519,16 +519,16 @@ export class BrowserFetcher {
|
|
|
519
519
|
}
|
|
520
520
|
// 记录内容长度
|
|
521
521
|
const contentLength = content.length;
|
|
522
|
-
log('browser.contentLength', debug, { length: contentLength
|
|
522
|
+
log('browser.contentLength', debug, { length: contentLength }, COMPONENTS.BROWSER_FETCH);
|
|
523
523
|
// 如果内容太长,截断它
|
|
524
524
|
let finalContent = content;
|
|
525
525
|
if (contentLength > 10 * 1024 * 1024) { // 10MB
|
|
526
526
|
finalContent = content.substring(0, 10 * 1024 * 1024);
|
|
527
|
-
log('browser.contentTruncated', debug, {
|
|
527
|
+
log('browser.contentTruncated', debug, {}, COMPONENTS.BROWSER_FETCH);
|
|
528
528
|
}
|
|
529
529
|
// 关闭页面
|
|
530
530
|
await page.close();
|
|
531
|
-
log('browser.pageClosed', debug, {
|
|
531
|
+
log('browser.pageClosed', debug, {}, COMPONENTS.BROWSER_FETCH);
|
|
532
532
|
// 如果需要关闭浏览器
|
|
533
533
|
if (closeBrowser) {
|
|
534
534
|
await this.closeBrowser(debug);
|
|
@@ -55,16 +55,16 @@ export class NodeFetcher {
|
|
|
55
55
|
return undefined;
|
|
56
56
|
}
|
|
57
57
|
// 检查环境变量 (Check environment variables)
|
|
58
|
-
log('
|
|
58
|
+
log('node.checkingProxyEnv', true, {}, COMPONENTS.NODE_FETCH);
|
|
59
59
|
const envVars = ['https_proxy', 'HTTPS_PROXY', 'http_proxy', 'HTTP_PROXY'];
|
|
60
60
|
for (const envVar of envVars) {
|
|
61
|
-
log('
|
|
61
|
+
log('node.envVarValue', true, {
|
|
62
62
|
envVar,
|
|
63
63
|
value: process.env[envVar]
|
|
64
64
|
}, COMPONENTS.NODE_FETCH);
|
|
65
65
|
if (process.env[envVar]) {
|
|
66
66
|
const proxyUrl = process.env[envVar];
|
|
67
|
-
log('
|
|
67
|
+
log('node.foundSystemProxy', true, { proxy: proxyUrl }, COMPONENTS.NODE_FETCH);
|
|
68
68
|
return proxyUrl;
|
|
69
69
|
}
|
|
70
70
|
}
|
|
@@ -72,22 +72,22 @@ export class NodeFetcher {
|
|
|
72
72
|
try {
|
|
73
73
|
const platform = process.platform;
|
|
74
74
|
let proxyUrl;
|
|
75
|
-
log('
|
|
75
|
+
log('node.checkingSystemEnvVars', true, { platform }, COMPONENTS.NODE_FETCH);
|
|
76
76
|
if (platform === 'win32') {
|
|
77
77
|
// Windows系统 - 使用set命令 (Windows - use set command)
|
|
78
78
|
try {
|
|
79
79
|
// 使用set命令获取代理环境变量 (Use set command to get proxy environment variables)
|
|
80
80
|
const setOutput = execSync('set http_proxy & set https_proxy & set HTTP_PROXY & set HTTPS_PROXY').toString();
|
|
81
|
-
log('
|
|
81
|
+
log('node.windowsEnvVars', true, { output: setOutput.trim() }, COMPONENTS.NODE_FETCH);
|
|
82
82
|
// 解析输出找到代理设置 (Parse output to find proxy settings)
|
|
83
83
|
const proxyMatch = setOutput.match(/(?:http_proxy|https_proxy|HTTP_PROXY|HTTPS_PROXY)=(https?:\/\/[^=\r\n]+)/i);
|
|
84
84
|
if (proxyMatch && proxyMatch[1]) {
|
|
85
85
|
proxyUrl = proxyMatch[1].trim();
|
|
86
|
-
log('
|
|
86
|
+
log('node.foundWindowsEnvProxy', true, { proxy: proxyUrl }, COMPONENTS.NODE_FETCH);
|
|
87
87
|
}
|
|
88
88
|
}
|
|
89
89
|
catch (winError) {
|
|
90
|
-
log('
|
|
90
|
+
log('node.errorGettingWindowsEnvVars', true, { error: String(winError) }, COMPONENTS.NODE_FETCH);
|
|
91
91
|
}
|
|
92
92
|
}
|
|
93
93
|
else {
|
|
@@ -95,31 +95,31 @@ export class NodeFetcher {
|
|
|
95
95
|
try {
|
|
96
96
|
// 使用env命令获取所有环境变量 (Use env command to get all environment variables)
|
|
97
97
|
const envOutput = execSync('env').toString();
|
|
98
|
-
log('
|
|
98
|
+
log('node.unixEnvVars', true, { output: envOutput.length > 200 ? envOutput.substring(0, 200) + '...' : envOutput }, COMPONENTS.NODE_FETCH);
|
|
99
99
|
// 解析输出找到代理设置 (Parse output to find proxy settings)
|
|
100
100
|
const proxyMatch = envOutput.match(/(?:http_proxy|https_proxy|HTTP_PROXY|HTTPS_PROXY)=(https?:\/\/[^=\n]+)/i);
|
|
101
101
|
if (proxyMatch && proxyMatch[1]) {
|
|
102
102
|
proxyUrl = proxyMatch[1].trim();
|
|
103
|
-
log('
|
|
103
|
+
log('node.foundUnixEnvProxy', true, { proxy: proxyUrl }, COMPONENTS.NODE_FETCH);
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
catch (unixError) {
|
|
107
|
-
log('
|
|
107
|
+
log('node.errorGettingUnixEnvVars', true, { error: String(unixError) }, COMPONENTS.NODE_FETCH);
|
|
108
108
|
}
|
|
109
109
|
}
|
|
110
110
|
if (proxyUrl) {
|
|
111
111
|
return proxyUrl;
|
|
112
112
|
}
|
|
113
113
|
// 如果没有找到代理,记录日志 (If no proxy is found, log a message)
|
|
114
|
-
log('
|
|
114
|
+
log('node.noSystemProxyFound', true, {}, COMPONENTS.NODE_FETCH);
|
|
115
115
|
}
|
|
116
116
|
catch (error) {
|
|
117
|
-
log('
|
|
117
|
+
log('node.errorGettingSystemEnvVars', true, { error: String(error) }, COMPONENTS.NODE_FETCH);
|
|
118
118
|
}
|
|
119
119
|
// 检查NO_PROXY环境变量 (Check NO_PROXY environment variable)
|
|
120
120
|
const noProxy = process.env.NO_PROXY || process.env.no_proxy;
|
|
121
121
|
if (noProxy) {
|
|
122
|
-
log('
|
|
122
|
+
log('node.foundNoProxy', true, { noProxy }, COMPONENTS.NODE_FETCH);
|
|
123
123
|
}
|
|
124
124
|
return undefined;
|
|
125
125
|
}
|
|
@@ -432,14 +432,14 @@ export class NodeFetcher {
|
|
|
432
432
|
*/
|
|
433
433
|
static async txt(requestPayload) {
|
|
434
434
|
const { debug = false } = requestPayload;
|
|
435
|
-
log('
|
|
435
|
+
log('node.startingTxtFetch', debug, {}, COMPONENTS.NODE_FETCH);
|
|
436
436
|
try {
|
|
437
437
|
// 执行请求 (Execute request)
|
|
438
438
|
const response = await this._fetchWithRedirects(requestPayload);
|
|
439
439
|
// 读取响应文本 (Read response text)
|
|
440
|
-
log('
|
|
440
|
+
log('node.readingText', debug, {}, COMPONENTS.NODE_FETCH);
|
|
441
441
|
const text = await response.text();
|
|
442
|
-
log('
|
|
442
|
+
log('node.textContentLength', debug, { length: text.length }, COMPONENTS.NODE_FETCH);
|
|
443
443
|
// 返回纯文本内容 (Return plain text content)
|
|
444
444
|
return {
|
|
445
445
|
text,
|
|
@@ -460,21 +460,21 @@ export class NodeFetcher {
|
|
|
460
460
|
*/
|
|
461
461
|
static async markdown(requestPayload) {
|
|
462
462
|
const { debug = false } = requestPayload;
|
|
463
|
-
log('
|
|
463
|
+
log('node.startingMarkdownFetch', debug, {}, COMPONENTS.NODE_FETCH);
|
|
464
464
|
try {
|
|
465
465
|
// 执行请求 (Execute request)
|
|
466
466
|
const response = await this._fetchWithRedirects(requestPayload);
|
|
467
467
|
// 读取响应文本 (Read response text)
|
|
468
|
-
log('
|
|
468
|
+
log('node.readingText', debug, {}, COMPONENTS.NODE_FETCH);
|
|
469
469
|
const html = await response.text();
|
|
470
|
-
log('
|
|
470
|
+
log('node.htmlContentLength', debug, { length: html.length }, COMPONENTS.NODE_FETCH);
|
|
471
471
|
// 创建Turndown服务 (Create Turndown service)
|
|
472
|
-
log('
|
|
472
|
+
log('node.creatingTurndown', debug, {}, COMPONENTS.NODE_FETCH);
|
|
473
473
|
const turndownService = new TurndownService();
|
|
474
474
|
// 将HTML转换为Markdown (Convert HTML to Markdown)
|
|
475
|
-
log('
|
|
475
|
+
log('node.convertingToMarkdown', debug, {}, COMPONENTS.NODE_FETCH);
|
|
476
476
|
const markdown = turndownService.turndown(html);
|
|
477
|
-
log('
|
|
477
|
+
log('node.markdownContentLength', debug, { length: markdown.length }, COMPONENTS.NODE_FETCH);
|
|
478
478
|
// 返回Markdown内容 (Return Markdown content)
|
|
479
479
|
return {
|
|
480
480
|
markdown,
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Author: Martin <lmccc.dev@gmail.com>
|
|
3
|
+
* Co-Author: AI Assistant (Claude)
|
|
4
|
+
* Description: This code was collaboratively developed by Martin and AI Assistant.
|
|
5
|
+
*/
|
|
6
|
+
import { Fetcher } from './fetchers/index.js';
|
|
7
|
+
/**
|
|
8
|
+
* 获取HTML内容 (Get HTML content)
|
|
9
|
+
* @param requestPayload 请求参数 (Request parameters)
|
|
10
|
+
* @returns HTML内容 (HTML content)
|
|
11
|
+
*/
|
|
12
|
+
export async function fetchHtml(requestPayload) {
|
|
13
|
+
return await Fetcher.html(requestPayload);
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* 获取JSON内容 (Get JSON content)
|
|
17
|
+
* @param requestPayload 请求参数 (Request parameters)
|
|
18
|
+
* @returns JSON内容 (JSON content)
|
|
19
|
+
*/
|
|
20
|
+
export async function fetchJson(requestPayload) {
|
|
21
|
+
return await Fetcher.json(requestPayload);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* 获取纯文本内容 (Get plain text content)
|
|
25
|
+
* @param requestPayload 请求参数 (Request parameters)
|
|
26
|
+
* @returns 纯文本内容 (Plain text content)
|
|
27
|
+
*/
|
|
28
|
+
export async function fetchTxt(requestPayload) {
|
|
29
|
+
return await Fetcher.txt(requestPayload);
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* 获取Markdown内容 (Get Markdown content)
|
|
33
|
+
* @param requestPayload 请求参数 (Request parameters)
|
|
34
|
+
* @returns Markdown内容 (Markdown content)
|
|
35
|
+
*/
|
|
36
|
+
export async function fetchMarkdown(requestPayload) {
|
|
37
|
+
return await Fetcher.markdown(requestPayload);
|
|
38
|
+
}
|
|
39
|
+
// 导出所有类型和类 (Export all types and classes)
|
|
40
|
+
export * from './fetchers/index.js';
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Author: Martin <lmccc.dev@gmail.com>
|
|
3
|
+
* Co-Author: AI Assistant (Claude)
|
|
4
|
+
* Description: This code was collaboratively developed by Martin and AI Assistant.
|
|
5
|
+
*/
|
|
6
|
+
import { BrowserFetcher } from './browser/BrowserFetcher.js';
|
|
7
|
+
import { NodeFetcher } from './node/NodeFetcher.js';
|
|
8
|
+
import { log, COMPONENTS } from '../logger.js';
|
|
9
|
+
/**
|
|
10
|
+
* 抓取器工厂类 (Fetcher factory class)
|
|
11
|
+
* 根据请求参数创建合适的抓取器实例 (Create appropriate fetcher instance based on request parameters)
|
|
12
|
+
*/
|
|
13
|
+
export class FetcherFactory {
|
|
14
|
+
/**
|
|
15
|
+
* 创建抓取器实例 (Create fetcher instance)
|
|
16
|
+
* @param requestPayload 请求参数 (Request parameters)
|
|
17
|
+
* @returns 抓取器实例 (Fetcher instance)
|
|
18
|
+
*/
|
|
19
|
+
static createFetcher(requestPayload) {
|
|
20
|
+
const { useBrowser = false, useNodeFetch = false, autoDetectMode = true, debug = false } = requestPayload;
|
|
21
|
+
// 如果明确指定使用浏览器模式 (If browser mode is explicitly specified)
|
|
22
|
+
if (useBrowser) {
|
|
23
|
+
log('factory.using_browser_mode', debug, {}, COMPONENTS.FETCHER_FACTORY);
|
|
24
|
+
return new BrowserFetcher();
|
|
25
|
+
}
|
|
26
|
+
// 如果明确指定使用Node模式 (If Node mode is explicitly specified)
|
|
27
|
+
if (useNodeFetch) {
|
|
28
|
+
log('factory.using_node_mode', debug, {}, COMPONENTS.FETCHER_FACTORY);
|
|
29
|
+
return new NodeFetcher();
|
|
30
|
+
}
|
|
31
|
+
// 如果启用自动检测模式 (If auto-detect mode is enabled)
|
|
32
|
+
if (autoDetectMode) {
|
|
33
|
+
// 这里可以添加更复杂的逻辑来决定使用哪种模式
|
|
34
|
+
// 例如,根据URL特征、请求头等判断
|
|
35
|
+
// 目前简单地默认使用Node模式,因为它更轻量级
|
|
36
|
+
log('factory.auto_detecting_mode', debug, { defaultToNode: true }, COMPONENTS.FETCHER_FACTORY);
|
|
37
|
+
return new NodeFetcher();
|
|
38
|
+
}
|
|
39
|
+
// 默认使用Node模式 (Default to Node mode)
|
|
40
|
+
log('factory.defaultingToNodeMode', debug, {}, COMPONENTS.FETCHER_FACTORY);
|
|
41
|
+
return new NodeFetcher();
|
|
42
|
+
}
|
|
43
|
+
}
|