@lmcc-dev/mult-fetch-mcp-server 1.1.0 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +8 -8
  2. package/README.zh.md +16 -8
  3. package/dist/i18n-test-report.json +8 -0
  4. package/dist/i18n-unused-keys-report.json +8 -0
  5. package/dist/src/client.js +70 -37
  6. package/dist/src/index.js +0 -0
  7. package/dist/src/lib/BrowserFetcher.js +12 -12
  8. package/dist/src/lib/NodeFetcher.js +22 -22
  9. package/dist/src/lib/fetch.js +40 -0
  10. package/dist/src/lib/fetchers/FetcherFactory.js +43 -0
  11. package/dist/src/lib/fetchers/browser/BrowserFetcher.js +381 -0
  12. package/dist/src/lib/fetchers/browser/BrowserInstance.js +180 -0
  13. package/dist/src/lib/fetchers/browser/PageOperations.js +168 -0
  14. package/dist/src/lib/fetchers/common/types.js +41 -0
  15. package/dist/src/lib/fetchers/common/utils.js +141 -0
  16. package/dist/src/lib/fetchers/index.js +62 -0
  17. package/dist/src/lib/fetchers/node/HttpClient.js +239 -0
  18. package/dist/src/lib/fetchers/node/NodeFetcher.js +180 -0
  19. package/dist/src/lib/i18n/index.js +105 -17
  20. package/dist/src/lib/i18n/keys/base.js +41 -0
  21. package/dist/src/lib/i18n/keys/browser.js +91 -0
  22. package/dist/src/lib/i18n/keys/client.js +68 -0
  23. package/dist/src/lib/i18n/keys/errors.js +49 -0
  24. package/dist/src/lib/i18n/keys/factory.js +46 -0
  25. package/dist/src/lib/i18n/keys/fetcher.js +87 -0
  26. package/dist/src/lib/i18n/keys/index.js +16 -0
  27. package/dist/src/lib/i18n/keys/node.js +69 -0
  28. package/dist/src/lib/i18n/keys/prompts.js +79 -0
  29. package/dist/src/lib/i18n/keys/resources.js +53 -0
  30. package/dist/src/lib/i18n/keys/server.js +67 -0
  31. package/dist/src/lib/i18n/keys/tools.js +39 -0
  32. package/dist/src/lib/i18n/keys.js +488 -119
  33. package/dist/src/lib/i18n/locales/en/browser.js +67 -57
  34. package/dist/src/lib/i18n/locales/en/client.js +57 -33
  35. package/dist/src/lib/i18n/locales/en/errors.js +36 -12
  36. package/dist/src/lib/i18n/locales/en/factory.js +38 -0
  37. package/dist/src/lib/i18n/locales/en/fetcher.js +8 -2
  38. package/dist/src/lib/i18n/locales/en/index.js +10 -8
  39. package/dist/src/lib/i18n/locales/en/node.js +57 -35
  40. package/dist/src/lib/i18n/locales/en/prompts.js +42 -65
  41. package/dist/src/lib/i18n/locales/en/resources.js +52 -55
  42. package/dist/src/lib/i18n/locales/en/server.js +49 -24
  43. package/dist/src/lib/i18n/locales/en/tools.js +27 -19
  44. package/dist/src/lib/i18n/locales/zh/browser.js +71 -62
  45. package/dist/src/lib/i18n/locales/zh/client.js +57 -32
  46. package/dist/src/lib/i18n/locales/zh/errors.js +36 -12
  47. package/dist/src/lib/i18n/locales/zh/factory.js +38 -0
  48. package/dist/src/lib/i18n/locales/zh/fetcher.js +36 -30
  49. package/dist/src/lib/i18n/locales/zh/index.js +10 -8
  50. package/dist/src/lib/i18n/locales/zh/node.js +61 -36
  51. package/dist/src/lib/i18n/locales/zh/prompts.js +35 -57
  52. package/dist/src/lib/i18n/locales/zh/resources.js +52 -55
  53. package/dist/src/lib/i18n/locales/zh/server.js +51 -25
  54. package/dist/src/lib/i18n/locales/zh/tools.js +27 -19
  55. package/dist/src/lib/i18n/logger.js +90 -19
  56. package/dist/src/lib/logger.js +135 -14
  57. package/dist/src/lib/server/browser.js +42 -10
  58. package/dist/src/lib/server/fetcher.js +87 -37
  59. package/dist/src/lib/server/index.js +2 -2
  60. package/dist/src/lib/server/prompts.js +1 -1
  61. package/dist/src/lib/server/resources.js +1 -1
  62. package/dist/src/lib/server/tools.js +46 -10
  63. package/dist/src/lib/utils/ErrorHandler.js +125 -0
  64. package/dist/src/mcp-server.js +0 -0
  65. package/dist/src/test-i18n.js +139 -0
  66. package/dist/tests/BrowserFetcher.test.js +0 -0
  67. package/dist/tests/NodeFetcher.test.js +0 -0
  68. package/dist/tests/client.test.js +42 -81
  69. package/dist/tests/fetch.test.js +70 -0
  70. package/dist/tests/fetchers/Fetcher.test.js +108 -0
  71. package/dist/tests/fetchers/browser/BrowserFetcher.test.js +195 -0
  72. package/dist/tests/fetchers/browser/BrowserInstance.test.js +163 -0
  73. package/dist/tests/fetchers/node/NodeFetcher.test.js +211 -0
  74. package/dist/tests/i18n-missing-keys.js +364 -0
  75. package/dist/tests/i18n-remove-unused-keys.js +236 -0
  76. package/dist/tests/i18n-test-report.json +2004 -0
  77. package/dist/tests/i18n-unused-keys.js +425 -0
  78. package/dist/tests/i18n.test.js +6 -7
  79. package/dist/tests/logger.test.js +14 -13
  80. package/dist/tests/mcp-server.test.js +7 -7
  81. package/dist/tests/server/browser.test.js +25 -21
  82. package/dist/tests/server/fetcher.test.js +58 -101
  83. package/dist/tests/server/index.test.js +30 -30
  84. package/dist/tests/server/prompts.test.js +36 -33
  85. package/dist/tests/server/resources.test.js +120 -84
  86. package/dist/tests/server/tools.test.js +7 -6
  87. package/dist/tests/setup.js +36 -0
  88. package/dist/tests/src/lib/i18n/index.js +108 -0
  89. package/dist/tests/src/lib/i18n/keys/base.js +47 -0
  90. package/dist/tests/src/lib/i18n/keys/browser.js +93 -0
  91. package/dist/tests/src/lib/i18n/keys/client.js +70 -0
  92. package/dist/tests/src/lib/i18n/keys/errors.js +34 -0
  93. package/dist/tests/src/lib/i18n/keys/fetcher.js +84 -0
  94. package/dist/tests/src/lib/i18n/keys/index.js +31 -0
  95. package/dist/tests/src/lib/i18n/keys/node.js +56 -0
  96. package/dist/tests/src/lib/i18n/keys/prompts.js +82 -0
  97. package/dist/tests/src/lib/i18n/keys/resources.js +50 -0
  98. package/dist/tests/src/lib/i18n/keys/server.js +64 -0
  99. package/dist/tests/src/lib/i18n/keys/tools.js +34 -0
  100. package/dist/tests/src/lib/i18n/locales/en/browser.js +88 -0
  101. package/dist/tests/src/lib/i18n/locales/en/client.js +66 -0
  102. package/dist/tests/src/lib/i18n/locales/en/errors.js +28 -0
  103. package/dist/tests/src/lib/i18n/locales/en/fetcher.js +71 -0
  104. package/dist/tests/src/lib/i18n/locales/en/index.js +29 -0
  105. package/dist/tests/src/lib/i18n/locales/en/node.js +51 -0
  106. package/dist/tests/src/lib/i18n/locales/en/prompts.js +52 -0
  107. package/dist/tests/src/lib/i18n/locales/en/resources.js +50 -0
  108. package/dist/tests/src/lib/i18n/locales/en/server.js +56 -0
  109. package/dist/tests/src/lib/i18n/locales/en/tools.js +28 -0
  110. package/dist/tests/src/lib/i18n/locales/zh/browser.js +87 -0
  111. package/dist/tests/src/lib/i18n/locales/zh/client.js +66 -0
  112. package/dist/tests/src/lib/i18n/locales/zh/errors.js +28 -0
  113. package/dist/tests/src/lib/i18n/locales/zh/fetcher.js +71 -0
  114. package/dist/tests/src/lib/i18n/locales/zh/index.js +29 -0
  115. package/dist/tests/src/lib/i18n/locales/zh/node.js +51 -0
  116. package/dist/tests/src/lib/i18n/locales/zh/prompts.js +53 -0
  117. package/dist/tests/src/lib/i18n/locales/zh/resources.js +50 -0
  118. package/dist/tests/src/lib/i18n/locales/zh/server.js +57 -0
  119. package/dist/tests/src/lib/i18n/locales/zh/tools.js +28 -0
  120. package/dist/tests/src/lib/i18n/logger.js +114 -0
  121. package/dist/tests/src/lib/logger.js +181 -0
  122. package/dist/tests/test-direct-client.js +0 -0
  123. package/dist/tests/test-i18n.js +530 -0
  124. package/dist/tests/test-mcp-methods.js +1 -1
  125. package/dist/tests/test-mcp.js +0 -0
  126. package/dist/tests/test-mini4k.js +0 -0
  127. package/dist/tests/tests/test-i18n.js +588 -0
  128. package/dist/tests/types.test.js +1 -1
  129. package/dist/tests/utils/ErrorHandler.test.js +144 -0
  130. package/dist/vitest.config.js +29 -0
  131. package/package.json +31 -27
  132. package/dist/src/cli-language.js +0 -78
  133. package/dist/src/client.d.ts +0 -6
  134. package/dist/src/client.js.map +0 -1
  135. package/dist/src/index.d.ts +0 -15
  136. package/dist/src/index.js.map +0 -1
  137. package/dist/src/lib/BrowserFetcher.d.ts +0 -175
  138. package/dist/src/lib/BrowserFetcher.js.map +0 -1
  139. package/dist/src/lib/NodeFetcher.d.ts +0 -92
  140. package/dist/src/lib/NodeFetcher.js.map +0 -1
  141. package/dist/src/lib/example.js +0 -93
  142. package/dist/src/lib/i18n/index.js.map +0 -1
  143. package/dist/src/lib/i18n/locales/en/browser.d.ts +0 -61
  144. package/dist/src/lib/i18n/locales/en/browser.js.map +0 -1
  145. package/dist/src/lib/i18n/locales/en/client.d.ts +0 -39
  146. package/dist/src/lib/i18n/locales/en/client.js.map +0 -1
  147. package/dist/src/lib/i18n/locales/en/errors.d.ts +0 -17
  148. package/dist/src/lib/i18n/locales/en/errors.js.map +0 -1
  149. package/dist/src/lib/i18n/locales/en/fetcher.d.ts +0 -66
  150. package/dist/src/lib/i18n/locales/en/fetcher.js.map +0 -1
  151. package/dist/src/lib/i18n/locales/en/index.d.ts +0 -344
  152. package/dist/src/lib/i18n/locales/en/index.js.map +0 -1
  153. package/dist/src/lib/i18n/locales/en/node.d.ts +0 -41
  154. package/dist/src/lib/i18n/locales/en/node.js.map +0 -1
  155. package/dist/src/lib/i18n/locales/en/prompts.d.ts +0 -65
  156. package/dist/src/lib/i18n/locales/en/prompts.js.map +0 -1
  157. package/dist/src/lib/i18n/locales/en/resources.d.ts +0 -41
  158. package/dist/src/lib/i18n/locales/en/resources.js.map +0 -1
  159. package/dist/src/lib/i18n/locales/en/server.d.ts +0 -30
  160. package/dist/src/lib/i18n/locales/en/server.js.map +0 -1
  161. package/dist/src/lib/i18n/locales/en/tools.d.ts +0 -24
  162. package/dist/src/lib/i18n/locales/en/tools.js.map +0 -1
  163. package/dist/src/lib/i18n/locales/zh/browser.d.ts +0 -61
  164. package/dist/src/lib/i18n/locales/zh/browser.js.map +0 -1
  165. package/dist/src/lib/i18n/locales/zh/client.d.ts +0 -38
  166. package/dist/src/lib/i18n/locales/zh/client.js.map +0 -1
  167. package/dist/src/lib/i18n/locales/zh/errors.d.ts +0 -17
  168. package/dist/src/lib/i18n/locales/zh/errors.js.map +0 -1
  169. package/dist/src/lib/i18n/locales/zh/fetcher.d.ts +0 -66
  170. package/dist/src/lib/i18n/locales/zh/fetcher.js.map +0 -1
  171. package/dist/src/lib/i18n/locales/zh/index.d.ts +0 -343
  172. package/dist/src/lib/i18n/locales/zh/index.js.map +0 -1
  173. package/dist/src/lib/i18n/locales/zh/node.d.ts +0 -41
  174. package/dist/src/lib/i18n/locales/zh/node.js.map +0 -1
  175. package/dist/src/lib/i18n/locales/zh/prompts.d.ts +0 -65
  176. package/dist/src/lib/i18n/locales/zh/prompts.js.map +0 -1
  177. package/dist/src/lib/i18n/locales/zh/resources.d.ts +0 -41
  178. package/dist/src/lib/i18n/locales/zh/resources.js.map +0 -1
  179. package/dist/src/lib/i18n/locales/zh/server.d.ts +0 -30
  180. package/dist/src/lib/i18n/locales/zh/server.js.map +0 -1
  181. package/dist/src/lib/i18n/locales/zh/tools.d.ts +0 -24
  182. package/dist/src/lib/i18n/locales/zh/tools.js.map +0 -1
  183. package/dist/src/lib/i18n/logger.d.ts +0 -38
  184. package/dist/src/lib/i18n/logger.js.map +0 -1
  185. package/dist/src/lib/logger.d.ts +0 -32
  186. package/dist/src/lib/logger.js.map +0 -1
  187. package/dist/src/lib/server/browser.d.ts +0 -34
  188. package/dist/src/lib/server/browser.js.map +0 -1
  189. package/dist/src/lib/server/fetcher.d.ts +0 -20
  190. package/dist/src/lib/server/fetcher.js.map +0 -1
  191. package/dist/src/lib/server/index.d.ts +0 -10
  192. package/dist/src/lib/server/index.js.map +0 -1
  193. package/dist/src/lib/server/logger.js +0 -23
  194. package/dist/src/lib/server/prompts.d.ts +0 -11
  195. package/dist/src/lib/server/prompts.js.map +0 -1
  196. package/dist/src/lib/server/resources.d.ts +0 -11
  197. package/dist/src/lib/server/resources.js.map +0 -1
  198. package/dist/src/lib/server/tools.d.ts +0 -12
  199. package/dist/src/lib/server/tools.js.map +0 -1
  200. package/dist/src/lib/server/types.d.ts +0 -71
  201. package/dist/src/lib/server/types.js.map +0 -1
  202. package/dist/src/lib/types.d.ts +0 -119
  203. package/dist/src/lib/types.js.map +0 -1
  204. package/dist/src/mcp-server.d.ts +0 -6
  205. package/dist/src/mcp-server.js.map +0 -1
  206. package/dist/tests/jest.setup.d.ts +0 -6
  207. package/dist/tests/jest.setup.js +0 -30
  208. package/dist/tests/jest.setup.js.map +0 -1
  209. package/dist/tests/test-direct-client.d.ts +0 -6
  210. package/dist/tests/test-direct-client.js.map +0 -1
  211. package/dist/tests/test-mcp-methods.d.ts +0 -6
  212. package/dist/tests/test-mcp-methods.js.map +0 -1
  213. package/dist/tests/test-mcp.d.ts +0 -6
  214. package/dist/tests/test-mcp.js.map +0 -1
  215. package/dist/tests/test-mini4k.d.ts +0 -6
  216. package/dist/tests/test-mini4k.js.map +0 -1
@@ -0,0 +1,168 @@
1
+ /**
2
+ * Author: Martin <lmccc.dev@gmail.com>
3
+ * Co-Author: AI Assistant (Claude)
4
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
5
+ */
6
+ import { log, COMPONENTS } from '../../logger.js';
7
+ import { getDomain } from '../common/utils.js';
8
+ /**
9
+ * Cookie存储类 (Cookie storage class)
10
+ * 管理不同域名的Cookie (Manage cookies for different domains)
11
+ */
12
+ export class CookieManager {
13
+ /**
14
+ * Cookie存储 (Cookie storage)
15
+ * 存储不同域名的Cookie (Store cookies for different domains)
16
+ */
17
+ static cookieStore = {};
18
+ /**
19
+ * 保存Cookie到存储 (Save cookies to storage)
20
+ * @param url 网址 (URL)
21
+ * @param cookies Cookie字符串 (Cookie string)
22
+ */
23
+ static saveCookies(url, cookies) {
24
+ const domain = getDomain(url);
25
+ if (domain) {
26
+ this.cookieStore[domain] = cookies;
27
+ }
28
+ }
29
+ /**
30
+ * 从存储获取Cookie (Get cookies from storage)
31
+ * @param url 网址 (URL)
32
+ * @returns Cookie字符串或undefined (Cookie string or undefined)
33
+ */
34
+ static getCookies(url) {
35
+ const domain = getDomain(url);
36
+ return domain ? this.cookieStore[domain] : undefined;
37
+ }
38
+ }
39
+ /**
40
+ * 页面操作类 (Page operations class)
41
+ * 提供浏览器页面的各种操作方法 (Provides various operation methods for browser pages)
42
+ */
43
+ export class PageOperations {
44
+ /**
45
+ * 自动滚动页面到底部 (Auto scroll page to bottom)
46
+ * @param page 页面对象 (Page object)
47
+ * @param debug 是否启用调试输出 (Whether to enable debug output)
48
+ */
49
+ static async autoScroll(page, debug) {
50
+ log('browser.scrolling', debug, { debug }, COMPONENTS.BROWSER_FETCH);
51
+ try {
52
+ await page.evaluate(async () => {
53
+ await new Promise((resolve) => {
54
+ let totalHeight = 0;
55
+ const distance = 100;
56
+ const timer = setInterval(() => {
57
+ const scrollHeight = document.body.scrollHeight;
58
+ window.scrollBy(0, distance);
59
+ totalHeight += distance;
60
+ if (totalHeight >= scrollHeight) {
61
+ clearInterval(timer);
62
+ resolve();
63
+ }
64
+ }, 100);
65
+ });
66
+ });
67
+ log('browser.scrollCompleted', debug, { debug }, COMPONENTS.BROWSER_FETCH);
68
+ }
69
+ catch (error) {
70
+ // 忽略滚动错误,不影响主要功能
71
+ log('browser.scrollError', debug, { error: String(error) }, COMPONENTS.BROWSER_FETCH);
72
+ }
73
+ }
74
+ /**
75
+ * 检查是否存在Cloudflare保护并尝试绕过 (Check if Cloudflare protection exists and try to bypass)
76
+ * @param page 页面对象 (Page object)
77
+ * @param url 网址 (URL)
78
+ * @param debug 是否启用调试输出 (Whether to enable debug output)
79
+ * @returns 是否成功绕过 (Whether successfully bypassed)
80
+ */
81
+ static async handleCloudflareProtection(page, url, debug) {
82
+ log('browser.checkingCloudflare', debug, { debug }, COMPONENTS.BROWSER_FETCH);
83
+ try {
84
+ // 检查是否存在Cloudflare挑战页面的特征
85
+ const isCloudflarePage = await page.evaluate(() => {
86
+ return document.title.includes('Cloudflare') ||
87
+ document.title.includes('Security Check') ||
88
+ document.body.textContent?.includes('Checking your browser') ||
89
+ document.body.textContent?.includes('Please wait') ||
90
+ document.querySelector('#challenge-form') !== null;
91
+ });
92
+ if (isCloudflarePage) {
93
+ log('browser.cloudflareDetected', debug, { debug }, COMPONENTS.BROWSER_FETCH);
94
+ // 模拟人类行为以通过Cloudflare检查
95
+ log('browser.simulatingHuman', debug, {}, COMPONENTS.BROWSER_FETCH);
96
+ await this.simulateHumanBehavior(page, debug);
97
+ // 等待页面加载完成
98
+ await page.waitForNavigation({
99
+ waitUntil: 'networkidle2',
100
+ timeout: 30000
101
+ }).catch(() => { });
102
+ // 再次检查是否仍在Cloudflare页面
103
+ const stillOnCloudflare = await page.evaluate(() => {
104
+ return document.title.includes('Cloudflare') ||
105
+ document.title.includes('Security Check') ||
106
+ document.body.textContent?.includes('Checking your browser') ||
107
+ document.querySelector('#challenge-form') !== null;
108
+ });
109
+ if (stillOnCloudflare) {
110
+ log('browser.stillOnCloudflare', debug, { debug }, COMPONENTS.BROWSER_FETCH);
111
+ // 再次尝试模拟人类行为
112
+ log('browser.simulatingHuman', debug, {}, COMPONENTS.BROWSER_FETCH);
113
+ await this.simulateHumanBehavior(page, debug);
114
+ // 再次等待页面加载
115
+ await page.waitForNavigation({
116
+ waitUntil: 'networkidle2',
117
+ timeout: 30000
118
+ }).catch(() => { });
119
+ // 最终检查
120
+ const bypassFailed = await page.evaluate(() => {
121
+ return document.title.includes('Cloudflare') ||
122
+ document.title.includes('Security Check') ||
123
+ document.body.textContent?.includes('Checking your browser') ||
124
+ document.querySelector('#challenge-form') !== null;
125
+ });
126
+ if (bypassFailed) {
127
+ log('browser.bypassFailed', debug, { debug }, COMPONENTS.BROWSER_FETCH);
128
+ return false;
129
+ }
130
+ }
131
+ return true;
132
+ }
133
+ return false;
134
+ }
135
+ catch (error) {
136
+ log('browser.cloudflareError', debug, { error: String(error) }, COMPONENTS.BROWSER_FETCH);
137
+ return false;
138
+ }
139
+ }
140
+ /**
141
+ * 模拟人类行为以绕过Cloudflare保护 (Simulate human behavior to bypass Cloudflare protection)
142
+ * @param page 页面对象 (Page object)
143
+ * @param debug 是否启用调试输出 (Whether to enable debug output)
144
+ */
145
+ static async simulateHumanBehavior(page, debug) {
146
+ log('browser.simulatingHuman', debug, {}, COMPONENTS.BROWSER_FETCH);
147
+ try {
148
+ // 随机移动鼠标
149
+ for (let i = 0; i < 5; i++) {
150
+ const x = Math.floor(Math.random() * 500);
151
+ const y = Math.floor(Math.random() * 500);
152
+ await page.mouse.move(x, y);
153
+ await new Promise(resolve => setTimeout(resolve, Math.random() * 500));
154
+ }
155
+ // 随机点击页面
156
+ await page.mouse.click(Math.floor(Math.random() * 500), Math.floor(Math.random() * 500));
157
+ // 随机滚动
158
+ await page.evaluate(() => {
159
+ window.scrollBy(0, Math.floor(Math.random() * 200));
160
+ });
161
+ // 等待一段时间
162
+ await new Promise(resolve => setTimeout(resolve, 2000 + Math.random() * 2000));
163
+ }
164
+ catch (error) {
165
+ log('browser.simulatingHumanError', debug, { error: String(error) }, COMPONENTS.BROWSER_FETCH);
166
+ }
167
+ }
168
+ }
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Author: Martin <lmccc.dev@gmail.com>
3
+ * Co-Author: AI Assistant (Claude)
4
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
5
+ */
6
+ import { z } from "zod";
7
+ /**
8
+ * 浏览器特定参数 (Browser-specific parameters)
9
+ */
10
+ export const BrowserParamsSchema = z.object({
11
+ waitForSelector: z.string().optional(),
12
+ waitForTimeout: z.number().optional(),
13
+ scrollToBottom: z.boolean().optional(),
14
+ saveCookies: z.boolean().optional(),
15
+ useBrowser: z.boolean().optional(),
16
+ useNodeFetch: z.boolean().optional(),
17
+ autoDetectMode: z.boolean().optional(),
18
+ closeBrowser: z.boolean().optional(),
19
+ });
20
+ /**
21
+ * 请求参数模式 (Request parameters schema)
22
+ */
23
+ export const RequestPayloadSchema = z.object({
24
+ url: z.string(),
25
+ method: z.string().optional(),
26
+ headers: z.record(z.string()).optional(),
27
+ proxy: z.string().optional(),
28
+ noDelay: z.boolean().optional(),
29
+ timeout: z.number().optional(),
30
+ maxRedirects: z.number().optional(),
31
+ useSystemProxy: z.boolean().optional(),
32
+ debug: z.boolean().optional(),
33
+ useBrowser: z.boolean().optional(),
34
+ useNodeFetch: z.boolean().optional(),
35
+ autoDetectMode: z.boolean().optional(),
36
+ waitForSelector: z.string().optional(),
37
+ waitForTimeout: z.number().optional(),
38
+ scrollToBottom: z.boolean().optional(),
39
+ closeBrowser: z.boolean().optional(),
40
+ saveCookies: z.boolean().optional(),
41
+ }).merge(BrowserParamsSchema);
@@ -0,0 +1,141 @@
1
+ /**
2
+ * Author: Martin <lmccc.dev@gmail.com>
3
+ * Co-Author: AI Assistant (Claude)
4
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
5
+ */
6
+ import { execSync } from 'child_process';
7
+ import { log, COMPONENTS } from '../../logger.js';
8
+ import { t } from '../../i18n/index.js';
9
+ /**
10
+ * 常用浏览器的User-Agent列表 (List of common browser User-Agents)
11
+ * 用于模拟不同浏览器的请求 (Used to simulate requests from different browsers)
12
+ */
13
+ export const userAgents = [
14
+ // Chrome
15
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
16
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
17
+ // Firefox
18
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
19
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:89.0) Gecko/20100101 Firefox/89.0',
20
+ // Safari
21
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15',
22
+ // Edge
23
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.59'
24
+ ];
25
+ /**
26
+ * 获取随机User-Agent (Get random User-Agent)
27
+ * @returns 随机的User-Agent字符串 (Random User-Agent string)
28
+ */
29
+ export function getRandomUserAgent() {
30
+ const index = Math.floor(Math.random() * userAgents.length);
31
+ return userAgents[index];
32
+ }
33
+ /**
34
+ * 随机延迟函数 (Random delay function)
35
+ * 模拟人类行为,避免被检测为机器人 (Simulate human behavior, avoid being detected as a bot)
36
+ * @param minMs 最小延迟毫秒数 (Minimum delay in milliseconds)
37
+ * @param maxMs 最大延迟毫秒数 (Maximum delay in milliseconds)
38
+ * @returns Promise对象 (Promise object)
39
+ */
40
+ export async function randomDelay(minMs = 500, maxMs = 3000) {
41
+ const delay = Math.floor(Math.random() * (maxMs - minMs + 1)) + minMs;
42
+ return new Promise(resolve => setTimeout(resolve, delay));
43
+ }
44
+ /**
45
+ * 从URL中提取域名 (Extract domain from URL)
46
+ * @param url URL字符串 (URL string)
47
+ * @returns 域名字符串 (Domain string)
48
+ */
49
+ export function getDomain(url) {
50
+ try {
51
+ const urlObj = new URL(url);
52
+ return urlObj.hostname;
53
+ }
54
+ catch (error) {
55
+ return '';
56
+ }
57
+ }
58
+ /**
59
+ * 获取系统代理设置 (Get system proxy settings)
60
+ * @param useSystemProxy 是否使用系统代理 (Whether to use system proxy)
61
+ * @param debug 是否启用调试输出 (Whether to enable debug output)
62
+ * @param component 组件名称 (Component name)
63
+ * @returns 系统代理URL或undefined (System proxy URL or undefined)
64
+ */
65
+ export function getSystemProxy(useSystemProxy = true, debug = false, component = COMPONENTS.NODE_FETCH) {
66
+ // 如果不使用系统代理,直接返回undefined (If not using system proxy, return undefined directly)
67
+ if (!useSystemProxy) {
68
+ log('fetcher.systemProxyDisabled', debug, {}, component);
69
+ return undefined;
70
+ }
71
+ // 检查所有可能的代理环境变量 (Check all possible proxy environment variables)
72
+ const proxyVars = [
73
+ 'HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy',
74
+ 'ALL_PROXY', 'all_proxy', 'NO_PROXY', 'no_proxy'
75
+ ];
76
+ // 输出所有环境变量的值,帮助调试 (Output all environment variable values to help debugging)
77
+ log('fetcher.checkingProxyEnv', debug, {}, component);
78
+ for (const varName of proxyVars) {
79
+ log('fetcher.envVarValue', debug, {
80
+ envVar: varName,
81
+ value: process.env[varName] || t('fetcher.notSet')
82
+ }, component);
83
+ }
84
+ // 尝试获取代理设置 (Try to get proxy settings)
85
+ const proxyUrl = process.env.HTTP_PROXY || process.env.HTTPS_PROXY ||
86
+ process.env.http_proxy || process.env.https_proxy ||
87
+ process.env.ALL_PROXY || process.env.all_proxy;
88
+ if (proxyUrl) {
89
+ log('fetcher.foundSystemProxy', debug, { proxy: proxyUrl }, component);
90
+ return proxyUrl;
91
+ }
92
+ // 尝试从系统命令获取代理设置
93
+ try {
94
+ let shellOutput = '';
95
+ if (process.platform === 'win32') {
96
+ // Windows 系统使用 set 命令,忽略错误 (Windows systems use set command, ignore errors)
97
+ try {
98
+ shellOutput = execSync('set http_proxy & set https_proxy & set HTTP_PROXY & set HTTPS_PROXY', { encoding: 'utf8' });
99
+ }
100
+ catch (winError) {
101
+ log('fetcher.errorGettingWindowsEnvVars', debug, { error: winError.toString() }, component);
102
+ }
103
+ }
104
+ else {
105
+ // Unix 系统使用 env 命令,忽略错误 (Unix systems use env command, ignore errors)
106
+ try {
107
+ // 使用 env 命令获取所有环境变量,不使用 grep 过滤,避免在没有匹配时返回非零退出码
108
+ shellOutput = execSync('env', { encoding: 'utf8' });
109
+ // 在代码中过滤代理相关的环境变量
110
+ shellOutput = shellOutput.split('\n')
111
+ .filter(line => line.toLowerCase().includes('proxy'))
112
+ .join('\n');
113
+ }
114
+ catch (unixError) {
115
+ log('fetcher.errorGettingUnixEnvVars', debug, { error: unixError.toString() }, component);
116
+ }
117
+ }
118
+ if (shellOutput.trim()) {
119
+ log('fetcher.systemCommandProxySettings', debug, { output: shellOutput.trim() }, component);
120
+ // 解析输出找到代理 URL
121
+ const proxyMatch = shellOutput.match(/(?:HTTP_PROXY|HTTPS_PROXY|http_proxy|https_proxy)=([^\s]+)/i);
122
+ if (proxyMatch && proxyMatch[1]) {
123
+ const systemProxyUrl = proxyMatch[1];
124
+ log('fetcher.foundProxyFromCommand', debug, { proxy: systemProxyUrl }, component);
125
+ return systemProxyUrl;
126
+ }
127
+ }
128
+ else {
129
+ log('fetcher.noSystemProxyFound', debug, {}, component);
130
+ }
131
+ }
132
+ catch (error) {
133
+ log('fetcher.errorGettingSystemEnvVars', debug, { error: error.toString() }, component);
134
+ }
135
+ // 检查是否设置了 NO_PROXY
136
+ const noProxy = process.env.NO_PROXY || process.env.no_proxy;
137
+ if (noProxy) {
138
+ log('fetcher.foundNoProxy', debug, { noProxy: noProxy }, component);
139
+ }
140
+ return undefined;
141
+ }
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Author: Martin <lmccc.dev@gmail.com>
3
+ * Co-Author: AI Assistant (Claude)
4
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
5
+ */
6
+ import { FetcherFactory } from './FetcherFactory.js';
7
+ import { log, COMPONENTS } from '../logger.js';
8
+ /**
9
+ * 统一抓取器类 (Unified fetcher class)
10
+ * 提供统一的接口来获取不同格式的内容 (Provides a unified interface to fetch content in different formats)
11
+ */
12
+ export class Fetcher {
13
+ /**
14
+ * 获取HTML内容 (Get HTML content)
15
+ * @param requestPayload 请求参数 (Request parameters)
16
+ * @returns HTML内容 (HTML content)
17
+ */
18
+ static async html(requestPayload) {
19
+ const { debug = false } = requestPayload;
20
+ log('fetcher.fetchingHtml', debug, { url: requestPayload.url }, COMPONENTS.SERVER);
21
+ const fetcher = FetcherFactory.createFetcher(requestPayload);
22
+ return await fetcher.html(requestPayload);
23
+ }
24
+ /**
25
+ * 获取JSON内容 (Get JSON content)
26
+ * @param requestPayload 请求参数 (Request parameters)
27
+ * @returns JSON内容 (JSON content)
28
+ */
29
+ static async json(requestPayload) {
30
+ const { debug = false } = requestPayload;
31
+ log('fetcher.fetchingJson', debug, { url: requestPayload.url }, COMPONENTS.SERVER);
32
+ const fetcher = FetcherFactory.createFetcher(requestPayload);
33
+ return await fetcher.json(requestPayload);
34
+ }
35
+ /**
36
+ * 获取纯文本内容 (Get plain text content)
37
+ * @param requestPayload 请求参数 (Request parameters)
38
+ * @returns 纯文本内容 (Plain text content)
39
+ */
40
+ static async txt(requestPayload) {
41
+ const { debug = false } = requestPayload;
42
+ log('fetcher.fetchingTxt', debug, { url: requestPayload.url }, COMPONENTS.SERVER);
43
+ const fetcher = FetcherFactory.createFetcher(requestPayload);
44
+ return await fetcher.txt(requestPayload);
45
+ }
46
+ /**
47
+ * 获取Markdown内容 (Get Markdown content)
48
+ * @param requestPayload 请求参数 (Request parameters)
49
+ * @returns Markdown内容 (Markdown content)
50
+ */
51
+ static async markdown(requestPayload) {
52
+ const { debug = false } = requestPayload;
53
+ log('fetcher.fetchingMarkdown', debug, { url: requestPayload.url }, COMPONENTS.SERVER);
54
+ const fetcher = FetcherFactory.createFetcher(requestPayload);
55
+ return await fetcher.markdown(requestPayload);
56
+ }
57
+ }
58
+ // 导出所有类型和类 (Export all types and classes)
59
+ export * from './common/types.js';
60
+ export * from './browser/BrowserFetcher.js';
61
+ export * from './node/NodeFetcher.js';
62
+ export * from './FetcherFactory.js';
@@ -0,0 +1,239 @@
1
+ /**
2
+ * Author: Martin <lmccc.dev@gmail.com>
3
+ * Co-Author: AI Assistant (Claude)
4
+ * Description: This code was collaboratively developed by Martin and AI Assistant.
5
+ */
6
+ import fetch from "node-fetch";
7
+ import { HttpProxyAgent } from "http-proxy-agent";
8
+ import { HttpsProxyAgent } from "https-proxy-agent";
9
+ import { log, COMPONENTS } from '../../logger.js';
10
+ import { getRandomUserAgent, randomDelay, getSystemProxy } from '../common/utils.js';
11
+ /**
12
+ * HTTP客户端类 (HTTP client class)
13
+ * 处理HTTP请求、重定向和错误 (Handle HTTP requests, redirects and errors)
14
+ */
15
+ export class HttpClient {
16
+ /**
17
+ * 准备请求头 (Prepare request headers)
18
+ * @param headers 原始请求头 (Original headers)
19
+ * @param debug 是否启用调试模式 (Whether debug mode is enabled)
20
+ * @returns 处理后的请求头 (Processed headers)
21
+ */
22
+ static prepareHeaders(headers = {}, debug) {
23
+ const requestHeaders = { ...headers };
24
+ // 添加随机User-Agent (Add random User-Agent)
25
+ if (!requestHeaders['User-Agent']) {
26
+ const userAgent = getRandomUserAgent();
27
+ requestHeaders['User-Agent'] = userAgent;
28
+ log('node.usingUserAgent', debug, { userAgent }, COMPONENTS.NODE_FETCH);
29
+ }
30
+ return requestHeaders;
31
+ }
32
+ /**
33
+ * 设置代理 (Set proxy)
34
+ * @param url 请求URL (Request URL)
35
+ * @param proxy 代理URL (Proxy URL)
36
+ * @param debug 是否启用调试模式 (Whether debug mode is enabled)
37
+ * @returns 代理Agent (Proxy agent)
38
+ */
39
+ static setupProxy(url, proxy, debug) {
40
+ if (!proxy)
41
+ return undefined;
42
+ let agent = undefined;
43
+ if (url.startsWith('https://')) {
44
+ agent = new HttpsProxyAgent(proxy);
45
+ log('node.usingHttpsProxy', debug, {}, COMPONENTS.NODE_FETCH);
46
+ }
47
+ else {
48
+ agent = new HttpProxyAgent(proxy);
49
+ log('node.usingHttpProxy', debug, {}, COMPONENTS.NODE_FETCH);
50
+ }
51
+ return agent;
52
+ }
53
+ /**
54
+ * 处理请求延迟 (Handle request delay)
55
+ * @param noDelay 是否禁用延迟 (Whether to disable delay)
56
+ * @param isRedirect 是否是重定向请求 (Whether it's a redirect request)
57
+ * @param debug 是否启用调试模式 (Whether debug mode is enabled)
58
+ */
59
+ static async handleDelay(noDelay, isRedirect, debug) {
60
+ if (!noDelay && isRedirect) {
61
+ await randomDelay();
62
+ }
63
+ }
64
+ /**
65
+ * 处理重定向URL (Handle redirect URL)
66
+ * @param location 重定向位置 (Redirect location)
67
+ * @param currentUrl 当前URL (Current URL)
68
+ * @param debug 是否启用调试模式 (Whether debug mode is enabled)
69
+ * @returns 完整的重定向URL (Complete redirect URL)
70
+ */
71
+ static buildRedirectUrl(location, currentUrl, debug) {
72
+ let redirectUrl = location;
73
+ if (location.startsWith('/')) {
74
+ const urlObj = new URL(currentUrl);
75
+ redirectUrl = `${urlObj.protocol}//${urlObj.host}${location}`;
76
+ }
77
+ else if (!location.startsWith('http')) {
78
+ redirectUrl = new URL(location, currentUrl).toString();
79
+ }
80
+ log('node.constructedFullRedirectUrl', debug, { redirectUrl }, COMPONENTS.NODE_FETCH);
81
+ return redirectUrl;
82
+ }
83
+ /**
84
+ * 处理错误响应 (Handle error response)
85
+ * @param response 响应对象 (Response object)
86
+ * @param debug 是否启用调试模式 (Whether debug mode is enabled)
87
+ * @returns Promise<never> 抛出错误 (Throws error)
88
+ */
89
+ static async handleErrorResponse(response, debug) {
90
+ log('node.errorResponse', debug, { status: response.status, statusText: response.statusText }, COMPONENTS.NODE_FETCH);
91
+ // 尝试读取错误响应体 (Try to read error response body)
92
+ let errorText = '';
93
+ try {
94
+ errorText = await response.text();
95
+ log('node.errorResponseBody', debug, { body: errorText.substring(0, 200) + (errorText.length > 200 ? '...' : '') }, COMPONENTS.NODE_FETCH);
96
+ }
97
+ catch (textError) {
98
+ log('node.errorReadingBody', debug, { error: String(textError) }, COMPONENTS.NODE_FETCH);
99
+ }
100
+ // 创建错误对象 (Create error object)
101
+ const error = new Error(`HTTP Error ${response.status}: ${response.statusText}`);
102
+ error.status = response.status;
103
+ error.statusText = response.statusText;
104
+ error.body = errorText;
105
+ throw error;
106
+ }
107
+ /**
108
+ * 处理请求错误 (Handle request error)
109
+ * @param error 错误对象 (Error object)
110
+ * @param fetchStart 请求开始时间 (Request start time)
111
+ * @param timeout 超时时间 (Timeout)
112
+ * @param redirectCount 重定向计数 (Redirect count)
113
+ * @param maxRedirects 最大重定向次数 (Maximum redirects)
114
+ * @param debug 是否启用调试模式 (Whether debug mode is enabled)
115
+ * @returns Promise<never> 抛出错误 (Throws error)
116
+ */
117
+ static handleRequestError(error, fetchStart, timeout, redirectCount, maxRedirects, debug) {
118
+ // 记录错误 (Log error)
119
+ log('node.fetchError', debug, { error: String(error) }, COMPONENTS.NODE_FETCH);
120
+ // 处理超时错误 (Handle timeout error)
121
+ if (error.name === 'AbortError') {
122
+ log('node.requestAborted', debug, { duration: Date.now() - fetchStart }, COMPONENTS.NODE_FETCH);
123
+ const timeoutError = new Error(`Request timeout after ${timeout}ms`);
124
+ timeoutError.code = 'ETIMEDOUT';
125
+ timeoutError.timeout = timeout;
126
+ throw timeoutError;
127
+ }
128
+ // 处理网络错误 (Handle network error)
129
+ if (error.code) {
130
+ log('node.networkError', debug, { code: error.code || error.name }, COMPONENTS.NODE_FETCH);
131
+ const networkError = new Error(`Network error: ${error.code || error.message}`);
132
+ networkError.code = error.code;
133
+ networkError.originalError = error;
134
+ throw networkError;
135
+ }
136
+ // 处理重定向错误 (Handle redirect error)
137
+ if (redirectCount >= maxRedirects) {
138
+ log('node.tooManyRedirects', debug, { redirects: maxRedirects }, COMPONENTS.NODE_FETCH);
139
+ const redirectError = new Error(`Too many redirects (${maxRedirects})`);
140
+ redirectError.code = 'EMAXREDIRECTS';
141
+ redirectError.redirects = redirectCount;
142
+ throw redirectError;
143
+ }
144
+ // 重新抛出其他错误 (Rethrow other errors)
145
+ throw error;
146
+ }
147
+ /**
148
+ * 执行带重定向处理的HTTP请求 (Execute HTTP request with redirect handling)
149
+ * 支持自动处理重定向、超时和代理 (Supports automatic handling of redirects, timeouts, and proxies)
150
+ */
151
+ static async fetchWithRedirects({ url, headers = {}, proxy, noDelay, timeout = 30000, // 默认30秒超时
152
+ maxRedirects = 10, // 最大重定向次数
153
+ useSystemProxy = true, // 是否使用系统代理
154
+ debug = false, // 是否启用调试模式
155
+ }) {
156
+ log('node.fetchingUrl', debug, { url }, COMPONENTS.NODE_FETCH);
157
+ // 处理代理设置 (Handle proxy settings)
158
+ const systemProxy = getSystemProxy(useSystemProxy, debug, COMPONENTS.NODE_FETCH);
159
+ const finalProxy = proxy || systemProxy;
160
+ if (finalProxy) {
161
+ log('node.usingProxy', debug, { proxy: finalProxy }, COMPONENTS.NODE_FETCH);
162
+ }
163
+ // 初始化重定向计数器 (Initialize redirect counter)
164
+ let redirectCount = 0;
165
+ let currentUrl = url;
166
+ // 记录请求开始时间 (Record request start time)
167
+ const fetchStart = Date.now();
168
+ // 创建AbortController用于超时控制 (Create AbortController for timeout control)
169
+ const controller = new AbortController();
170
+ const timeoutId = setTimeout(() => controller.abort(), timeout);
171
+ try {
172
+ // 处理重定向循环 (Handle redirect loop)
173
+ while (redirectCount < maxRedirects) {
174
+ log('node.fetchingUrl', debug, { url: currentUrl, redirect: redirectCount }, COMPONENTS.NODE_FETCH);
175
+ // 处理延迟 (Handle delay)
176
+ await this.handleDelay(noDelay, redirectCount > 0, debug);
177
+ // 准备请求头 (Prepare request headers)
178
+ const requestHeaders = this.prepareHeaders(headers, debug);
179
+ // 设置代理 (Set proxy)
180
+ const agent = this.setupProxy(currentUrl, finalProxy, debug);
181
+ // 准备请求选项 (Prepare request options)
182
+ const fetchOptions = {
183
+ method: 'GET',
184
+ headers: requestHeaders,
185
+ timeout,
186
+ signal: controller.signal,
187
+ };
188
+ if (agent) {
189
+ fetchOptions.agent = agent;
190
+ }
191
+ // 记录请求详情 (Log request details)
192
+ log('node.requestDetails', debug, {
193
+ url: currentUrl,
194
+ method: fetchOptions.method || 'GET',
195
+ headers: requestHeaders,
196
+ proxy: finalProxy,
197
+ timeout,
198
+ }, COMPONENTS.NODE_FETCH);
199
+ // 执行请求 (Execute request)
200
+ const response = await fetch(currentUrl, fetchOptions);
201
+ // 记录响应状态 (Log response status)
202
+ log('node.responseStatus', debug, {
203
+ status: response.status,
204
+ statusText: response.statusText,
205
+ headers: Object.fromEntries(response.headers.entries()),
206
+ }, COMPONENTS.NODE_FETCH);
207
+ // 处理重定向 (Handle redirects)
208
+ if (response.status >= 300 && response.status < 400 && response.headers.has('location')) {
209
+ redirectCount++;
210
+ // 获取重定向URL (Get redirect URL)
211
+ const location = response.headers.get('location');
212
+ log('node.redirectingTo', debug, { location }, COMPONENTS.NODE_FETCH);
213
+ // 构建完整的重定向URL (Build complete redirect URL)
214
+ currentUrl = this.buildRedirectUrl(location, currentUrl, debug);
215
+ continue;
216
+ }
217
+ // 如果响应成功,返回响应 (If response is successful, return response)
218
+ if (response.ok) {
219
+ log('node.requestSuccess', debug, {}, COMPONENTS.NODE_FETCH);
220
+ return response;
221
+ }
222
+ else {
223
+ // 处理错误响应 (Handle error response)
224
+ return await this.handleErrorResponse(response, debug);
225
+ }
226
+ }
227
+ // 如果达到最大重定向次数,抛出错误 (If maximum redirects reached, throw error)
228
+ throw new Error(`Too many redirects (${maxRedirects})`);
229
+ }
230
+ catch (error) {
231
+ // 处理请求错误 (Handle request error)
232
+ this.handleRequestError(error, fetchStart, timeout, redirectCount, maxRedirects, debug);
233
+ }
234
+ finally {
235
+ // 清除超时定时器 (Clear timeout timer)
236
+ clearTimeout(timeoutId);
237
+ }
238
+ }
239
+ }