@j0hanz/superfetch 1.1.2 → 1.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/README.md +52 -30
  2. package/dist/config/formatting.d.ts +9 -0
  3. package/dist/config/formatting.d.ts.map +1 -0
  4. package/dist/config/formatting.js +11 -0
  5. package/dist/config/formatting.js.map +1 -0
  6. package/dist/config/index.d.ts +11 -2
  7. package/dist/config/index.d.ts.map +1 -1
  8. package/dist/config/index.js +35 -13
  9. package/dist/config/index.js.map +1 -1
  10. package/dist/config/types/content.d.ts +107 -0
  11. package/dist/config/types/content.d.ts.map +1 -0
  12. package/dist/config/types/content.js +2 -0
  13. package/dist/config/types/content.js.map +1 -0
  14. package/dist/config/types/runtime.d.ts +78 -0
  15. package/dist/config/types/runtime.d.ts.map +1 -0
  16. package/dist/config/types/runtime.js +2 -0
  17. package/dist/config/types/runtime.js.map +1 -0
  18. package/dist/config/types/tools.d.ts +99 -0
  19. package/dist/config/types/tools.d.ts.map +1 -0
  20. package/dist/config/types/tools.js +2 -0
  21. package/dist/config/types/tools.js.map +1 -0
  22. package/dist/config/types.d.ts +3 -297
  23. package/dist/config/types.d.ts.map +1 -1
  24. package/dist/http/auth.d.ts +3 -0
  25. package/dist/http/auth.d.ts.map +1 -0
  26. package/dist/http/auth.js +34 -0
  27. package/dist/http/auth.js.map +1 -0
  28. package/dist/http/cors.d.ts +8 -0
  29. package/dist/http/cors.d.ts.map +1 -0
  30. package/dist/http/cors.js +47 -0
  31. package/dist/http/cors.js.map +1 -0
  32. package/dist/http/mcp-routes.d.ts +5 -0
  33. package/dist/http/mcp-routes.d.ts.map +1 -0
  34. package/dist/http/mcp-routes.js +110 -0
  35. package/dist/http/mcp-routes.js.map +1 -0
  36. package/dist/http/mcp-session.d.ts +12 -0
  37. package/dist/http/mcp-session.d.ts.map +1 -0
  38. package/dist/http/mcp-session.js +209 -0
  39. package/dist/http/mcp-session.js.map +1 -0
  40. package/dist/http/mcp-validation.d.ts +3 -0
  41. package/dist/http/mcp-validation.d.ts.map +1 -0
  42. package/dist/http/mcp-validation.js +34 -0
  43. package/dist/http/mcp-validation.js.map +1 -0
  44. package/dist/http/rate-limit.d.ts +13 -0
  45. package/dist/http/rate-limit.d.ts.map +1 -0
  46. package/dist/http/rate-limit.js +91 -0
  47. package/dist/http/rate-limit.js.map +1 -0
  48. package/dist/http/server.d.ts +4 -0
  49. package/dist/http/server.d.ts.map +1 -0
  50. package/dist/http/server.js +183 -0
  51. package/dist/http/server.js.map +1 -0
  52. package/dist/http/sessions.d.ts +15 -0
  53. package/dist/http/sessions.d.ts.map +1 -0
  54. package/dist/http/sessions.js +64 -0
  55. package/dist/http/sessions.js.map +1 -0
  56. package/dist/index.js +26 -341
  57. package/dist/index.js.map +1 -1
  58. package/dist/middleware/error-handler.d.ts +2 -2
  59. package/dist/middleware/error-handler.d.ts.map +1 -1
  60. package/dist/middleware/error-handler.js +46 -15
  61. package/dist/middleware/error-handler.js.map +1 -1
  62. package/dist/resources/cached-content.d.ts.map +1 -1
  63. package/dist/resources/cached-content.js +104 -44
  64. package/dist/resources/cached-content.js.map +1 -1
  65. package/dist/resources/index.d.ts.map +1 -1
  66. package/dist/resources/index.js +77 -69
  67. package/dist/resources/index.js.map +1 -1
  68. package/dist/server.d.ts.map +1 -1
  69. package/dist/server.js +9 -3
  70. package/dist/server.js.map +1 -1
  71. package/dist/services/cache.d.ts +13 -1
  72. package/dist/services/cache.d.ts.map +1 -1
  73. package/dist/services/cache.js +89 -16
  74. package/dist/services/cache.js.map +1 -1
  75. package/dist/services/context.d.ts +1 -1
  76. package/dist/services/context.d.ts.map +1 -1
  77. package/dist/services/context.js +1 -1
  78. package/dist/services/context.js.map +1 -1
  79. package/dist/services/extractor.d.ts.map +1 -1
  80. package/dist/services/extractor.js +122 -87
  81. package/dist/services/extractor.js.map +1 -1
  82. package/dist/services/fetcher/agents.d.ts +4 -0
  83. package/dist/services/fetcher/agents.d.ts.map +1 -0
  84. package/dist/services/fetcher/agents.js +111 -0
  85. package/dist/services/fetcher/agents.js.map +1 -0
  86. package/dist/services/fetcher/errors.d.ts +5 -0
  87. package/dist/services/fetcher/errors.d.ts.map +1 -0
  88. package/dist/services/fetcher/errors.js +71 -0
  89. package/dist/services/fetcher/errors.js.map +1 -0
  90. package/dist/services/fetcher/headers.d.ts +2 -0
  91. package/dist/services/fetcher/headers.d.ts.map +1 -0
  92. package/dist/services/fetcher/headers.js +28 -0
  93. package/dist/services/fetcher/headers.js.map +1 -0
  94. package/dist/services/fetcher/interceptors.d.ts +10 -0
  95. package/dist/services/fetcher/interceptors.d.ts.map +1 -0
  96. package/dist/services/fetcher/interceptors.js +82 -0
  97. package/dist/services/fetcher/interceptors.js.map +1 -0
  98. package/dist/services/fetcher/redirects.d.ts +6 -0
  99. package/dist/services/fetcher/redirects.d.ts.map +1 -0
  100. package/dist/services/fetcher/redirects.js +67 -0
  101. package/dist/services/fetcher/redirects.js.map +1 -0
  102. package/dist/services/fetcher/response.d.ts +5 -0
  103. package/dist/services/fetcher/response.d.ts.map +1 -0
  104. package/dist/services/fetcher/response.js +39 -0
  105. package/dist/services/fetcher/response.js.map +1 -0
  106. package/dist/services/fetcher/retry-policy.d.ts +28 -0
  107. package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
  108. package/dist/services/fetcher/retry-policy.js +138 -0
  109. package/dist/services/fetcher/retry-policy.js.map +1 -0
  110. package/dist/services/fetcher.d.ts +2 -1
  111. package/dist/services/fetcher.d.ts.map +1 -1
  112. package/dist/services/fetcher.js +62 -315
  113. package/dist/services/fetcher.js.map +1 -1
  114. package/dist/services/logger.js +4 -4
  115. package/dist/services/logger.js.map +1 -1
  116. package/dist/services/parser.d.ts +1 -0
  117. package/dist/services/parser.d.ts.map +1 -1
  118. package/dist/services/parser.js +55 -35
  119. package/dist/services/parser.js.map +1 -1
  120. package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
  121. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
  122. package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
  123. package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
  124. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  125. package/dist/tools/handlers/fetch-links.tool.js +78 -116
  126. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  127. package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
  128. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  129. package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
  130. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  131. package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
  132. package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
  133. package/dist/tools/handlers/fetch-single.shared.js +49 -0
  134. package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
  135. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  136. package/dist/tools/handlers/fetch-url.tool.js +82 -54
  137. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  138. package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
  139. package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
  140. package/dist/tools/handlers/fetch-urls/processor.js +153 -0
  141. package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
  142. package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
  143. package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
  144. package/dist/tools/handlers/fetch-urls/response.js +58 -0
  145. package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
  146. package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
  147. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
  148. package/dist/tools/handlers/fetch-urls/validation.js +18 -0
  149. package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
  150. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
  151. package/dist/tools/handlers/fetch-urls.tool.js +104 -202
  152. package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
  153. package/dist/tools/index.d.ts.map +1 -1
  154. package/dist/tools/index.js +36 -237
  155. package/dist/tools/index.js.map +1 -1
  156. package/dist/tools/schemas.d.ts +357 -0
  157. package/dist/tools/schemas.d.ts.map +1 -0
  158. package/dist/tools/schemas.js +272 -0
  159. package/dist/tools/schemas.js.map +1 -0
  160. package/dist/tools/utils/cache-vary.d.ts +3 -0
  161. package/dist/tools/utils/cache-vary.d.ts.map +1 -0
  162. package/dist/tools/utils/cache-vary.js +44 -0
  163. package/dist/tools/utils/cache-vary.js.map +1 -0
  164. package/dist/tools/utils/common.d.ts +2 -2
  165. package/dist/tools/utils/common.d.ts.map +1 -1
  166. package/dist/tools/utils/common.js +5 -1
  167. package/dist/tools/utils/common.js.map +1 -1
  168. package/dist/tools/utils/content-transform.d.ts +16 -0
  169. package/dist/tools/utils/content-transform.d.ts.map +1 -0
  170. package/dist/tools/utils/content-transform.js +49 -0
  171. package/dist/tools/utils/content-transform.js.map +1 -0
  172. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  173. package/dist/tools/utils/fetch-pipeline.js +32 -18
  174. package/dist/tools/utils/fetch-pipeline.js.map +1 -1
  175. package/dist/tools/utils/inline-content.d.ts +11 -0
  176. package/dist/tools/utils/inline-content.d.ts.map +1 -0
  177. package/dist/tools/utils/inline-content.js +39 -0
  178. package/dist/tools/utils/inline-content.js.map +1 -0
  179. package/dist/tools/utils/markdown-toc.d.ts +3 -0
  180. package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
  181. package/dist/tools/utils/markdown-toc.js +35 -0
  182. package/dist/tools/utils/markdown-toc.js.map +1 -0
  183. package/dist/tools/utils/tool-response.d.ts +9 -0
  184. package/dist/tools/utils/tool-response.d.ts.map +1 -0
  185. package/dist/tools/utils/tool-response.js +19 -0
  186. package/dist/tools/utils/tool-response.js.map +1 -0
  187. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  188. package/dist/transformers/jsonl.transformer.js +51 -28
  189. package/dist/transformers/jsonl.transformer.js.map +1 -1
  190. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  191. package/dist/transformers/markdown.transformer.js +82 -111
  192. package/dist/transformers/markdown.transformer.js.map +1 -1
  193. package/dist/utils/header-normalizer.d.ts +5 -0
  194. package/dist/utils/header-normalizer.d.ts.map +1 -0
  195. package/dist/utils/header-normalizer.js +25 -0
  196. package/dist/utils/header-normalizer.js.map +1 -0
  197. package/dist/utils/tool-error-handler.d.ts +1 -0
  198. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  199. package/dist/utils/tool-error-handler.js +29 -1
  200. package/dist/utils/tool-error-handler.js.map +1 -1
  201. package/dist/utils/url-validator.d.ts +0 -3
  202. package/dist/utils/url-validator.d.ts.map +1 -1
  203. package/dist/utils/url-validator.js +97 -18
  204. package/dist/utils/url-validator.js.map +1 -1
  205. package/package.json +11 -6
@@ -1,57 +1,136 @@
1
+ import { BlockList, isIP } from 'node:net';
1
2
  import { config } from '../config/index.js';
2
- /**
3
- * Check if an IP address is in a blocked private range
4
- */
3
+ const BLOCK_LIST = new BlockList();
4
+ const BLOCKED_IPV4_SUBNETS = [
5
+ { subnet: '0.0.0.0', prefix: 8 },
6
+ { subnet: '10.0.0.0', prefix: 8 },
7
+ { subnet: '100.64.0.0', prefix: 10 },
8
+ { subnet: '127.0.0.0', prefix: 8 },
9
+ { subnet: '169.254.0.0', prefix: 16 },
10
+ { subnet: '172.16.0.0', prefix: 12 },
11
+ { subnet: '192.168.0.0', prefix: 16 },
12
+ { subnet: '224.0.0.0', prefix: 4 },
13
+ { subnet: '240.0.0.0', prefix: 4 },
14
+ ];
15
+ const BLOCKED_IPV6_SUBNETS = [
16
+ { subnet: '::', prefix: 128 },
17
+ { subnet: '::1', prefix: 128 },
18
+ { subnet: '64:ff9b::', prefix: 96 },
19
+ { subnet: '64:ff9b:1::', prefix: 48 },
20
+ { subnet: '2001::', prefix: 32 },
21
+ { subnet: '2002::', prefix: 16 },
22
+ { subnet: 'fc00::', prefix: 7 },
23
+ { subnet: 'fe80::', prefix: 10 },
24
+ { subnet: 'ff00::', prefix: 8 },
25
+ ];
26
+ for (const entry of BLOCKED_IPV4_SUBNETS) {
27
+ BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv4');
28
+ }
29
+ for (const entry of BLOCKED_IPV6_SUBNETS) {
30
+ BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv6');
31
+ }
32
+ function matchesBlockedIpPatterns(resolvedIp) {
33
+ for (const pattern of config.security.blockedIpPatterns) {
34
+ if (pattern.test(resolvedIp)) {
35
+ return true;
36
+ }
37
+ }
38
+ return false;
39
+ }
5
40
  export function isBlockedIp(ip) {
6
- return config.security.blockedIpPatterns.some((pattern) => pattern.test(ip));
41
+ if (config.security.blockedHosts.has(ip)) {
42
+ return true;
43
+ }
44
+ const ipType = resolveIpType(ip);
45
+ if (!ipType)
46
+ return false;
47
+ const normalizedIp = ip.toLowerCase();
48
+ if (isBlockedByList(normalizedIp, ipType))
49
+ return true;
50
+ return matchesBlockedIpPatterns(normalizedIp);
7
51
  }
8
- export function validateAndNormalizeUrl(urlString) {
52
+ function resolveIpType(ip) {
53
+ const ipType = isIP(ip);
54
+ return ipType === 4 || ipType === 6 ? ipType : null;
55
+ }
56
+ function isBlockedByList(ip, ipType) {
57
+ if (ipType === 4) {
58
+ return BLOCK_LIST.check(ip, 'ipv4');
59
+ }
60
+ return BLOCK_LIST.check(ip, 'ipv6');
61
+ }
62
+ function assertUrlProvided(urlString) {
9
63
  if (!urlString || typeof urlString !== 'string') {
10
64
  throw new Error('URL is required');
11
65
  }
12
- const trimmedUrl = urlString.trim();
66
+ }
67
+ function assertUrlNotEmpty(trimmedUrl) {
13
68
  if (!trimmedUrl) {
14
69
  throw new Error('URL cannot be empty');
15
70
  }
71
+ }
72
+ function assertUrlLength(trimmedUrl) {
16
73
  if (trimmedUrl.length > config.constants.maxUrlLength) {
17
74
  throw new Error(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`);
18
75
  }
19
- let url;
20
- try {
21
- url = new URL(trimmedUrl);
22
- }
23
- catch {
76
+ }
77
+ function parseUrl(trimmedUrl) {
78
+ if (!URL.canParse(trimmedUrl)) {
24
79
  throw new Error('Invalid URL format');
25
80
  }
81
+ return new URL(trimmedUrl);
82
+ }
83
+ function assertProtocolAllowed(url) {
26
84
  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
27
85
  throw new Error(`Invalid protocol: ${url.protocol}. Only http: and https: are allowed`);
28
86
  }
87
+ }
88
+ function assertNoCredentials(url) {
29
89
  if (url.username || url.password) {
30
90
  throw new Error('URLs with embedded credentials are not allowed');
31
91
  }
32
- const hostname = url.hostname.toLowerCase();
92
+ }
93
+ function assertHostnamePresent(hostname) {
33
94
  if (!hostname) {
34
95
  throw new Error('URL must have a valid hostname');
35
96
  }
97
+ }
98
+ function assertHostnameAllowed(hostname) {
36
99
  if (config.security.blockedHosts.has(hostname)) {
37
100
  throw new Error(`Blocked host: ${hostname}. Internal hosts are not allowed`);
38
101
  }
102
+ }
103
+ function assertHostnameNotIpBlocked(hostname) {
39
104
  if (isBlockedIp(hostname)) {
40
105
  throw new Error(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
41
106
  }
107
+ }
108
+ function assertHostnameSuffixAllowed(hostname) {
42
109
  if (hostname.endsWith('.local') || hostname.endsWith('.internal')) {
43
110
  throw new Error(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
44
111
  }
112
+ }
113
+ export function validateAndNormalizeUrl(urlString) {
114
+ assertUrlProvided(urlString);
115
+ const trimmedUrl = urlString.trim();
116
+ assertUrlNotEmpty(trimmedUrl);
117
+ assertUrlLength(trimmedUrl);
118
+ const url = parseUrl(trimmedUrl);
119
+ assertProtocolAllowed(url);
120
+ assertNoCredentials(url);
121
+ const hostname = url.hostname.toLowerCase();
122
+ assertHostnamePresent(hostname);
123
+ assertHostnameAllowed(hostname);
124
+ assertHostnameNotIpBlocked(hostname);
125
+ assertHostnameSuffixAllowed(hostname);
45
126
  return url.href;
46
127
  }
47
128
  export function isInternalUrl(url, baseUrl) {
48
- try {
49
- const urlObj = new URL(url, baseUrl);
50
- const baseUrlObj = new URL(baseUrl);
51
- return urlObj.hostname === baseUrlObj.hostname;
52
- }
53
- catch {
129
+ if (!URL.canParse(baseUrl) || !URL.canParse(url, baseUrl)) {
54
130
  return false;
55
131
  }
132
+ const urlObj = new URL(url, baseUrl);
133
+ const baseUrlObj = new URL(baseUrl);
134
+ return urlObj.hostname === baseUrlObj.hostname;
56
135
  }
57
136
  //# sourceMappingURL=url-validator.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;IACJ,CAAC;IAED,IAAI,GAAQ,CAAC;IAEb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe;IACxD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;AACnC,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE;IAChC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;CAC1B,CAAC;AACX,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7B,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE;IAC9B,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;IACnC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;IAC/B,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;CACvB,CAAC;AAEX,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AACD,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAkB;IAClD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,OAAO,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,EAAU,EAAE,MAAa;IAChD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,UAAkB;IAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,UAAkB;IACzC,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,UAAkB;IAClC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAQ;IACrC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAQ;IACnC,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,0BAA0B,CAAC,QAAgB;IAClD,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAAC,QAAgB;IACnD,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAC9B,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjC,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC3B,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC5C,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAChC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAChC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACrC,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAEtC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe;IACxD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CAAC;AACjD,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@j0hanz/superfetch",
3
- "version": "1.1.2",
3
+ "version": "1.1.4",
4
4
  "mcpName": "io.github.j0hanz/superfetch",
5
5
  "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
6
6
  "type": "module",
@@ -44,37 +44,42 @@
44
44
  "type-check": "tsc --noEmit",
45
45
  "lint": "eslint .",
46
46
  "lint:fix": "eslint . --fix",
47
+ "test": "vitest run",
48
+ "test:coverage": "vitest run --coverage",
49
+ "bench": "npm run build && node scripts/bench.mjs",
47
50
  "knip": "knip",
48
51
  "knip:fix": "knip --fix"
49
52
  },
50
53
  "dependencies": {
51
54
  "@modelcontextprotocol/sdk": "^1.25.1",
52
55
  "@mozilla/readability": "^0.6.0",
53
- "axios": "^1.7.9",
54
56
  "cheerio": "^1.1.2",
55
57
  "domhandler": "^5.0.3",
56
58
  "express": "^5.2.1",
57
- "jsdom": "^27.3.0",
59
+ "linkedom": "^0.18.12",
58
60
  "node-cache": "^5.1.2",
61
+ "safe-regex": "^2.1.1",
59
62
  "turndown": "^7.2.2",
63
+ "undici": "^6.22.0",
60
64
  "zod": "^3.24.1"
61
65
  },
62
66
  "devDependencies": {
63
67
  "@eslint/js": "^9.39.2",
64
68
  "@trivago/prettier-plugin-sort-imports": "^6.0.0",
65
69
  "@types/express": "^5.0.6",
66
- "@types/jsdom": "^27.0.0",
67
70
  "@types/node": "^22.19.3",
68
71
  "@types/turndown": "^5.0.6",
69
72
  "eslint": "^9.23.2",
70
73
  "eslint-config-prettier": "^10.1.8",
71
74
  "eslint-plugin-unused-imports": "^4.3.0",
72
- "knip": "^5.76.2",
75
+ "knip": "^5.77.1",
73
76
  "prettier": "^3.7.4",
74
77
  "shx": "^0.4.0",
75
78
  "tsx": "^4.21.0",
76
79
  "typescript": "^5.9.3",
77
- "typescript-eslint": "^8.50.0"
80
+ "typescript-eslint": "^8.50.1",
81
+ "vitest": "^2.1.9",
82
+ "@vitest/coverage-v8": "^2.1.9"
78
83
  },
79
84
  "engines": {
80
85
  "node": ">=20.0.0"