@j0hanz/superfetch 1.1.1 → 1.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +57 -32
  2. package/dist/config/formatting.d.ts +9 -0
  3. package/dist/config/formatting.d.ts.map +1 -0
  4. package/dist/config/formatting.js +11 -0
  5. package/dist/config/formatting.js.map +1 -0
  6. package/dist/config/index.d.ts +16 -2
  7. package/dist/config/index.d.ts.map +1 -1
  8. package/dist/config/index.js +43 -14
  9. package/dist/config/index.js.map +1 -1
  10. package/dist/config/types/content.d.ts +107 -0
  11. package/dist/config/types/content.d.ts.map +1 -0
  12. package/dist/config/types/content.js +2 -0
  13. package/dist/config/types/content.js.map +1 -0
  14. package/dist/config/types/runtime.d.ts +78 -0
  15. package/dist/config/types/runtime.d.ts.map +1 -0
  16. package/dist/config/types/runtime.js +2 -0
  17. package/dist/config/types/runtime.js.map +1 -0
  18. package/dist/config/types/tools.d.ts +99 -0
  19. package/dist/config/types/tools.d.ts.map +1 -0
  20. package/dist/config/types/tools.js +2 -0
  21. package/dist/config/types/tools.js.map +1 -0
  22. package/dist/config/types.d.ts +3 -296
  23. package/dist/config/types.d.ts.map +1 -1
  24. package/dist/http/auth.d.ts +3 -0
  25. package/dist/http/auth.d.ts.map +1 -0
  26. package/dist/http/auth.js +34 -0
  27. package/dist/http/auth.js.map +1 -0
  28. package/dist/http/cors.d.ts +8 -0
  29. package/dist/http/cors.d.ts.map +1 -0
  30. package/dist/http/cors.js +47 -0
  31. package/dist/http/cors.js.map +1 -0
  32. package/dist/http/mcp-routes.d.ts +5 -0
  33. package/dist/http/mcp-routes.d.ts.map +1 -0
  34. package/dist/http/mcp-routes.js +110 -0
  35. package/dist/http/mcp-routes.js.map +1 -0
  36. package/dist/http/mcp-session.d.ts +12 -0
  37. package/dist/http/mcp-session.d.ts.map +1 -0
  38. package/dist/http/mcp-session.js +209 -0
  39. package/dist/http/mcp-session.js.map +1 -0
  40. package/dist/http/mcp-validation.d.ts +3 -0
  41. package/dist/http/mcp-validation.d.ts.map +1 -0
  42. package/dist/http/mcp-validation.js +34 -0
  43. package/dist/http/mcp-validation.js.map +1 -0
  44. package/dist/http/rate-limit.d.ts +13 -0
  45. package/dist/http/rate-limit.d.ts.map +1 -0
  46. package/dist/http/rate-limit.js +91 -0
  47. package/dist/http/rate-limit.js.map +1 -0
  48. package/dist/http/server.d.ts +4 -0
  49. package/dist/http/server.d.ts.map +1 -0
  50. package/dist/http/server.js +183 -0
  51. package/dist/http/server.js.map +1 -0
  52. package/dist/http/sessions.d.ts +15 -0
  53. package/dist/http/sessions.d.ts.map +1 -0
  54. package/dist/http/sessions.js +64 -0
  55. package/dist/http/sessions.js.map +1 -0
  56. package/dist/index.js +26 -223
  57. package/dist/index.js.map +1 -1
  58. package/dist/middleware/error-handler.d.ts +2 -2
  59. package/dist/middleware/error-handler.d.ts.map +1 -1
  60. package/dist/middleware/error-handler.js +46 -15
  61. package/dist/middleware/error-handler.js.map +1 -1
  62. package/dist/resources/cached-content.d.ts.map +1 -1
  63. package/dist/resources/cached-content.js +104 -44
  64. package/dist/resources/cached-content.js.map +1 -1
  65. package/dist/resources/index.d.ts.map +1 -1
  66. package/dist/resources/index.js +77 -69
  67. package/dist/resources/index.js.map +1 -1
  68. package/dist/server.d.ts.map +1 -1
  69. package/dist/server.js +9 -3
  70. package/dist/server.js.map +1 -1
  71. package/dist/services/cache.d.ts +13 -1
  72. package/dist/services/cache.d.ts.map +1 -1
  73. package/dist/services/cache.js +90 -13
  74. package/dist/services/cache.js.map +1 -1
  75. package/dist/services/context.d.ts +9 -0
  76. package/dist/services/context.d.ts.map +1 -0
  77. package/dist/services/context.js +9 -0
  78. package/dist/services/context.js.map +1 -0
  79. package/dist/services/extractor.d.ts.map +1 -1
  80. package/dist/services/extractor.js +122 -87
  81. package/dist/services/extractor.js.map +1 -1
  82. package/dist/services/fetcher/agents.d.ts +4 -0
  83. package/dist/services/fetcher/agents.d.ts.map +1 -0
  84. package/dist/services/fetcher/agents.js +111 -0
  85. package/dist/services/fetcher/agents.js.map +1 -0
  86. package/dist/services/fetcher/errors.d.ts +5 -0
  87. package/dist/services/fetcher/errors.d.ts.map +1 -0
  88. package/dist/services/fetcher/errors.js +71 -0
  89. package/dist/services/fetcher/errors.js.map +1 -0
  90. package/dist/services/fetcher/headers.d.ts +2 -0
  91. package/dist/services/fetcher/headers.d.ts.map +1 -0
  92. package/dist/services/fetcher/headers.js +28 -0
  93. package/dist/services/fetcher/headers.js.map +1 -0
  94. package/dist/services/fetcher/interceptors.d.ts +10 -0
  95. package/dist/services/fetcher/interceptors.d.ts.map +1 -0
  96. package/dist/services/fetcher/interceptors.js +82 -0
  97. package/dist/services/fetcher/interceptors.js.map +1 -0
  98. package/dist/services/fetcher/redirects.d.ts +6 -0
  99. package/dist/services/fetcher/redirects.d.ts.map +1 -0
  100. package/dist/services/fetcher/redirects.js +67 -0
  101. package/dist/services/fetcher/redirects.js.map +1 -0
  102. package/dist/services/fetcher/response.d.ts +5 -0
  103. package/dist/services/fetcher/response.d.ts.map +1 -0
  104. package/dist/services/fetcher/response.js +39 -0
  105. package/dist/services/fetcher/response.js.map +1 -0
  106. package/dist/services/fetcher/retry-policy.d.ts +28 -0
  107. package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
  108. package/dist/services/fetcher/retry-policy.js +138 -0
  109. package/dist/services/fetcher/retry-policy.js.map +1 -0
  110. package/dist/services/fetcher.d.ts +2 -1
  111. package/dist/services/fetcher.d.ts.map +1 -1
  112. package/dist/services/fetcher.js +61 -254
  113. package/dist/services/fetcher.js.map +1 -1
  114. package/dist/services/logger.d.ts.map +1 -1
  115. package/dist/services/logger.js +14 -5
  116. package/dist/services/logger.js.map +1 -1
  117. package/dist/services/parser.d.ts +1 -0
  118. package/dist/services/parser.d.ts.map +1 -1
  119. package/dist/services/parser.js +55 -35
  120. package/dist/services/parser.js.map +1 -1
  121. package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
  122. package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
  123. package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
  124. package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
  125. package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
  126. package/dist/tools/handlers/fetch-links.tool.js +78 -116
  127. package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
  128. package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
  129. package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
  130. package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
  131. package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
  132. package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
  133. package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
  134. package/dist/tools/handlers/fetch-single.shared.js +49 -0
  135. package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
  136. package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
  137. package/dist/tools/handlers/fetch-url.tool.js +82 -54
  138. package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
  139. package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
  140. package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
  141. package/dist/tools/handlers/fetch-urls/processor.js +153 -0
  142. package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
  143. package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
  144. package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
  145. package/dist/tools/handlers/fetch-urls/response.js +58 -0
  146. package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
  147. package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
  148. package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
  149. package/dist/tools/handlers/fetch-urls/validation.js +18 -0
  150. package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
  151. package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
  152. package/dist/tools/handlers/fetch-urls.tool.js +104 -197
  153. package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
  154. package/dist/tools/index.d.ts.map +1 -1
  155. package/dist/tools/index.js +36 -237
  156. package/dist/tools/index.js.map +1 -1
  157. package/dist/tools/schemas.d.ts +357 -0
  158. package/dist/tools/schemas.d.ts.map +1 -0
  159. package/dist/tools/schemas.js +272 -0
  160. package/dist/tools/schemas.js.map +1 -0
  161. package/dist/tools/utils/cache-vary.d.ts +3 -0
  162. package/dist/tools/utils/cache-vary.d.ts.map +1 -0
  163. package/dist/tools/utils/cache-vary.js +44 -0
  164. package/dist/tools/utils/cache-vary.js.map +1 -0
  165. package/dist/tools/utils/common.d.ts +2 -2
  166. package/dist/tools/utils/common.d.ts.map +1 -1
  167. package/dist/tools/utils/common.js +5 -1
  168. package/dist/tools/utils/common.js.map +1 -1
  169. package/dist/tools/utils/content-transform.d.ts +16 -0
  170. package/dist/tools/utils/content-transform.d.ts.map +1 -0
  171. package/dist/tools/utils/content-transform.js +49 -0
  172. package/dist/tools/utils/content-transform.js.map +1 -0
  173. package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
  174. package/dist/tools/utils/fetch-pipeline.js +32 -18
  175. package/dist/tools/utils/fetch-pipeline.js.map +1 -1
  176. package/dist/tools/utils/inline-content.d.ts +11 -0
  177. package/dist/tools/utils/inline-content.d.ts.map +1 -0
  178. package/dist/tools/utils/inline-content.js +39 -0
  179. package/dist/tools/utils/inline-content.js.map +1 -0
  180. package/dist/tools/utils/markdown-toc.d.ts +3 -0
  181. package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
  182. package/dist/tools/utils/markdown-toc.js +35 -0
  183. package/dist/tools/utils/markdown-toc.js.map +1 -0
  184. package/dist/tools/utils/tool-response.d.ts +9 -0
  185. package/dist/tools/utils/tool-response.d.ts.map +1 -0
  186. package/dist/tools/utils/tool-response.js +19 -0
  187. package/dist/tools/utils/tool-response.js.map +1 -0
  188. package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
  189. package/dist/transformers/jsonl.transformer.js +51 -28
  190. package/dist/transformers/jsonl.transformer.js.map +1 -1
  191. package/dist/transformers/markdown.transformer.d.ts.map +1 -1
  192. package/dist/transformers/markdown.transformer.js +82 -111
  193. package/dist/transformers/markdown.transformer.js.map +1 -1
  194. package/dist/utils/header-normalizer.d.ts +5 -0
  195. package/dist/utils/header-normalizer.d.ts.map +1 -0
  196. package/dist/utils/header-normalizer.js +25 -0
  197. package/dist/utils/header-normalizer.js.map +1 -0
  198. package/dist/utils/tool-error-handler.d.ts +1 -0
  199. package/dist/utils/tool-error-handler.d.ts.map +1 -1
  200. package/dist/utils/tool-error-handler.js +29 -1
  201. package/dist/utils/tool-error-handler.js.map +1 -1
  202. package/dist/utils/url-validator.d.ts +0 -3
  203. package/dist/utils/url-validator.d.ts.map +1 -1
  204. package/dist/utils/url-validator.js +98 -18
  205. package/dist/utils/url-validator.js.map +1 -1
  206. package/package.json +11 -6
@@ -1,57 +1,137 @@
1
+ import { BlockList, isIP } from 'node:net';
1
2
  import { config } from '../config/index.js';
2
- /**
3
- * Check if an IP address is in a blocked private range
4
- */
3
+ const BLOCK_LIST = new BlockList();
4
+ const BLOCKED_IPV4_SUBNETS = [
5
+ { subnet: '0.0.0.0', prefix: 8 },
6
+ { subnet: '10.0.0.0', prefix: 8 },
7
+ { subnet: '100.64.0.0', prefix: 10 },
8
+ { subnet: '127.0.0.0', prefix: 8 },
9
+ { subnet: '169.254.0.0', prefix: 16 },
10
+ { subnet: '172.16.0.0', prefix: 12 },
11
+ { subnet: '192.168.0.0', prefix: 16 },
12
+ { subnet: '224.0.0.0', prefix: 4 },
13
+ { subnet: '240.0.0.0', prefix: 4 },
14
+ ];
15
+ const BLOCKED_IPV6_SUBNETS = [
16
+ { subnet: '::', prefix: 128 },
17
+ { subnet: '::1', prefix: 128 },
18
+ { subnet: '::ffff:0:0', prefix: 96 },
19
+ { subnet: '64:ff9b::', prefix: 96 },
20
+ { subnet: '64:ff9b:1::', prefix: 48 },
21
+ { subnet: '2001::', prefix: 32 },
22
+ { subnet: '2002::', prefix: 16 },
23
+ { subnet: 'fc00::', prefix: 7 },
24
+ { subnet: 'fe80::', prefix: 10 },
25
+ { subnet: 'ff00::', prefix: 8 },
26
+ ];
27
+ for (const entry of BLOCKED_IPV4_SUBNETS) {
28
+ BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv4');
29
+ }
30
+ for (const entry of BLOCKED_IPV6_SUBNETS) {
31
+ BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv6');
32
+ }
33
+ function matchesBlockedIpPatterns(resolvedIp) {
34
+ for (const pattern of config.security.blockedIpPatterns) {
35
+ if (pattern.test(resolvedIp)) {
36
+ return true;
37
+ }
38
+ }
39
+ return false;
40
+ }
5
41
  export function isBlockedIp(ip) {
6
- return config.security.blockedIpPatterns.some((pattern) => pattern.test(ip));
42
+ if (config.security.blockedHosts.has(ip)) {
43
+ return true;
44
+ }
45
+ const ipType = resolveIpType(ip);
46
+ if (!ipType)
47
+ return false;
48
+ const normalizedIp = ip.toLowerCase();
49
+ if (isBlockedByList(normalizedIp, ipType))
50
+ return true;
51
+ return matchesBlockedIpPatterns(normalizedIp);
7
52
  }
8
- export function validateAndNormalizeUrl(urlString) {
53
+ function resolveIpType(ip) {
54
+ const ipType = isIP(ip);
55
+ return ipType === 4 || ipType === 6 ? ipType : null;
56
+ }
57
+ function isBlockedByList(ip, ipType) {
58
+ if (ipType === 4) {
59
+ return BLOCK_LIST.check(ip, 'ipv4');
60
+ }
61
+ return BLOCK_LIST.check(ip, 'ipv6');
62
+ }
63
+ function assertUrlProvided(urlString) {
9
64
  if (!urlString || typeof urlString !== 'string') {
10
65
  throw new Error('URL is required');
11
66
  }
12
- const trimmedUrl = urlString.trim();
67
+ }
68
+ function assertUrlNotEmpty(trimmedUrl) {
13
69
  if (!trimmedUrl) {
14
70
  throw new Error('URL cannot be empty');
15
71
  }
72
+ }
73
+ function assertUrlLength(trimmedUrl) {
16
74
  if (trimmedUrl.length > config.constants.maxUrlLength) {
17
75
  throw new Error(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`);
18
76
  }
19
- let url;
20
- try {
21
- url = new URL(trimmedUrl);
22
- }
23
- catch {
77
+ }
78
+ function parseUrl(trimmedUrl) {
79
+ if (!URL.canParse(trimmedUrl)) {
24
80
  throw new Error('Invalid URL format');
25
81
  }
82
+ return new URL(trimmedUrl);
83
+ }
84
+ function assertProtocolAllowed(url) {
26
85
  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
27
86
  throw new Error(`Invalid protocol: ${url.protocol}. Only http: and https: are allowed`);
28
87
  }
88
+ }
89
+ function assertNoCredentials(url) {
29
90
  if (url.username || url.password) {
30
91
  throw new Error('URLs with embedded credentials are not allowed');
31
92
  }
32
- const hostname = url.hostname.toLowerCase();
93
+ }
94
+ function assertHostnamePresent(hostname) {
33
95
  if (!hostname) {
34
96
  throw new Error('URL must have a valid hostname');
35
97
  }
98
+ }
99
+ function assertHostnameAllowed(hostname) {
36
100
  if (config.security.blockedHosts.has(hostname)) {
37
101
  throw new Error(`Blocked host: ${hostname}. Internal hosts are not allowed`);
38
102
  }
103
+ }
104
+ function assertHostnameNotIpBlocked(hostname) {
39
105
  if (isBlockedIp(hostname)) {
40
106
  throw new Error(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
41
107
  }
108
+ }
109
+ function assertHostnameSuffixAllowed(hostname) {
42
110
  if (hostname.endsWith('.local') || hostname.endsWith('.internal')) {
43
111
  throw new Error(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
44
112
  }
113
+ }
114
+ export function validateAndNormalizeUrl(urlString) {
115
+ assertUrlProvided(urlString);
116
+ const trimmedUrl = urlString.trim();
117
+ assertUrlNotEmpty(trimmedUrl);
118
+ assertUrlLength(trimmedUrl);
119
+ const url = parseUrl(trimmedUrl);
120
+ assertProtocolAllowed(url);
121
+ assertNoCredentials(url);
122
+ const hostname = url.hostname.toLowerCase();
123
+ assertHostnamePresent(hostname);
124
+ assertHostnameAllowed(hostname);
125
+ assertHostnameNotIpBlocked(hostname);
126
+ assertHostnameSuffixAllowed(hostname);
45
127
  return url.href;
46
128
  }
47
129
  export function isInternalUrl(url, baseUrl) {
48
- try {
49
- const urlObj = new URL(url, baseUrl);
50
- const baseUrlObj = new URL(baseUrl);
51
- return urlObj.hostname === baseUrlObj.hostname;
52
- }
53
- catch {
130
+ if (!URL.canParse(baseUrl) || !URL.canParse(url, baseUrl)) {
54
131
  return false;
55
132
  }
133
+ const urlObj = new URL(url, baseUrl);
134
+ const baseUrlObj = new URL(baseUrl);
135
+ return urlObj.hostname === baseUrlObj.hostname;
56
136
  }
57
137
  //# sourceMappingURL=url-validator.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;IACJ,CAAC;IAED,IAAI,GAAQ,CAAC;IAEb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe;IACxD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;AACnC,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE;IAChC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;CAC1B,CAAC;AACX,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7B,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE;IAC9B,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;IACnC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;IAC/B,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;CACvB,CAAC;AAEX,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AACD,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAkB;IAClD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,OAAO,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,EAAU,EAAE,MAAa;IAChD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,UAAkB;IAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,UAAkB;IACzC,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,UAAkB;IAClC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAQ;IACrC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAQ;IACnC,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,0BAA0B,CAAC,QAAgB;IAClD,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAAC,QAAgB;IACnD,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAC9B,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjC,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC3B,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC5C,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAChC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAChC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACrC,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAEtC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe;IACxD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CAAC;AACjD,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@j0hanz/superfetch",
3
- "version": "1.1.1",
3
+ "version": "1.1.3",
4
4
  "mcpName": "io.github.j0hanz/superfetch",
5
5
  "description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
6
6
  "type": "module",
@@ -44,37 +44,42 @@
44
44
  "type-check": "tsc --noEmit",
45
45
  "lint": "eslint .",
46
46
  "lint:fix": "eslint . --fix",
47
+ "test": "vitest run",
48
+ "test:coverage": "vitest run --coverage",
49
+ "bench": "npm run build && node scripts/bench.mjs",
47
50
  "knip": "knip",
48
51
  "knip:fix": "knip --fix"
49
52
  },
50
53
  "dependencies": {
51
54
  "@modelcontextprotocol/sdk": "^1.25.1",
52
55
  "@mozilla/readability": "^0.6.0",
53
- "axios": "^1.7.9",
54
56
  "cheerio": "^1.1.2",
55
57
  "domhandler": "^5.0.3",
56
58
  "express": "^5.2.1",
57
- "jsdom": "^27.3.0",
59
+ "linkedom": "^0.18.12",
58
60
  "node-cache": "^5.1.2",
61
+ "safe-regex": "^2.1.1",
59
62
  "turndown": "^7.2.2",
63
+ "undici": "^6.22.0",
60
64
  "zod": "^3.24.1"
61
65
  },
62
66
  "devDependencies": {
63
67
  "@eslint/js": "^9.39.2",
64
68
  "@trivago/prettier-plugin-sort-imports": "^6.0.0",
65
69
  "@types/express": "^5.0.6",
66
- "@types/jsdom": "^27.0.0",
67
70
  "@types/node": "^22.19.3",
68
71
  "@types/turndown": "^5.0.6",
69
72
  "eslint": "^9.23.2",
70
73
  "eslint-config-prettier": "^10.1.8",
71
74
  "eslint-plugin-unused-imports": "^4.3.0",
72
- "knip": "^5.75.1",
75
+ "knip": "^5.77.1",
73
76
  "prettier": "^3.7.4",
74
77
  "shx": "^0.4.0",
75
78
  "tsx": "^4.21.0",
76
79
  "typescript": "^5.9.3",
77
- "typescript-eslint": "^8.50.0"
80
+ "typescript-eslint": "^8.50.1",
81
+ "vitest": "^2.1.9",
82
+ "@vitest/coverage-v8": "^2.1.9"
78
83
  },
79
84
  "engines": {
80
85
  "node": ">=20.0.0"