@j0hanz/superfetch 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -32
- package/dist/config/formatting.d.ts +9 -0
- package/dist/config/formatting.d.ts.map +1 -0
- package/dist/config/formatting.js +11 -0
- package/dist/config/formatting.js.map +1 -0
- package/dist/config/index.d.ts +16 -2
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +43 -14
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +107 -0
- package/dist/config/types/content.d.ts.map +1 -0
- package/dist/config/types/content.js +2 -0
- package/dist/config/types/content.js.map +1 -0
- package/dist/config/types/runtime.d.ts +78 -0
- package/dist/config/types/runtime.d.ts.map +1 -0
- package/dist/config/types/runtime.js +2 -0
- package/dist/config/types/runtime.js.map +1 -0
- package/dist/config/types/tools.d.ts +99 -0
- package/dist/config/types/tools.d.ts.map +1 -0
- package/dist/config/types/tools.js +2 -0
- package/dist/config/types/tools.js.map +1 -0
- package/dist/config/types.d.ts +3 -296
- package/dist/config/types.d.ts.map +1 -1
- package/dist/http/auth.d.ts +3 -0
- package/dist/http/auth.d.ts.map +1 -0
- package/dist/http/auth.js +34 -0
- package/dist/http/auth.js.map +1 -0
- package/dist/http/cors.d.ts +8 -0
- package/dist/http/cors.d.ts.map +1 -0
- package/dist/http/cors.js +47 -0
- package/dist/http/cors.js.map +1 -0
- package/dist/http/mcp-routes.d.ts +5 -0
- package/dist/http/mcp-routes.d.ts.map +1 -0
- package/dist/http/mcp-routes.js +110 -0
- package/dist/http/mcp-routes.js.map +1 -0
- package/dist/http/mcp-session.d.ts +12 -0
- package/dist/http/mcp-session.d.ts.map +1 -0
- package/dist/http/mcp-session.js +209 -0
- package/dist/http/mcp-session.js.map +1 -0
- package/dist/http/mcp-validation.d.ts +3 -0
- package/dist/http/mcp-validation.d.ts.map +1 -0
- package/dist/http/mcp-validation.js +34 -0
- package/dist/http/mcp-validation.js.map +1 -0
- package/dist/http/rate-limit.d.ts +13 -0
- package/dist/http/rate-limit.d.ts.map +1 -0
- package/dist/http/rate-limit.js +91 -0
- package/dist/http/rate-limit.js.map +1 -0
- package/dist/http/server.d.ts +4 -0
- package/dist/http/server.d.ts.map +1 -0
- package/dist/http/server.js +183 -0
- package/dist/http/server.js.map +1 -0
- package/dist/http/sessions.d.ts +15 -0
- package/dist/http/sessions.d.ts.map +1 -0
- package/dist/http/sessions.js +64 -0
- package/dist/http/sessions.js.map +1 -0
- package/dist/index.js +26 -223
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +2 -2
- package/dist/middleware/error-handler.d.ts.map +1 -1
- package/dist/middleware/error-handler.js +46 -15
- package/dist/middleware/error-handler.js.map +1 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +104 -44
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts.map +1 -1
- package/dist/resources/index.js +77 -69
- package/dist/resources/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +9 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +13 -1
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +90 -13
- package/dist/services/cache.js.map +1 -1
- package/dist/services/context.d.ts +9 -0
- package/dist/services/context.d.ts.map +1 -0
- package/dist/services/context.js +9 -0
- package/dist/services/context.js.map +1 -0
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +122 -87
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher/agents.d.ts +4 -0
- package/dist/services/fetcher/agents.d.ts.map +1 -0
- package/dist/services/fetcher/agents.js +111 -0
- package/dist/services/fetcher/agents.js.map +1 -0
- package/dist/services/fetcher/errors.d.ts +5 -0
- package/dist/services/fetcher/errors.d.ts.map +1 -0
- package/dist/services/fetcher/errors.js +71 -0
- package/dist/services/fetcher/errors.js.map +1 -0
- package/dist/services/fetcher/headers.d.ts +2 -0
- package/dist/services/fetcher/headers.d.ts.map +1 -0
- package/dist/services/fetcher/headers.js +28 -0
- package/dist/services/fetcher/headers.js.map +1 -0
- package/dist/services/fetcher/interceptors.d.ts +10 -0
- package/dist/services/fetcher/interceptors.d.ts.map +1 -0
- package/dist/services/fetcher/interceptors.js +82 -0
- package/dist/services/fetcher/interceptors.js.map +1 -0
- package/dist/services/fetcher/redirects.d.ts +6 -0
- package/dist/services/fetcher/redirects.d.ts.map +1 -0
- package/dist/services/fetcher/redirects.js +67 -0
- package/dist/services/fetcher/redirects.js.map +1 -0
- package/dist/services/fetcher/response.d.ts +5 -0
- package/dist/services/fetcher/response.d.ts.map +1 -0
- package/dist/services/fetcher/response.js +39 -0
- package/dist/services/fetcher/response.js.map +1 -0
- package/dist/services/fetcher/retry-policy.d.ts +28 -0
- package/dist/services/fetcher/retry-policy.d.ts.map +1 -0
- package/dist/services/fetcher/retry-policy.js +138 -0
- package/dist/services/fetcher/retry-policy.js.map +1 -0
- package/dist/services/fetcher.d.ts +2 -1
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +61 -254
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +14 -5
- package/dist/services/logger.js.map +1 -1
- package/dist/services/parser.d.ts +1 -0
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +55 -35
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts +4 -0
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-links/link-extractor.js +163 -0
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -0
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +78 -116
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +3 -13
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +74 -83
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +26 -0
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-single.shared.js +49 -0
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -0
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +82 -54
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls/processor.d.ts +13 -0
- package/dist/tools/handlers/fetch-urls/processor.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/processor.js +153 -0
- package/dist/tools/handlers/fetch-urls/processor.js.map +1 -0
- package/dist/tools/handlers/fetch-urls/response.d.ts +3 -0
- package/dist/tools/handlers/fetch-urls/response.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/response.js +58 -0
- package/dist/tools/handlers/fetch-urls/response.js.map +1 -0
- package/dist/tools/handlers/fetch-urls/validation.d.ts +6 -0
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -0
- package/dist/tools/handlers/fetch-urls/validation.js +18 -0
- package/dist/tools/handlers/fetch-urls/validation.js.map +1 -0
- package/dist/tools/handlers/fetch-urls.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls.tool.js +104 -197
- package/dist/tools/handlers/fetch-urls.tool.js.map +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +36 -237
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/schemas.d.ts +357 -0
- package/dist/tools/schemas.d.ts.map +1 -0
- package/dist/tools/schemas.js +272 -0
- package/dist/tools/schemas.js.map +1 -0
- package/dist/tools/utils/cache-vary.d.ts +3 -0
- package/dist/tools/utils/cache-vary.d.ts.map +1 -0
- package/dist/tools/utils/cache-vary.js +44 -0
- package/dist/tools/utils/cache-vary.js.map +1 -0
- package/dist/tools/utils/common.d.ts +2 -2
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +5 -1
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +16 -0
- package/dist/tools/utils/content-transform.d.ts.map +1 -0
- package/dist/tools/utils/content-transform.js +49 -0
- package/dist/tools/utils/content-transform.js.map +1 -0
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +32 -18
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/tools/utils/inline-content.d.ts +11 -0
- package/dist/tools/utils/inline-content.d.ts.map +1 -0
- package/dist/tools/utils/inline-content.js +39 -0
- package/dist/tools/utils/inline-content.js.map +1 -0
- package/dist/tools/utils/markdown-toc.d.ts +3 -0
- package/dist/tools/utils/markdown-toc.d.ts.map +1 -0
- package/dist/tools/utils/markdown-toc.js +35 -0
- package/dist/tools/utils/markdown-toc.js.map +1 -0
- package/dist/tools/utils/tool-response.d.ts +9 -0
- package/dist/tools/utils/tool-response.d.ts.map +1 -0
- package/dist/tools/utils/tool-response.js +19 -0
- package/dist/tools/utils/tool-response.js.map +1 -0
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +51 -28
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +82 -111
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/header-normalizer.d.ts +5 -0
- package/dist/utils/header-normalizer.d.ts.map +1 -0
- package/dist/utils/header-normalizer.js +25 -0
- package/dist/utils/header-normalizer.js.map +1 -0
- package/dist/utils/tool-error-handler.d.ts +1 -0
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +29 -1
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-validator.d.ts +0 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +98 -18
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +11 -6
|
@@ -1,57 +1,137 @@
|
|
|
1
|
+
import { BlockList, isIP } from 'node:net';
|
|
1
2
|
import { config } from '../config/index.js';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
const BLOCK_LIST = new BlockList();
|
|
4
|
+
const BLOCKED_IPV4_SUBNETS = [
|
|
5
|
+
{ subnet: '0.0.0.0', prefix: 8 },
|
|
6
|
+
{ subnet: '10.0.0.0', prefix: 8 },
|
|
7
|
+
{ subnet: '100.64.0.0', prefix: 10 },
|
|
8
|
+
{ subnet: '127.0.0.0', prefix: 8 },
|
|
9
|
+
{ subnet: '169.254.0.0', prefix: 16 },
|
|
10
|
+
{ subnet: '172.16.0.0', prefix: 12 },
|
|
11
|
+
{ subnet: '192.168.0.0', prefix: 16 },
|
|
12
|
+
{ subnet: '224.0.0.0', prefix: 4 },
|
|
13
|
+
{ subnet: '240.0.0.0', prefix: 4 },
|
|
14
|
+
];
|
|
15
|
+
const BLOCKED_IPV6_SUBNETS = [
|
|
16
|
+
{ subnet: '::', prefix: 128 },
|
|
17
|
+
{ subnet: '::1', prefix: 128 },
|
|
18
|
+
{ subnet: '::ffff:0:0', prefix: 96 },
|
|
19
|
+
{ subnet: '64:ff9b::', prefix: 96 },
|
|
20
|
+
{ subnet: '64:ff9b:1::', prefix: 48 },
|
|
21
|
+
{ subnet: '2001::', prefix: 32 },
|
|
22
|
+
{ subnet: '2002::', prefix: 16 },
|
|
23
|
+
{ subnet: 'fc00::', prefix: 7 },
|
|
24
|
+
{ subnet: 'fe80::', prefix: 10 },
|
|
25
|
+
{ subnet: 'ff00::', prefix: 8 },
|
|
26
|
+
];
|
|
27
|
+
for (const entry of BLOCKED_IPV4_SUBNETS) {
|
|
28
|
+
BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv4');
|
|
29
|
+
}
|
|
30
|
+
for (const entry of BLOCKED_IPV6_SUBNETS) {
|
|
31
|
+
BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv6');
|
|
32
|
+
}
|
|
33
|
+
function matchesBlockedIpPatterns(resolvedIp) {
|
|
34
|
+
for (const pattern of config.security.blockedIpPatterns) {
|
|
35
|
+
if (pattern.test(resolvedIp)) {
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
5
41
|
export function isBlockedIp(ip) {
|
|
6
|
-
|
|
42
|
+
if (config.security.blockedHosts.has(ip)) {
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
const ipType = resolveIpType(ip);
|
|
46
|
+
if (!ipType)
|
|
47
|
+
return false;
|
|
48
|
+
const normalizedIp = ip.toLowerCase();
|
|
49
|
+
if (isBlockedByList(normalizedIp, ipType))
|
|
50
|
+
return true;
|
|
51
|
+
return matchesBlockedIpPatterns(normalizedIp);
|
|
7
52
|
}
|
|
8
|
-
|
|
53
|
+
function resolveIpType(ip) {
|
|
54
|
+
const ipType = isIP(ip);
|
|
55
|
+
return ipType === 4 || ipType === 6 ? ipType : null;
|
|
56
|
+
}
|
|
57
|
+
function isBlockedByList(ip, ipType) {
|
|
58
|
+
if (ipType === 4) {
|
|
59
|
+
return BLOCK_LIST.check(ip, 'ipv4');
|
|
60
|
+
}
|
|
61
|
+
return BLOCK_LIST.check(ip, 'ipv6');
|
|
62
|
+
}
|
|
63
|
+
function assertUrlProvided(urlString) {
|
|
9
64
|
if (!urlString || typeof urlString !== 'string') {
|
|
10
65
|
throw new Error('URL is required');
|
|
11
66
|
}
|
|
12
|
-
|
|
67
|
+
}
|
|
68
|
+
function assertUrlNotEmpty(trimmedUrl) {
|
|
13
69
|
if (!trimmedUrl) {
|
|
14
70
|
throw new Error('URL cannot be empty');
|
|
15
71
|
}
|
|
72
|
+
}
|
|
73
|
+
function assertUrlLength(trimmedUrl) {
|
|
16
74
|
if (trimmedUrl.length > config.constants.maxUrlLength) {
|
|
17
75
|
throw new Error(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`);
|
|
18
76
|
}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
}
|
|
23
|
-
catch {
|
|
77
|
+
}
|
|
78
|
+
function parseUrl(trimmedUrl) {
|
|
79
|
+
if (!URL.canParse(trimmedUrl)) {
|
|
24
80
|
throw new Error('Invalid URL format');
|
|
25
81
|
}
|
|
82
|
+
return new URL(trimmedUrl);
|
|
83
|
+
}
|
|
84
|
+
function assertProtocolAllowed(url) {
|
|
26
85
|
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
27
86
|
throw new Error(`Invalid protocol: ${url.protocol}. Only http: and https: are allowed`);
|
|
28
87
|
}
|
|
88
|
+
}
|
|
89
|
+
function assertNoCredentials(url) {
|
|
29
90
|
if (url.username || url.password) {
|
|
30
91
|
throw new Error('URLs with embedded credentials are not allowed');
|
|
31
92
|
}
|
|
32
|
-
|
|
93
|
+
}
|
|
94
|
+
function assertHostnamePresent(hostname) {
|
|
33
95
|
if (!hostname) {
|
|
34
96
|
throw new Error('URL must have a valid hostname');
|
|
35
97
|
}
|
|
98
|
+
}
|
|
99
|
+
function assertHostnameAllowed(hostname) {
|
|
36
100
|
if (config.security.blockedHosts.has(hostname)) {
|
|
37
101
|
throw new Error(`Blocked host: ${hostname}. Internal hosts are not allowed`);
|
|
38
102
|
}
|
|
103
|
+
}
|
|
104
|
+
function assertHostnameNotIpBlocked(hostname) {
|
|
39
105
|
if (isBlockedIp(hostname)) {
|
|
40
106
|
throw new Error(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
|
|
41
107
|
}
|
|
108
|
+
}
|
|
109
|
+
function assertHostnameSuffixAllowed(hostname) {
|
|
42
110
|
if (hostname.endsWith('.local') || hostname.endsWith('.internal')) {
|
|
43
111
|
throw new Error(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
|
|
44
112
|
}
|
|
113
|
+
}
|
|
114
|
+
export function validateAndNormalizeUrl(urlString) {
|
|
115
|
+
assertUrlProvided(urlString);
|
|
116
|
+
const trimmedUrl = urlString.trim();
|
|
117
|
+
assertUrlNotEmpty(trimmedUrl);
|
|
118
|
+
assertUrlLength(trimmedUrl);
|
|
119
|
+
const url = parseUrl(trimmedUrl);
|
|
120
|
+
assertProtocolAllowed(url);
|
|
121
|
+
assertNoCredentials(url);
|
|
122
|
+
const hostname = url.hostname.toLowerCase();
|
|
123
|
+
assertHostnamePresent(hostname);
|
|
124
|
+
assertHostnameAllowed(hostname);
|
|
125
|
+
assertHostnameNotIpBlocked(hostname);
|
|
126
|
+
assertHostnameSuffixAllowed(hostname);
|
|
45
127
|
return url.href;
|
|
46
128
|
}
|
|
47
129
|
export function isInternalUrl(url, baseUrl) {
|
|
48
|
-
|
|
49
|
-
const urlObj = new URL(url, baseUrl);
|
|
50
|
-
const baseUrlObj = new URL(baseUrl);
|
|
51
|
-
return urlObj.hostname === baseUrlObj.hostname;
|
|
52
|
-
}
|
|
53
|
-
catch {
|
|
130
|
+
if (!URL.canParse(baseUrl) || !URL.canParse(url, baseUrl)) {
|
|
54
131
|
return false;
|
|
55
132
|
}
|
|
133
|
+
const urlObj = new URL(url, baseUrl);
|
|
134
|
+
const baseUrlObj = new URL(baseUrl);
|
|
135
|
+
return urlObj.hostname === baseUrlObj.hostname;
|
|
56
136
|
}
|
|
57
137
|
//# sourceMappingURL=url-validator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C
|
|
1
|
+
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;AACnC,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE;IAChC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;CAC1B,CAAC;AACX,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7B,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE;IAC9B,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;IACnC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;IAC/B,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;CACvB,CAAC;AAEX,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AACD,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAkB;IAClD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,OAAO,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,EAAU,EAAE,MAAa;IAChD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,UAAkB;IAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,UAAkB;IACzC,IAAI,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACb,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,UAAkB;IAClC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAQ;IACrC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAQ;IACnC,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CACb,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,0BAA0B,CAAC,QAAgB;IAClD,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAAC,QAAgB;IACnD,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAE7B,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAC9B,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjC,qBAAqB,CAAC,GAAG,CAAC,CAAC;IAC3B,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC5C,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAChC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAChC,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACrC,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAEtC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe;IACxD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;QAC1D,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,CAAC;AACjD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@j0hanz/superfetch",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.3",
|
|
4
4
|
"mcpName": "io.github.j0hanz/superfetch",
|
|
5
5
|
"description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
|
|
6
6
|
"type": "module",
|
|
@@ -44,37 +44,42 @@
|
|
|
44
44
|
"type-check": "tsc --noEmit",
|
|
45
45
|
"lint": "eslint .",
|
|
46
46
|
"lint:fix": "eslint . --fix",
|
|
47
|
+
"test": "vitest run",
|
|
48
|
+
"test:coverage": "vitest run --coverage",
|
|
49
|
+
"bench": "npm run build && node scripts/bench.mjs",
|
|
47
50
|
"knip": "knip",
|
|
48
51
|
"knip:fix": "knip --fix"
|
|
49
52
|
},
|
|
50
53
|
"dependencies": {
|
|
51
54
|
"@modelcontextprotocol/sdk": "^1.25.1",
|
|
52
55
|
"@mozilla/readability": "^0.6.0",
|
|
53
|
-
"axios": "^1.7.9",
|
|
54
56
|
"cheerio": "^1.1.2",
|
|
55
57
|
"domhandler": "^5.0.3",
|
|
56
58
|
"express": "^5.2.1",
|
|
57
|
-
"
|
|
59
|
+
"linkedom": "^0.18.12",
|
|
58
60
|
"node-cache": "^5.1.2",
|
|
61
|
+
"safe-regex": "^2.1.1",
|
|
59
62
|
"turndown": "^7.2.2",
|
|
63
|
+
"undici": "^6.22.0",
|
|
60
64
|
"zod": "^3.24.1"
|
|
61
65
|
},
|
|
62
66
|
"devDependencies": {
|
|
63
67
|
"@eslint/js": "^9.39.2",
|
|
64
68
|
"@trivago/prettier-plugin-sort-imports": "^6.0.0",
|
|
65
69
|
"@types/express": "^5.0.6",
|
|
66
|
-
"@types/jsdom": "^27.0.0",
|
|
67
70
|
"@types/node": "^22.19.3",
|
|
68
71
|
"@types/turndown": "^5.0.6",
|
|
69
72
|
"eslint": "^9.23.2",
|
|
70
73
|
"eslint-config-prettier": "^10.1.8",
|
|
71
74
|
"eslint-plugin-unused-imports": "^4.3.0",
|
|
72
|
-
"knip": "^5.
|
|
75
|
+
"knip": "^5.77.1",
|
|
73
76
|
"prettier": "^3.7.4",
|
|
74
77
|
"shx": "^0.4.0",
|
|
75
78
|
"tsx": "^4.21.0",
|
|
76
79
|
"typescript": "^5.9.3",
|
|
77
|
-
"typescript-eslint": "^8.50.
|
|
80
|
+
"typescript-eslint": "^8.50.1",
|
|
81
|
+
"vitest": "^2.1.9",
|
|
82
|
+
"@vitest/coverage-v8": "^2.1.9"
|
|
78
83
|
},
|
|
79
84
|
"engines": {
|
|
80
85
|
"node": ">=20.0.0"
|