@j0hanz/superfetch 1.1.9 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +179 -469
- package/dist/config/constants.d.ts +19 -0
- package/dist/config/constants.d.ts.map +1 -0
- package/dist/config/constants.js +24 -0
- package/dist/config/constants.js.map +1 -0
- package/dist/config/formatting.d.ts +0 -2
- package/dist/config/formatting.d.ts.map +1 -1
- package/dist/config/formatting.js +1 -3
- package/dist/config/formatting.js.map +1 -1
- package/dist/config/index.d.ts +9 -3
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +19 -16
- package/dist/config/index.js.map +1 -1
- package/dist/config/types/content.d.ts +1 -20
- package/dist/config/types/content.d.ts.map +1 -1
- package/dist/config/types/content.js +0 -1
- package/dist/config/types/runtime.d.ts +7 -5
- package/dist/config/types/runtime.d.ts.map +1 -1
- package/dist/config/types/runtime.js +0 -1
- package/dist/config/types/tools.d.ts +5 -50
- package/dist/config/types/tools.d.ts.map +1 -1
- package/dist/config/types/tools.js +0 -1
- package/dist/errors/app-error.d.ts +0 -1
- package/dist/errors/app-error.js +0 -1
- package/dist/http/auth.d.ts +0 -1
- package/dist/http/auth.d.ts.map +1 -1
- package/dist/http/auth.js +17 -13
- package/dist/http/auth.js.map +1 -1
- package/dist/http/cors.d.ts +0 -1
- package/dist/http/cors.js +4 -1
- package/dist/http/cors.js.map +1 -1
- package/dist/http/download-routes.d.ts +14 -0
- package/dist/http/download-routes.d.ts.map +1 -0
- package/dist/http/download-routes.js +131 -0
- package/dist/http/download-routes.js.map +1 -0
- package/dist/http/mcp-routes.d.ts +1 -2
- package/dist/http/mcp-routes.d.ts.map +1 -1
- package/dist/http/mcp-routes.js +1 -2
- package/dist/http/mcp-routes.js.map +1 -1
- package/dist/http/mcp-session-helpers.d.ts +13 -0
- package/dist/http/mcp-session-helpers.d.ts.map +1 -0
- package/dist/http/mcp-session-helpers.js +64 -0
- package/dist/http/mcp-session-helpers.js.map +1 -0
- package/dist/http/mcp-session.d.ts +1 -3
- package/dist/http/mcp-session.d.ts.map +1 -1
- package/dist/http/mcp-session.js +7 -71
- package/dist/http/mcp-session.js.map +1 -1
- package/dist/http/mcp-validation.d.ts +1 -2
- package/dist/http/mcp-validation.d.ts.map +1 -1
- package/dist/http/mcp-validation.js +6 -27
- package/dist/http/mcp-validation.js.map +1 -1
- package/dist/http/rate-limit.d.ts +1 -2
- package/dist/http/rate-limit.d.ts.map +1 -1
- package/dist/http/rate-limit.js +0 -1
- package/dist/http/rate-limit.js.map +1 -1
- package/dist/http/server-middleware.d.ts +9 -0
- package/dist/http/server-middleware.d.ts.map +1 -0
- package/dist/http/server-middleware.js +111 -0
- package/dist/http/server-middleware.js.map +1 -0
- package/dist/http/server.d.ts +0 -1
- package/dist/http/server.d.ts.map +1 -1
- package/dist/http/server.js +20 -99
- package/dist/http/server.js.map +1 -1
- package/dist/http/session-cleanup.d.ts +2 -0
- package/dist/http/session-cleanup.d.ts.map +1 -0
- package/dist/http/session-cleanup.js +37 -0
- package/dist/http/session-cleanup.js.map +1 -0
- package/dist/http/sessions.d.ts +1 -2
- package/dist/http/sessions.d.ts.map +1 -1
- package/dist/http/sessions.js +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +13 -6
- package/dist/index.js.map +1 -1
- package/dist/middleware/error-handler.d.ts +0 -1
- package/dist/middleware/error-handler.js +0 -1
- package/dist/resources/cached-content.d.ts +0 -1
- package/dist/resources/cached-content.d.ts.map +1 -1
- package/dist/resources/cached-content.js +76 -12
- package/dist/resources/cached-content.js.map +1 -1
- package/dist/resources/index.d.ts +0 -1
- package/dist/resources/index.js +0 -1
- package/dist/server.d.ts +0 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +8 -3
- package/dist/server.js.map +1 -1
- package/dist/services/cache.d.ts +7 -4
- package/dist/services/cache.d.ts.map +1 -1
- package/dist/services/cache.js +86 -26
- package/dist/services/cache.js.map +1 -1
- package/dist/services/context.d.ts +2 -2
- package/dist/services/context.d.ts.map +1 -1
- package/dist/services/context.js +0 -1
- package/dist/services/extractor.d.ts +1 -2
- package/dist/services/extractor.d.ts.map +1 -1
- package/dist/services/extractor.js +45 -18
- package/dist/services/extractor.js.map +1 -1
- package/dist/services/fetcher/agents.d.ts +0 -1
- package/dist/services/fetcher/agents.d.ts.map +1 -1
- package/dist/services/fetcher/agents.js +3 -7
- package/dist/services/fetcher/agents.js.map +1 -1
- package/dist/services/fetcher/errors.d.ts +0 -1
- package/dist/services/fetcher/errors.js +0 -1
- package/dist/services/fetcher/headers.d.ts.map +1 -1
- package/dist/services/fetcher/headers.js +2 -24
- package/dist/services/fetcher/headers.js.map +1 -1
- package/dist/services/fetcher/interceptors.d.ts +2 -2
- package/dist/services/fetcher/interceptors.d.ts.map +1 -1
- package/dist/services/fetcher/interceptors.js +30 -21
- package/dist/services/fetcher/interceptors.js.map +1 -1
- package/dist/services/fetcher/redirects.d.ts +0 -2
- package/dist/services/fetcher/redirects.d.ts.map +1 -1
- package/dist/services/fetcher/redirects.js +20 -18
- package/dist/services/fetcher/redirects.js.map +1 -1
- package/dist/services/fetcher/response.d.ts +0 -1
- package/dist/services/fetcher/response.js +4 -5
- package/dist/services/fetcher/retry-policy.d.ts +1 -28
- package/dist/services/fetcher/retry-policy.d.ts.map +1 -1
- package/dist/services/fetcher/retry-policy.js +119 -126
- package/dist/services/fetcher/retry-policy.js.map +1 -1
- package/dist/services/fetcher.d.ts +1 -2
- package/dist/services/fetcher.d.ts.map +1 -1
- package/dist/services/fetcher.js +18 -13
- package/dist/services/fetcher.js.map +1 -1
- package/dist/services/logger.d.ts +1 -2
- package/dist/services/logger.d.ts.map +1 -1
- package/dist/services/logger.js +0 -1
- package/dist/services/parser.d.ts +1 -3
- package/dist/services/parser.d.ts.map +1 -1
- package/dist/services/parser.js +5 -39
- package/dist/services/parser.js.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links/link-extractor.js +15 -19
- package/dist/tools/handlers/fetch-links/link-extractor.js.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-links.tool.js +0 -2
- package/dist/tools/handlers/fetch-links.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-markdown.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-markdown.tool.js +50 -20
- package/dist/tools/handlers/fetch-markdown.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.d.ts +14 -3
- package/dist/tools/handlers/fetch-single.shared.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-single.shared.js +66 -3
- package/dist/tools/handlers/fetch-single.shared.js.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.d.ts +1 -2
- package/dist/tools/handlers/fetch-url.tool.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-url.tool.js +39 -17
- package/dist/tools/handlers/fetch-url.tool.js.map +1 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts +0 -1
- package/dist/tools/handlers/fetch-urls/validation.d.ts.map +1 -1
- package/dist/tools/handlers/fetch-urls/validation.js +1 -1
- package/dist/tools/handlers/fetch-urls/validation.js.map +1 -1
- package/dist/tools/index.d.ts +0 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +1 -20
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/schemas.d.ts +57 -250
- package/dist/tools/schemas.d.ts.map +1 -1
- package/dist/tools/schemas.js +38 -198
- package/dist/tools/schemas.js.map +1 -1
- package/dist/tools/utils/cache-vary.d.ts +0 -2
- package/dist/tools/utils/cache-vary.d.ts.map +1 -1
- package/dist/tools/utils/cache-vary.js +8 -40
- package/dist/tools/utils/cache-vary.js.map +1 -1
- package/dist/tools/utils/common.d.ts +2 -4
- package/dist/tools/utils/common.d.ts.map +1 -1
- package/dist/tools/utils/common.js +6 -7
- package/dist/tools/utils/common.js.map +1 -1
- package/dist/tools/utils/content-transform.d.ts +1 -3
- package/dist/tools/utils/content-transform.d.ts.map +1 -1
- package/dist/tools/utils/content-transform.js +65 -14
- package/dist/tools/utils/content-transform.js.map +1 -1
- package/dist/tools/utils/fetch-pipeline.d.ts +1 -2
- package/dist/tools/utils/fetch-pipeline.d.ts.map +1 -1
- package/dist/tools/utils/fetch-pipeline.js +25 -21
- package/dist/tools/utils/fetch-pipeline.js.map +1 -1
- package/dist/tools/utils/inline-content.d.ts +3 -3
- package/dist/tools/utils/inline-content.d.ts.map +1 -1
- package/dist/tools/utils/inline-content.js +0 -1
- package/dist/transformers/jsonl.transformer.d.ts +1 -2
- package/dist/transformers/jsonl.transformer.d.ts.map +1 -1
- package/dist/transformers/jsonl.transformer.js +0 -1
- package/dist/transformers/jsonl.transformer.js.map +1 -1
- package/dist/transformers/markdown.transformer.d.ts +1 -2
- package/dist/transformers/markdown.transformer.d.ts.map +1 -1
- package/dist/transformers/markdown.transformer.js +11 -7
- package/dist/transformers/markdown.transformer.js.map +1 -1
- package/dist/utils/code-language.d.ts +2 -0
- package/dist/utils/code-language.d.ts.map +1 -0
- package/dist/utils/code-language.js +56 -0
- package/dist/utils/code-language.js.map +1 -0
- package/dist/utils/content-cleaner.d.ts +0 -2
- package/dist/utils/content-cleaner.d.ts.map +1 -1
- package/dist/utils/content-cleaner.js +0 -4
- package/dist/utils/content-cleaner.js.map +1 -1
- package/dist/utils/crypto.d.ts +2 -0
- package/dist/utils/crypto.d.ts.map +1 -0
- package/dist/utils/crypto.js +32 -0
- package/dist/utils/crypto.js.map +1 -0
- package/dist/utils/download-url.d.ts +8 -0
- package/dist/utils/download-url.d.ts.map +1 -0
- package/dist/utils/download-url.js +27 -0
- package/dist/utils/download-url.js.map +1 -0
- package/dist/utils/error-utils.d.ts +3 -0
- package/dist/utils/error-utils.d.ts.map +1 -0
- package/dist/utils/error-utils.js +12 -0
- package/dist/utils/error-utils.js.map +1 -0
- package/dist/utils/filename-generator.d.ts +1 -0
- package/dist/utils/filename-generator.d.ts.map +1 -0
- package/dist/utils/filename-generator.js +59 -0
- package/dist/utils/filename-generator.js.map +1 -0
- package/dist/utils/header-normalizer.d.ts +7 -4
- package/dist/utils/header-normalizer.d.ts.map +1 -1
- package/dist/utils/header-normalizer.js +23 -17
- package/dist/utils/header-normalizer.js.map +1 -1
- package/dist/utils/html-truncator.d.ts +0 -1
- package/dist/utils/html-truncator.js +0 -1
- package/dist/utils/sanitizer.d.ts +0 -1
- package/dist/utils/sanitizer.js +0 -1
- package/dist/utils/tool-error-handler.d.ts +1 -3
- package/dist/utils/tool-error-handler.d.ts.map +1 -1
- package/dist/utils/tool-error-handler.js +11 -6
- package/dist/utils/tool-error-handler.js.map +1 -1
- package/dist/utils/url-sanitizer.d.ts +2 -0
- package/dist/utils/url-sanitizer.d.ts.map +1 -0
- package/dist/utils/url-sanitizer.js +12 -0
- package/dist/utils/url-sanitizer.js.map +1 -0
- package/dist/utils/url-validator.d.ts +1 -3
- package/dist/utils/url-validator.d.ts.map +1 -1
- package/dist/utils/url-validator.js +89 -53
- package/dist/utils/url-validator.js.map +1 -1
- package/package.json +7 -9
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { lookup } from 'node:dns/promises';
|
|
1
2
|
import { BlockList, isIP } from 'node:net';
|
|
2
3
|
import { config } from '../config/index.js';
|
|
4
|
+
import { createErrorWithCode } from './error-utils.js';
|
|
3
5
|
const BLOCK_LIST = new BlockList();
|
|
4
6
|
const BLOCKED_IPV4_SUBNETS = [
|
|
5
7
|
{ subnet: '0.0.0.0', prefix: 8 },
|
|
@@ -29,6 +31,7 @@ for (const entry of BLOCKED_IPV4_SUBNETS) {
|
|
|
29
31
|
for (const entry of BLOCKED_IPV6_SUBNETS) {
|
|
30
32
|
BLOCK_LIST.addSubnet(entry.subnet, entry.prefix, 'ipv6');
|
|
31
33
|
}
|
|
34
|
+
const DNS_LOOKUP_TIMEOUT_MS = 5000;
|
|
32
35
|
function matchesBlockedIpPatterns(resolvedIp) {
|
|
33
36
|
for (const pattern of config.security.blockedIpPatterns) {
|
|
34
37
|
if (pattern.test(resolvedIp)) {
|
|
@@ -59,78 +62,111 @@ function isBlockedByList(ip, ipType) {
|
|
|
59
62
|
}
|
|
60
63
|
return BLOCK_LIST.check(ip, 'ipv6');
|
|
61
64
|
}
|
|
62
|
-
function
|
|
63
|
-
|
|
64
|
-
|
|
65
|
+
function lookupWithTimeout(hostname) {
|
|
66
|
+
return new Promise((resolve, reject) => {
|
|
67
|
+
const timer = setTimeout(() => {
|
|
68
|
+
reject(createValidationError(`DNS lookup timed out for ${hostname}`));
|
|
69
|
+
}, DNS_LOOKUP_TIMEOUT_MS);
|
|
70
|
+
lookup(hostname, { all: true })
|
|
71
|
+
.then((result) => {
|
|
72
|
+
clearTimeout(timer);
|
|
73
|
+
resolve(result);
|
|
74
|
+
})
|
|
75
|
+
.catch((error) => {
|
|
76
|
+
clearTimeout(timer);
|
|
77
|
+
reject(error instanceof Error ? error : createValidationError(String(error)));
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
async function assertResolvedAddressesAllowed(hostname) {
|
|
82
|
+
try {
|
|
83
|
+
const result = await lookupWithTimeout(hostname);
|
|
84
|
+
const addresses = Array.isArray(result) ? result : [result];
|
|
85
|
+
if (addresses.length === 0) {
|
|
86
|
+
throw createValidationError(`Unable to resolve hostname: ${hostname}`);
|
|
87
|
+
}
|
|
88
|
+
for (const { address } of addresses) {
|
|
89
|
+
if (isBlockedIp(address.toLowerCase())) {
|
|
90
|
+
throw createValidationError(`Blocked IP range resolved from hostname: ${hostname}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
const code = error?.code;
|
|
96
|
+
if (code === 'ENOTFOUND' || code === 'EAI_AGAIN') {
|
|
97
|
+
throw createValidationError(`Unable to resolve hostname: ${hostname}`);
|
|
98
|
+
}
|
|
99
|
+
if (error instanceof Error) {
|
|
100
|
+
throw error;
|
|
101
|
+
}
|
|
102
|
+
throw createValidationError(String(error));
|
|
65
103
|
}
|
|
66
104
|
}
|
|
67
|
-
function
|
|
105
|
+
export async function validateAndNormalizeUrl(urlString) {
|
|
106
|
+
const trimmedUrl = requireTrimmedUrl(urlString);
|
|
107
|
+
assertUrlLength(trimmedUrl);
|
|
108
|
+
const url = parseUrl(trimmedUrl);
|
|
109
|
+
assertHttpProtocol(url);
|
|
110
|
+
assertNoCredentials(url);
|
|
111
|
+
const hostname = normalizeHostname(url);
|
|
112
|
+
assertHostnameAllowed(hostname);
|
|
113
|
+
await assertResolvedAddressesAllowed(hostname);
|
|
114
|
+
return url.href;
|
|
115
|
+
}
|
|
116
|
+
const VALIDATION_ERROR_CODE = 'VALIDATION_ERROR';
|
|
117
|
+
function createValidationError(message) {
|
|
118
|
+
return createErrorWithCode(message, VALIDATION_ERROR_CODE);
|
|
119
|
+
}
|
|
120
|
+
function requireTrimmedUrl(urlString) {
|
|
121
|
+
if (!urlString || typeof urlString !== 'string') {
|
|
122
|
+
throw createValidationError('URL is required');
|
|
123
|
+
}
|
|
124
|
+
const trimmedUrl = urlString.trim();
|
|
68
125
|
if (!trimmedUrl) {
|
|
69
|
-
throw
|
|
126
|
+
throw createValidationError('URL cannot be empty');
|
|
70
127
|
}
|
|
128
|
+
return trimmedUrl;
|
|
71
129
|
}
|
|
72
|
-
function assertUrlLength(
|
|
73
|
-
if (
|
|
74
|
-
|
|
75
|
-
}
|
|
130
|
+
function assertUrlLength(url) {
|
|
131
|
+
if (url.length <= config.constants.maxUrlLength)
|
|
132
|
+
return;
|
|
133
|
+
throw createValidationError(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`);
|
|
76
134
|
}
|
|
77
|
-
function parseUrl(
|
|
78
|
-
if (!URL.canParse(
|
|
79
|
-
throw
|
|
135
|
+
function parseUrl(urlString) {
|
|
136
|
+
if (!URL.canParse(urlString)) {
|
|
137
|
+
throw createValidationError('Invalid URL format');
|
|
80
138
|
}
|
|
81
|
-
return new URL(
|
|
139
|
+
return new URL(urlString);
|
|
82
140
|
}
|
|
83
|
-
function
|
|
84
|
-
if (url.protocol
|
|
85
|
-
|
|
86
|
-
}
|
|
141
|
+
function assertHttpProtocol(url) {
|
|
142
|
+
if (url.protocol === 'http:' || url.protocol === 'https:')
|
|
143
|
+
return;
|
|
144
|
+
throw createValidationError(`Invalid protocol: ${url.protocol}. Only http: and https: are allowed`);
|
|
87
145
|
}
|
|
88
146
|
function assertNoCredentials(url) {
|
|
89
|
-
if (url.username
|
|
90
|
-
|
|
91
|
-
|
|
147
|
+
if (!url.username && !url.password)
|
|
148
|
+
return;
|
|
149
|
+
throw createValidationError('URLs with embedded credentials are not allowed');
|
|
92
150
|
}
|
|
93
|
-
function
|
|
151
|
+
function normalizeHostname(url) {
|
|
152
|
+
const hostname = url.hostname.toLowerCase();
|
|
94
153
|
if (!hostname) {
|
|
95
|
-
throw
|
|
154
|
+
throw createValidationError('URL must have a valid hostname');
|
|
96
155
|
}
|
|
156
|
+
return hostname;
|
|
97
157
|
}
|
|
158
|
+
const BLOCKED_HOST_SUFFIXES = ['.local', '.internal'];
|
|
98
159
|
function assertHostnameAllowed(hostname) {
|
|
99
160
|
if (config.security.blockedHosts.has(hostname)) {
|
|
100
|
-
throw
|
|
161
|
+
throw createValidationError(`Blocked host: ${hostname}. Internal hosts are not allowed`);
|
|
101
162
|
}
|
|
102
|
-
}
|
|
103
|
-
function assertHostnameNotIpBlocked(hostname) {
|
|
104
163
|
if (isBlockedIp(hostname)) {
|
|
105
|
-
throw
|
|
164
|
+
throw createValidationError(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
|
|
106
165
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
if (hostname.endsWith('.local') || hostname.endsWith('.internal')) {
|
|
110
|
-
throw new Error(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
|
|
166
|
+
if (matchesBlockedSuffix(hostname)) {
|
|
167
|
+
throw createValidationError(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
|
|
111
168
|
}
|
|
112
169
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
const trimmedUrl = urlString.trim();
|
|
116
|
-
assertUrlNotEmpty(trimmedUrl);
|
|
117
|
-
assertUrlLength(trimmedUrl);
|
|
118
|
-
const url = parseUrl(trimmedUrl);
|
|
119
|
-
assertProtocolAllowed(url);
|
|
120
|
-
assertNoCredentials(url);
|
|
121
|
-
const hostname = url.hostname.toLowerCase();
|
|
122
|
-
assertHostnamePresent(hostname);
|
|
123
|
-
assertHostnameAllowed(hostname);
|
|
124
|
-
assertHostnameNotIpBlocked(hostname);
|
|
125
|
-
assertHostnameSuffixAllowed(hostname);
|
|
126
|
-
return url.href;
|
|
127
|
-
}
|
|
128
|
-
export function isInternalUrl(url, baseUrl) {
|
|
129
|
-
if (!URL.canParse(baseUrl) || !URL.canParse(url, baseUrl)) {
|
|
130
|
-
return false;
|
|
131
|
-
}
|
|
132
|
-
const urlObj = new URL(url, baseUrl);
|
|
133
|
-
const baseUrlObj = new URL(baseUrl);
|
|
134
|
-
return urlObj.hostname === baseUrlObj.hostname;
|
|
170
|
+
function matchesBlockedSuffix(hostname) {
|
|
171
|
+
return BLOCKED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix));
|
|
135
172
|
}
|
|
136
|
-
//# sourceMappingURL=url-validator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,MAAM,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;AACnC,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE;IAChC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;CAC1B,CAAC;AACX,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7B,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE;IAC9B,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;IACnC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;IAC/B,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;CACvB,CAAC;AAEX,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AACD,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAkB;IAClD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,OAAO,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,EAAU,EAAE,MAAa;IAChD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,
|
|
1
|
+
{"version":3,"file":"url-validator.js","sourceRoot":"","sources":["../../src/utils/url-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAEvD,MAAM,UAAU,GAAG,IAAI,SAAS,EAAE,CAAC;AACnC,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE;IAChC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;IACjC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE;IACpC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;IAClC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,EAAE;CAC1B,CAAC;AACX,MAAM,oBAAoB,GAAG;IAC3B,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IAC7B,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE;IAC9B,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;IACnC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE;IACrC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;IAC/B,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE;IAChC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE;CACvB,CAAC;AAEX,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AACD,KAAK,MAAM,KAAK,IAAI,oBAAoB,EAAE,CAAC;IACzC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,wBAAwB,CAAC,UAAkB;IAClD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAU;IACpC,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,YAAY,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,OAAO,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACxB,OAAO,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED,SAAS,eAAe,CAAC,EAAU,EAAE,MAAa;IAChD,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,SAAiB;IACvD,MAAM,UAAU,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAChD,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACxB,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACxC,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAEhC,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,kBAAkB,CAAC;AAEjD,SAAS,qBAAqB,CAAC,OAAe;IAC5C,OAAO,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,qBAAqB,CAAC,iBAAiB,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,qBAAqB,CAAC,qBAAqB,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY;QAAE,OAAO;IACxD,MAAM,qBAAqB,CACzB,iCAAiC,MAAM,CAAC,SAAS,CAAC,YAAY,aAAa,CAC5E,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,SAAiB;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,MAAM,qBAAqB,CAAC,oBAAoB,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAQ;IAClC,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO;IAClE,MAAM,qBAAqB,CACzB,qBAAqB,GAAG,CAAC,QAAQ,qCAAqC,CACvE,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAQ;IACnC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,OAAO;IAC3C,MAAM,qBAAqB,CAAC,gDAAgD,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAQ;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,qBAAqB,CAAC,gCAAgC,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,qBAAqB,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAU,CAAC;AAE/D,SAAS,qBAAqB,CAAC,QAAgB;IAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,qBAAqB,CACzB,iBAAiB,QAAQ,kCAAkC,CAC5D,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,qBAAqB,CACzB,qBAAqB,QAAQ,+BAA+B,CAC7D,CAAC;IACJ,CAAC;IAED,IAAI,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,MAAM,qBAAqB,CACzB,6BAA6B,QAAQ,4CAA4C,CAClF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AAC3E,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@j0hanz/superfetch",
|
|
3
|
-
"version": "1.1
|
|
3
|
+
"version": "1.2.1",
|
|
4
4
|
"mcpName": "io.github.j0hanz/superfetch",
|
|
5
5
|
"description": "Intelligent web content fetcher MCP server that converts HTML to clean, AI-readable JSONL format",
|
|
6
6
|
"type": "module",
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
],
|
|
36
36
|
"scripts": {
|
|
37
37
|
"dev": "tsx watch src/index.ts",
|
|
38
|
-
"build": "tsc && shx chmod +x dist/*.js",
|
|
38
|
+
"build": "tsc -p tsconfig.build.json && shx chmod +x dist/*.js",
|
|
39
39
|
"prepare": "npm run build",
|
|
40
40
|
"prepublishOnly": "npm run build && npm run lint",
|
|
41
41
|
"start": "node dist/index.js",
|
|
@@ -57,8 +57,6 @@
|
|
|
57
57
|
"domhandler": "^5.0.3",
|
|
58
58
|
"express": "^5.2.1",
|
|
59
59
|
"linkedom": "^0.18.12",
|
|
60
|
-
"node-cache": "^5.1.2",
|
|
61
|
-
"safe-regex": "^2.1.1",
|
|
62
60
|
"turndown": "^7.2.2",
|
|
63
61
|
"undici": "^6.22.0",
|
|
64
62
|
"zod": "^3.24.1"
|
|
@@ -69,19 +67,19 @@
|
|
|
69
67
|
"@types/express": "^5.0.6",
|
|
70
68
|
"@types/node": "^22.19.3",
|
|
71
69
|
"@types/turndown": "^5.0.6",
|
|
70
|
+
"@vitest/coverage-v8": "^2.1.9",
|
|
72
71
|
"eslint": "^9.23.2",
|
|
73
72
|
"eslint-config-prettier": "^10.1.8",
|
|
74
73
|
"eslint-plugin-unused-imports": "^4.3.0",
|
|
75
|
-
"knip": "^5.
|
|
74
|
+
"knip": "^5.78.0",
|
|
76
75
|
"prettier": "^3.7.4",
|
|
77
76
|
"shx": "^0.4.0",
|
|
78
77
|
"tsx": "^4.21.0",
|
|
79
78
|
"typescript": "^5.9.3",
|
|
80
|
-
"typescript-eslint": "^8.
|
|
81
|
-
"vitest": "^2.1.9"
|
|
82
|
-
"@vitest/coverage-v8": "^2.1.9"
|
|
79
|
+
"typescript-eslint": "^8.51.0",
|
|
80
|
+
"vitest": "^2.1.9"
|
|
83
81
|
},
|
|
84
82
|
"engines": {
|
|
85
|
-
"node": ">=20.
|
|
83
|
+
"node": ">=20.12.0"
|
|
86
84
|
}
|
|
87
85
|
}
|