@j0hanz/superfetch 2.4.3 → 2.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts +8 -8
- package/dist/cache.js +277 -264
- package/dist/crypto.js +4 -3
- package/dist/dom-noise-removal.js +355 -297
- package/dist/fetch.d.ts +13 -7
- package/dist/fetch.js +636 -690
- package/dist/http-native.js +535 -474
- package/dist/instructions.md +38 -27
- package/dist/language-detection.js +190 -153
- package/dist/markdown-cleanup.js +171 -158
- package/dist/mcp.js +161 -1
- package/dist/resources.d.ts +2 -0
- package/dist/resources.js +44 -0
- package/dist/session.js +144 -105
- package/dist/tasks.d.ts +37 -0
- package/dist/tasks.js +66 -0
- package/dist/tools.d.ts +6 -9
- package/dist/tools.js +166 -136
- package/dist/transform.d.ts +3 -1
- package/dist/transform.js +680 -778
- package/package.json +6 -6
package/dist/fetch.js
CHANGED
|
@@ -46,129 +46,135 @@ const BLOCKED_IPV6_SUBNETS = [
|
|
|
46
46
|
{ subnet: IPV6_FE80, prefix: 10 },
|
|
47
47
|
{ subnet: IPV6_FF00, prefix: 8 },
|
|
48
48
|
];
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
49
|
+
class IpBlocker {
|
|
50
|
+
cachedBlockList;
|
|
51
|
+
isBlockedIp(candidate) {
|
|
52
|
+
if (config.security.blockedHosts.has(candidate))
|
|
53
|
+
return true;
|
|
54
|
+
const ipType = this.resolveIpType(candidate);
|
|
55
|
+
if (!ipType)
|
|
56
|
+
return false;
|
|
57
|
+
const normalized = candidate.toLowerCase();
|
|
58
|
+
if (this.isBlockedBySubnetList(normalized, ipType))
|
|
59
|
+
return true;
|
|
60
|
+
return (config.security.blockedIpPattern.test(normalized) ||
|
|
61
|
+
config.security.blockedIpv4MappedPattern.test(normalized));
|
|
62
|
+
}
|
|
63
|
+
resolveIpType(ip) {
|
|
64
|
+
const ipType = isIP(ip);
|
|
65
|
+
return ipType === 4 || ipType === 6 ? ipType : null;
|
|
66
|
+
}
|
|
67
|
+
isBlockedBySubnetList(ip, ipType) {
|
|
68
|
+
const list = this.getBlockList();
|
|
69
|
+
return ipType === 4 ? list.check(ip, 'ipv4') : list.check(ip, 'ipv6');
|
|
70
|
+
}
|
|
71
|
+
getBlockList() {
|
|
72
|
+
if (!this.cachedBlockList) {
|
|
73
|
+
const list = new BlockList();
|
|
74
|
+
for (const entry of BLOCKED_IPV4_SUBNETS)
|
|
75
|
+
list.addSubnet(entry.subnet, entry.prefix, 'ipv4');
|
|
76
|
+
for (const entry of BLOCKED_IPV6_SUBNETS)
|
|
77
|
+
list.addSubnet(entry.subnet, entry.prefix, 'ipv6');
|
|
78
|
+
this.cachedBlockList = list;
|
|
58
79
|
}
|
|
80
|
+
return this.cachedBlockList;
|
|
59
81
|
}
|
|
60
|
-
return cachedBlockList;
|
|
61
|
-
}
|
|
62
|
-
function matchesBlockedIpPatterns(resolvedIp) {
|
|
63
|
-
return (config.security.blockedIpPattern.test(resolvedIp) ||
|
|
64
|
-
config.security.blockedIpv4MappedPattern.test(resolvedIp));
|
|
65
82
|
}
|
|
83
|
+
const ipBlocker = new IpBlocker();
|
|
84
|
+
/** Backwards-compatible export */
|
|
66
85
|
export function isBlockedIp(ip) {
|
|
67
|
-
|
|
68
|
-
return true;
|
|
69
|
-
}
|
|
70
|
-
const ipType = resolveIpType(ip);
|
|
71
|
-
if (!ipType)
|
|
72
|
-
return false;
|
|
73
|
-
const normalizedIp = ip.toLowerCase();
|
|
74
|
-
if (isBlockedByList(normalizedIp, ipType))
|
|
75
|
-
return true;
|
|
76
|
-
return matchesBlockedIpPatterns(normalizedIp);
|
|
77
|
-
}
|
|
78
|
-
function resolveIpType(ip) {
|
|
79
|
-
const ipType = isIP(ip);
|
|
80
|
-
return ipType === 4 || ipType === 6 ? ipType : null;
|
|
81
|
-
}
|
|
82
|
-
function isBlockedByList(ip, ipType) {
|
|
83
|
-
const blockList = getBlockList();
|
|
84
|
-
if (ipType === 4) {
|
|
85
|
-
return blockList.check(ip, 'ipv4');
|
|
86
|
-
}
|
|
87
|
-
return blockList.check(ip, 'ipv6');
|
|
88
|
-
}
|
|
89
|
-
export function normalizeUrl(urlString) {
|
|
90
|
-
const trimmedUrl = requireTrimmedUrl(urlString);
|
|
91
|
-
assertUrlLength(trimmedUrl);
|
|
92
|
-
const url = parseUrl(trimmedUrl);
|
|
93
|
-
assertHttpProtocol(url);
|
|
94
|
-
assertNoCredentials(url);
|
|
95
|
-
const hostname = normalizeHostname(url);
|
|
96
|
-
assertHostnameAllowed(hostname);
|
|
97
|
-
// Canonicalize hostname to avoid trailing-dot variants and keep url.href consistent.
|
|
98
|
-
url.hostname = hostname;
|
|
99
|
-
return { normalizedUrl: url.href, hostname };
|
|
100
|
-
}
|
|
101
|
-
export function validateAndNormalizeUrl(urlString) {
|
|
102
|
-
return normalizeUrl(urlString).normalizedUrl;
|
|
86
|
+
return ipBlocker.isBlockedIp(ip);
|
|
103
87
|
}
|
|
88
|
+
/* -------------------------------------------------------------------------------------------------
|
|
89
|
+
* URL normalization & hostname policy
|
|
90
|
+
* ------------------------------------------------------------------------------------------------- */
|
|
104
91
|
const VALIDATION_ERROR_CODE = 'VALIDATION_ERROR';
|
|
105
92
|
function createValidationError(message) {
|
|
106
93
|
return createErrorWithCode(message, VALIDATION_ERROR_CODE);
|
|
107
94
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
95
|
+
const BLOCKED_HOST_SUFFIXES = ['.local', '.internal'];
|
|
96
|
+
class UrlNormalizer {
|
|
97
|
+
normalize(urlString) {
|
|
98
|
+
const trimmedUrl = this.requireTrimmedUrl(urlString);
|
|
99
|
+
this.assertUrlLength(trimmedUrl);
|
|
100
|
+
const url = this.parseUrl(trimmedUrl);
|
|
101
|
+
this.assertHttpProtocol(url);
|
|
102
|
+
this.assertNoCredentials(url);
|
|
103
|
+
const hostname = this.normalizeHostname(url);
|
|
104
|
+
this.assertHostnameAllowed(hostname);
|
|
105
|
+
// Canonicalize hostname to avoid trailing-dot variants and keep url.href consistent.
|
|
106
|
+
url.hostname = hostname;
|
|
107
|
+
return { normalizedUrl: url.href, hostname };
|
|
108
|
+
}
|
|
109
|
+
validateAndNormalize(urlString) {
|
|
110
|
+
return this.normalize(urlString).normalizedUrl;
|
|
111
|
+
}
|
|
112
|
+
requireTrimmedUrl(urlString) {
|
|
113
|
+
if (!urlString || typeof urlString !== 'string') {
|
|
114
|
+
throw createValidationError('URL is required');
|
|
115
|
+
}
|
|
116
|
+
const trimmed = urlString.trim();
|
|
117
|
+
if (!trimmed)
|
|
118
|
+
throw createValidationError('URL cannot be empty');
|
|
119
|
+
return trimmed;
|
|
111
120
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
121
|
+
assertUrlLength(url) {
|
|
122
|
+
if (url.length <= config.constants.maxUrlLength)
|
|
123
|
+
return;
|
|
124
|
+
throw createValidationError(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`);
|
|
115
125
|
}
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
return;
|
|
121
|
-
throw createValidationError(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`);
|
|
122
|
-
}
|
|
123
|
-
function parseUrl(urlString) {
|
|
124
|
-
if (!URL.canParse(urlString)) {
|
|
125
|
-
throw createValidationError('Invalid URL format');
|
|
126
|
+
parseUrl(urlString) {
|
|
127
|
+
if (!URL.canParse(urlString))
|
|
128
|
+
throw createValidationError('Invalid URL format');
|
|
129
|
+
return new URL(urlString);
|
|
126
130
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
return;
|
|
132
|
-
throw createValidationError(`Invalid protocol: ${url.protocol}. Only http: and https: are allowed`);
|
|
133
|
-
}
|
|
134
|
-
function assertNoCredentials(url) {
|
|
135
|
-
if (!url.username && !url.password)
|
|
136
|
-
return;
|
|
137
|
-
throw createValidationError('URLs with embedded credentials are not allowed');
|
|
138
|
-
}
|
|
139
|
-
function normalizeHostname(url) {
|
|
140
|
-
let hostname = url.hostname.toLowerCase();
|
|
141
|
-
while (hostname.endsWith('.')) {
|
|
142
|
-
hostname = hostname.slice(0, -1);
|
|
131
|
+
assertHttpProtocol(url) {
|
|
132
|
+
if (url.protocol === 'http:' || url.protocol === 'https:')
|
|
133
|
+
return;
|
|
134
|
+
throw createValidationError(`Invalid protocol: ${url.protocol}. Only http: and https: are allowed`);
|
|
143
135
|
}
|
|
144
|
-
|
|
145
|
-
|
|
136
|
+
assertNoCredentials(url) {
|
|
137
|
+
if (!url.username && !url.password)
|
|
138
|
+
return;
|
|
139
|
+
throw createValidationError('URLs with embedded credentials are not allowed');
|
|
140
|
+
}
|
|
141
|
+
normalizeHostname(url) {
|
|
142
|
+
let hostname = url.hostname.toLowerCase();
|
|
143
|
+
while (hostname.endsWith('.'))
|
|
144
|
+
hostname = hostname.slice(0, -1);
|
|
145
|
+
if (!hostname)
|
|
146
|
+
throw createValidationError('URL must have a valid hostname');
|
|
147
|
+
return hostname;
|
|
148
|
+
}
|
|
149
|
+
assertHostnameAllowed(hostname) {
|
|
150
|
+
this.assertNotBlockedHost(hostname);
|
|
151
|
+
this.assertNotBlockedIp(hostname);
|
|
152
|
+
this.assertNotBlockedHostnameSuffix(hostname);
|
|
153
|
+
}
|
|
154
|
+
assertNotBlockedHost(hostname) {
|
|
155
|
+
if (!config.security.blockedHosts.has(hostname))
|
|
156
|
+
return;
|
|
157
|
+
throw createValidationError(`Blocked host: ${hostname}. Internal hosts are not allowed`);
|
|
158
|
+
}
|
|
159
|
+
assertNotBlockedIp(hostname) {
|
|
160
|
+
if (!ipBlocker.isBlockedIp(hostname))
|
|
161
|
+
return;
|
|
162
|
+
throw createValidationError(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
|
|
163
|
+
}
|
|
164
|
+
assertNotBlockedHostnameSuffix(hostname) {
|
|
165
|
+
const blocked = BLOCKED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix));
|
|
166
|
+
if (!blocked)
|
|
167
|
+
return;
|
|
168
|
+
throw createValidationError(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
|
|
146
169
|
}
|
|
147
|
-
return hostname;
|
|
148
|
-
}
|
|
149
|
-
const BLOCKED_HOST_SUFFIXES = ['.local', '.internal'];
|
|
150
|
-
function assertHostnameAllowed(hostname) {
|
|
151
|
-
assertNotBlockedHost(hostname);
|
|
152
|
-
assertNotBlockedIp(hostname);
|
|
153
|
-
assertNotBlockedHostnameSuffix(hostname);
|
|
154
|
-
}
|
|
155
|
-
function assertNotBlockedHost(hostname) {
|
|
156
|
-
if (!config.security.blockedHosts.has(hostname))
|
|
157
|
-
return;
|
|
158
|
-
throw createValidationError(`Blocked host: ${hostname}. Internal hosts are not allowed`);
|
|
159
|
-
}
|
|
160
|
-
function assertNotBlockedIp(hostname) {
|
|
161
|
-
if (!isBlockedIp(hostname))
|
|
162
|
-
return;
|
|
163
|
-
throw createValidationError(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
|
|
164
170
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
171
|
+
const urlNormalizer = new UrlNormalizer();
|
|
172
|
+
/** Backwards-compatible exports */
|
|
173
|
+
export function normalizeUrl(urlString) {
|
|
174
|
+
return urlNormalizer.normalize(urlString);
|
|
169
175
|
}
|
|
170
|
-
function
|
|
171
|
-
return
|
|
176
|
+
export function validateAndNormalizeUrl(urlString) {
|
|
177
|
+
return urlNormalizer.validateAndNormalize(urlString);
|
|
172
178
|
}
|
|
173
179
|
const GITHUB_BLOB_RULE = {
|
|
174
180
|
name: 'github',
|
|
@@ -221,61 +227,6 @@ const TRANSFORM_RULES = [
|
|
|
221
227
|
BITBUCKET_SRC_RULE,
|
|
222
228
|
];
|
|
223
229
|
const BITBUCKET_RAW_RE = /bitbucket\.org\/[^/]+\/[^/]+\/raw\//;
|
|
224
|
-
function isRawUrl(url) {
|
|
225
|
-
const lowerUrl = url.toLowerCase();
|
|
226
|
-
return (lowerUrl.includes('raw.githubusercontent.com') ||
|
|
227
|
-
lowerUrl.includes('gist.githubusercontent.com') ||
|
|
228
|
-
lowerUrl.includes('/-/raw/') ||
|
|
229
|
-
BITBUCKET_RAW_RE.test(lowerUrl));
|
|
230
|
-
}
|
|
231
|
-
function getUrlWithoutParams(url) {
|
|
232
|
-
const hashIndex = url.indexOf('#');
|
|
233
|
-
const queryIndex = url.indexOf('?');
|
|
234
|
-
const endIndex = Math.min(queryIndex === -1 ? url.length : queryIndex, hashIndex === -1 ? url.length : hashIndex);
|
|
235
|
-
const hash = hashIndex !== -1 ? url.slice(hashIndex) : '';
|
|
236
|
-
return {
|
|
237
|
-
base: url.slice(0, endIndex),
|
|
238
|
-
hash,
|
|
239
|
-
};
|
|
240
|
-
}
|
|
241
|
-
function resolveUrlToMatch(rule, base, hash) {
|
|
242
|
-
if (rule.name !== 'github-gist')
|
|
243
|
-
return base;
|
|
244
|
-
if (!hash.startsWith('#file-'))
|
|
245
|
-
return base;
|
|
246
|
-
return base + hash;
|
|
247
|
-
}
|
|
248
|
-
function applyTransformRules(base, hash) {
|
|
249
|
-
for (const rule of TRANSFORM_RULES) {
|
|
250
|
-
const urlToMatch = resolveUrlToMatch(rule, base, hash);
|
|
251
|
-
const match = rule.pattern.exec(urlToMatch);
|
|
252
|
-
if (match) {
|
|
253
|
-
return { url: rule.transform(match), platform: rule.name };
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
return null;
|
|
257
|
-
}
|
|
258
|
-
export function transformToRawUrl(url) {
|
|
259
|
-
if (!url)
|
|
260
|
-
return { url, transformed: false };
|
|
261
|
-
if (isRawUrl(url)) {
|
|
262
|
-
return { url, transformed: false };
|
|
263
|
-
}
|
|
264
|
-
const { base, hash } = getUrlWithoutParams(url);
|
|
265
|
-
const result = applyTransformRules(base, hash);
|
|
266
|
-
if (!result)
|
|
267
|
-
return { url, transformed: false };
|
|
268
|
-
logDebug('URL transformed to raw content URL', {
|
|
269
|
-
platform: result.platform,
|
|
270
|
-
original: url.substring(0, 100),
|
|
271
|
-
transformed: result.url.substring(0, 100),
|
|
272
|
-
});
|
|
273
|
-
return {
|
|
274
|
-
url: result.url,
|
|
275
|
-
transformed: true,
|
|
276
|
-
platform: result.platform,
|
|
277
|
-
};
|
|
278
|
-
}
|
|
279
230
|
const RAW_TEXT_EXTENSIONS = new Set([
|
|
280
231
|
'.md',
|
|
281
232
|
'.markdown',
|
|
@@ -290,226 +241,261 @@ const RAW_TEXT_EXTENSIONS = new Set([
|
|
|
290
241
|
'.adoc',
|
|
291
242
|
'.org',
|
|
292
243
|
]);
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
return
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
if (family !== 4 && family !== 6) {
|
|
345
|
-
return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
|
|
346
|
-
}
|
|
347
|
-
const ip = typeof addr === 'string' ? addr : addr.address;
|
|
348
|
-
if (isBlockedIp(ip)) {
|
|
349
|
-
return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
|
|
244
|
+
class RawUrlTransformer {
|
|
245
|
+
transformToRawUrl(url) {
|
|
246
|
+
if (!url)
|
|
247
|
+
return { url, transformed: false };
|
|
248
|
+
if (this.isRawUrl(url))
|
|
249
|
+
return { url, transformed: false };
|
|
250
|
+
const { base, hash } = this.splitParams(url);
|
|
251
|
+
const result = this.applyRules(base, hash);
|
|
252
|
+
if (!result)
|
|
253
|
+
return { url, transformed: false };
|
|
254
|
+
logDebug('URL transformed to raw content URL', {
|
|
255
|
+
platform: result.platform,
|
|
256
|
+
original: url.substring(0, 100),
|
|
257
|
+
transformed: result.url.substring(0, 100),
|
|
258
|
+
});
|
|
259
|
+
return { url: result.url, transformed: true, platform: result.platform };
|
|
260
|
+
}
|
|
261
|
+
isRawTextContentUrl(url) {
|
|
262
|
+
if (!url)
|
|
263
|
+
return false;
|
|
264
|
+
if (this.isRawUrl(url))
|
|
265
|
+
return true;
|
|
266
|
+
const { base } = this.splitParams(url);
|
|
267
|
+
const lowerBase = base.toLowerCase();
|
|
268
|
+
const lastDot = lowerBase.lastIndexOf('.');
|
|
269
|
+
if (lastDot === -1)
|
|
270
|
+
return false;
|
|
271
|
+
return RAW_TEXT_EXTENSIONS.has(lowerBase.slice(lastDot));
|
|
272
|
+
}
|
|
273
|
+
isRawUrl(url) {
|
|
274
|
+
const lower = url.toLowerCase();
|
|
275
|
+
return (lower.includes('raw.githubusercontent.com') ||
|
|
276
|
+
lower.includes('gist.githubusercontent.com') ||
|
|
277
|
+
lower.includes('/-/raw/') ||
|
|
278
|
+
BITBUCKET_RAW_RE.test(lower));
|
|
279
|
+
}
|
|
280
|
+
splitParams(url) {
|
|
281
|
+
const hashIndex = url.indexOf('#');
|
|
282
|
+
const queryIndex = url.indexOf('?');
|
|
283
|
+
const endIndex = Math.min(queryIndex === -1 ? url.length : queryIndex, hashIndex === -1 ? url.length : hashIndex);
|
|
284
|
+
const hash = hashIndex !== -1 ? url.slice(hashIndex) : '';
|
|
285
|
+
return { base: url.slice(0, endIndex), hash };
|
|
286
|
+
}
|
|
287
|
+
applyRules(base, hash) {
|
|
288
|
+
for (const rule of TRANSFORM_RULES) {
|
|
289
|
+
const urlToMatch = rule.name === 'github-gist' && hash.startsWith('#file-')
|
|
290
|
+
? base + hash
|
|
291
|
+
: base;
|
|
292
|
+
const match = rule.pattern.exec(urlToMatch);
|
|
293
|
+
if (match)
|
|
294
|
+
return { url: rule.transform(match), platform: rule.name };
|
|
350
295
|
}
|
|
296
|
+
return null;
|
|
351
297
|
}
|
|
352
|
-
return null;
|
|
353
298
|
}
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
return
|
|
299
|
+
const rawUrlTransformer = new RawUrlTransformer();
|
|
300
|
+
/** Backwards-compatible exports */
|
|
301
|
+
export function transformToRawUrl(url) {
|
|
302
|
+
return rawUrlTransformer.transformToRawUrl(url);
|
|
358
303
|
}
|
|
359
|
-
function
|
|
360
|
-
|
|
304
|
+
export function isRawTextContentUrl(url) {
|
|
305
|
+
return rawUrlTransformer.isRawTextContentUrl(url);
|
|
361
306
|
}
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
307
|
+
const DNS_LOOKUP_TIMEOUT_MS = 5000;
|
|
308
|
+
class SafeDnsLookup {
|
|
309
|
+
lookup(hostname, options, callback) {
|
|
310
|
+
const normalizedOptions = this.normalizeOptions(options);
|
|
311
|
+
const useAll = Boolean(normalizedOptions.all);
|
|
312
|
+
const resolvedFamily = this.resolveFamily(normalizedOptions.family);
|
|
313
|
+
const lookupOptions = {
|
|
314
|
+
family: normalizedOptions.family,
|
|
315
|
+
hints: normalizedOptions.hints,
|
|
316
|
+
all: true, // Always request all results; we select based on caller preference.
|
|
317
|
+
order: this.resolveOrder(normalizedOptions),
|
|
318
|
+
};
|
|
319
|
+
const timeout = this.createTimeout(hostname, callback);
|
|
320
|
+
const safeCallback = (err, address, family) => {
|
|
321
|
+
if (timeout.isDone())
|
|
322
|
+
return;
|
|
323
|
+
timeout.markDone();
|
|
324
|
+
callback(err, address, family);
|
|
325
|
+
};
|
|
326
|
+
(async () => {
|
|
327
|
+
try {
|
|
328
|
+
const result = await dns.promises.lookup(hostname, lookupOptions);
|
|
329
|
+
const addresses = Array.isArray(result) ? result : [result];
|
|
330
|
+
this.handleLookupResult(null, addresses, hostname, resolvedFamily, useAll, safeCallback);
|
|
331
|
+
}
|
|
332
|
+
catch (error) {
|
|
333
|
+
this.handleLookupResult(error, [], hostname, resolvedFamily, useAll, safeCallback);
|
|
334
|
+
}
|
|
335
|
+
})().catch((error) => {
|
|
336
|
+
if (!timeout.isDone()) {
|
|
337
|
+
safeCallback(error, []);
|
|
338
|
+
}
|
|
339
|
+
});
|
|
366
340
|
}
|
|
367
|
-
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
341
|
+
normalizeOptions(options) {
|
|
342
|
+
return typeof options === 'number' ? { family: options } : options;
|
|
343
|
+
}
|
|
344
|
+
resolveFamily(family) {
|
|
345
|
+
if (family === 'IPv4')
|
|
346
|
+
return 4;
|
|
347
|
+
if (family === 'IPv6')
|
|
348
|
+
return 6;
|
|
349
|
+
return family;
|
|
350
|
+
}
|
|
351
|
+
resolveOrder(options) {
|
|
352
|
+
if (options.order)
|
|
353
|
+
return options.order;
|
|
354
|
+
// legacy `verbatim` option support
|
|
355
|
+
if (isObject(options)) {
|
|
356
|
+
const legacy = options.verbatim;
|
|
357
|
+
if (typeof legacy === 'boolean')
|
|
358
|
+
return legacy ? 'verbatim' : 'ipv4first';
|
|
359
|
+
}
|
|
360
|
+
return 'verbatim';
|
|
373
361
|
}
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
362
|
+
handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
|
|
363
|
+
if (error) {
|
|
364
|
+
callback(error, addresses);
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
const list = this.normalizeResults(addresses, resolvedFamily);
|
|
368
|
+
const validationError = this.validateResults(list, hostname);
|
|
369
|
+
if (validationError) {
|
|
370
|
+
callback(validationError, list);
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
const selection = this.selectResult(list, useAll, hostname);
|
|
374
|
+
if (selection.error) {
|
|
375
|
+
callback(selection.error, selection.fallback);
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
callback(null, selection.address, selection.family);
|
|
378
379
|
}
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
const lookupOptions = buildLookupOptions(normalizedOptions);
|
|
384
|
-
const timeout = createLookupTimeout(hostname, callback);
|
|
385
|
-
const safeCallback = wrapLookupCallback(callback, timeout);
|
|
386
|
-
dns.lookup(hostname, lookupOptions, createLookupCallback(hostname, resolvedFamily, useAll, safeCallback));
|
|
387
|
-
}
|
|
388
|
-
function normalizeLookupOptions(options) {
|
|
389
|
-
return typeof options === 'number' ? { family: options } : options;
|
|
390
|
-
}
|
|
391
|
-
function buildLookupContext(options) {
|
|
392
|
-
const normalizedOptions = normalizeLookupOptions(options);
|
|
393
|
-
return {
|
|
394
|
-
normalizedOptions,
|
|
395
|
-
useAll: Boolean(normalizedOptions.all),
|
|
396
|
-
resolvedFamily: resolveFamily(normalizedOptions.family),
|
|
397
|
-
};
|
|
398
|
-
}
|
|
399
|
-
const DEFAULT_DNS_ORDER = 'verbatim';
|
|
400
|
-
function resolveResultOrder(options) {
|
|
401
|
-
if (options.order)
|
|
402
|
-
return options.order;
|
|
403
|
-
const legacyVerbatim = getLegacyVerbatim(options);
|
|
404
|
-
if (legacyVerbatim !== undefined) {
|
|
405
|
-
return legacyVerbatim ? 'verbatim' : 'ipv4first';
|
|
380
|
+
normalizeResults(addresses, family) {
|
|
381
|
+
if (Array.isArray(addresses))
|
|
382
|
+
return addresses;
|
|
383
|
+
return [{ address: addresses, family: family ?? 4 }];
|
|
406
384
|
}
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
const
|
|
412
|
-
|
|
385
|
+
validateResults(list, hostname) {
|
|
386
|
+
if (list.length === 0) {
|
|
387
|
+
return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
|
|
388
|
+
}
|
|
389
|
+
for (const addr of list) {
|
|
390
|
+
if (addr.family !== 4 && addr.family !== 6) {
|
|
391
|
+
return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
|
|
392
|
+
}
|
|
393
|
+
if (ipBlocker.isBlockedIp(addr.address)) {
|
|
394
|
+
return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
return null;
|
|
413
398
|
}
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
if (done)
|
|
440
|
-
return;
|
|
441
|
-
done = true;
|
|
442
|
-
callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
|
|
443
|
-
}, DNS_LOOKUP_TIMEOUT_MS);
|
|
444
|
-
timer.unref();
|
|
445
|
-
return {
|
|
446
|
-
isDone: () => done,
|
|
447
|
-
markDone: () => {
|
|
399
|
+
selectResult(list, useAll, hostname) {
|
|
400
|
+
if (list.length === 0) {
|
|
401
|
+
return {
|
|
402
|
+
error: createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA'),
|
|
403
|
+
fallback: [],
|
|
404
|
+
address: [],
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
if (useAll)
|
|
408
|
+
return { address: list, fallback: list };
|
|
409
|
+
const first = list.at(0);
|
|
410
|
+
if (!first) {
|
|
411
|
+
return {
|
|
412
|
+
error: createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA'),
|
|
413
|
+
fallback: [],
|
|
414
|
+
address: [],
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
return { address: first.address, family: first.family, fallback: list };
|
|
418
|
+
}
|
|
419
|
+
createTimeout(hostname, callback) {
|
|
420
|
+
let done = false;
|
|
421
|
+
const timer = setTimeout(() => {
|
|
422
|
+
if (done)
|
|
423
|
+
return;
|
|
448
424
|
done = true;
|
|
449
|
-
|
|
450
|
-
},
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
}
|
|
460
|
-
}
|
|
425
|
+
callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
|
|
426
|
+
}, DNS_LOOKUP_TIMEOUT_MS);
|
|
427
|
+
timer.unref();
|
|
428
|
+
return {
|
|
429
|
+
isDone: () => done,
|
|
430
|
+
markDone: () => {
|
|
431
|
+
done = true;
|
|
432
|
+
clearTimeout(timer);
|
|
433
|
+
},
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
const safeDns = new SafeDnsLookup();
|
|
438
|
+
/* -------------------------------------------------------------------------------------------------
|
|
439
|
+
* Dispatcher / Agent lifecycle
|
|
440
|
+
* ------------------------------------------------------------------------------------------------- */
|
|
461
441
|
function getAgentOptions() {
|
|
462
442
|
const cpuCount = os.availableParallelism();
|
|
463
443
|
return {
|
|
464
444
|
keepAliveTimeout: 60000,
|
|
465
445
|
connections: Math.max(cpuCount * 2, 25),
|
|
466
446
|
pipelining: 1,
|
|
467
|
-
connect: { lookup:
|
|
447
|
+
connect: { lookup: safeDns.lookup.bind(safeDns) },
|
|
468
448
|
};
|
|
469
449
|
}
|
|
470
450
|
export const dispatcher = new Agent(getAgentOptions());
|
|
471
451
|
export function destroyAgents() {
|
|
472
452
|
void dispatcher.close();
|
|
473
453
|
}
|
|
454
|
+
/* -------------------------------------------------------------------------------------------------
|
|
455
|
+
* Fetch error mapping (request-level)
|
|
456
|
+
* ------------------------------------------------------------------------------------------------- */
|
|
474
457
|
function parseRetryAfter(header) {
|
|
475
458
|
if (!header)
|
|
476
459
|
return 60;
|
|
477
|
-
const parsed = parseInt(header, 10);
|
|
460
|
+
const parsed = Number.parseInt(header, 10);
|
|
478
461
|
return Number.isNaN(parsed) ? 60 : parsed;
|
|
479
462
|
}
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
timeout
|
|
488
|
-
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
|
|
463
|
+
class FetchErrorFactory {
|
|
464
|
+
canceled(url) {
|
|
465
|
+
return new FetchError('Request was canceled', url, 499, {
|
|
466
|
+
reason: 'aborted',
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
timeout(url, timeoutMs) {
|
|
470
|
+
return new FetchError(`Request timeout after ${timeoutMs}ms`, url, 504, {
|
|
471
|
+
timeout: timeoutMs,
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
rateLimited(url, retryAfterHeader) {
|
|
475
|
+
return new FetchError('Too many requests', url, 429, {
|
|
476
|
+
retryAfter: parseRetryAfter(retryAfterHeader),
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
http(url, status, statusText) {
|
|
480
|
+
return new FetchError(`HTTP ${status}: ${statusText}`, url, status);
|
|
481
|
+
}
|
|
482
|
+
tooManyRedirects(url) {
|
|
483
|
+
return new FetchError('Too many redirects', url);
|
|
484
|
+
}
|
|
485
|
+
missingRedirectLocation(url) {
|
|
486
|
+
return new FetchError('Redirect response missing Location header', url);
|
|
487
|
+
}
|
|
488
|
+
sizeLimit(url, maxBytes) {
|
|
489
|
+
return new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
|
|
490
|
+
}
|
|
491
|
+
network(url, message) {
|
|
492
|
+
return new FetchError(`Network error: Could not reach ${url}`, url, undefined, message ? { message } : {});
|
|
493
|
+
}
|
|
494
|
+
unknown(url, message) {
|
|
495
|
+
return new FetchError(message, url);
|
|
496
|
+
}
|
|
512
497
|
}
|
|
498
|
+
const fetchErrors = new FetchErrorFactory();
|
|
513
499
|
function isAbortError(error) {
|
|
514
500
|
return (error instanceof Error &&
|
|
515
501
|
(error.name === 'AbortError' || error.name === 'TimeoutError'));
|
|
@@ -517,350 +503,322 @@ function isAbortError(error) {
|
|
|
517
503
|
function isTimeoutError(error) {
|
|
518
504
|
return error instanceof Error && error.name === 'TimeoutError';
|
|
519
505
|
}
|
|
520
|
-
function getRequestUrl(record) {
|
|
521
|
-
const value = record.requestUrl;
|
|
522
|
-
return typeof value === 'string' ? value : null;
|
|
523
|
-
}
|
|
524
506
|
function resolveErrorUrl(error, fallback) {
|
|
525
507
|
if (error instanceof FetchError)
|
|
526
508
|
return error.url;
|
|
527
509
|
if (!isObject(error))
|
|
528
510
|
return fallback;
|
|
529
|
-
const requestUrl =
|
|
530
|
-
|
|
531
|
-
return requestUrl;
|
|
532
|
-
return fallback;
|
|
533
|
-
}
|
|
534
|
-
function resolveAbortFetchError(error, url, timeoutMs) {
|
|
535
|
-
if (!isAbortError(error))
|
|
536
|
-
return null;
|
|
537
|
-
if (isTimeoutError(error)) {
|
|
538
|
-
return createTimeoutError(url, timeoutMs);
|
|
539
|
-
}
|
|
540
|
-
return createCanceledError(url);
|
|
541
|
-
}
|
|
542
|
-
function resolveUnexpectedFetchError(error, url) {
|
|
543
|
-
if (error instanceof Error) {
|
|
544
|
-
return createNetworkError(url, error.message);
|
|
545
|
-
}
|
|
546
|
-
return createUnknownError(url, 'Unexpected error');
|
|
511
|
+
const { requestUrl } = error;
|
|
512
|
+
return typeof requestUrl === 'string' ? requestUrl : fallback;
|
|
547
513
|
}
|
|
548
514
|
function mapFetchError(error, fallbackUrl, timeoutMs) {
|
|
549
515
|
if (error instanceof FetchError)
|
|
550
516
|
return error;
|
|
551
517
|
const url = resolveErrorUrl(error, fallbackUrl);
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
}
|
|
557
|
-
const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
|
|
558
|
-
function publishFetchEvent(event) {
|
|
559
|
-
if (!fetchChannel.hasSubscribers)
|
|
560
|
-
return;
|
|
561
|
-
try {
|
|
562
|
-
fetchChannel.publish(event);
|
|
563
|
-
}
|
|
564
|
-
catch {
|
|
565
|
-
// Avoid crashing the publisher if a subscriber throws.
|
|
518
|
+
if (isAbortError(error)) {
|
|
519
|
+
return isTimeoutError(error)
|
|
520
|
+
? fetchErrors.timeout(url, timeoutMs)
|
|
521
|
+
: fetchErrors.canceled(url);
|
|
566
522
|
}
|
|
523
|
+
if (error instanceof Error)
|
|
524
|
+
return fetchErrors.network(url, error.message);
|
|
525
|
+
return fetchErrors.unknown(url, 'Unexpected error');
|
|
567
526
|
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
527
|
+
const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
|
|
528
|
+
const SLOW_REQUEST_THRESHOLD_MS = 5000;
|
|
529
|
+
class FetchTelemetry {
|
|
530
|
+
start(url, method) {
|
|
531
|
+
const safeUrl = redactUrl(url);
|
|
532
|
+
const contextRequestId = getRequestId();
|
|
533
|
+
const operationId = getOperationId();
|
|
534
|
+
const ctx = {
|
|
535
|
+
requestId: randomUUID(),
|
|
536
|
+
startTime: performance.now(),
|
|
537
|
+
url: safeUrl,
|
|
538
|
+
method: method.toUpperCase(),
|
|
539
|
+
...(contextRequestId ? { contextRequestId } : {}),
|
|
540
|
+
...(operationId ? { operationId } : {}),
|
|
541
|
+
};
|
|
542
|
+
this.publish({
|
|
543
|
+
v: 1,
|
|
544
|
+
type: 'start',
|
|
545
|
+
requestId: ctx.requestId,
|
|
546
|
+
method: ctx.method,
|
|
547
|
+
url: ctx.url,
|
|
548
|
+
...(ctx.contextRequestId
|
|
549
|
+
? { contextRequestId: ctx.contextRequestId }
|
|
550
|
+
: {}),
|
|
551
|
+
...(ctx.operationId ? { operationId: ctx.operationId } : {}),
|
|
552
|
+
});
|
|
553
|
+
logDebug('HTTP Request', {
|
|
554
|
+
requestId: ctx.requestId,
|
|
555
|
+
method: ctx.method,
|
|
556
|
+
url: ctx.url,
|
|
557
|
+
...(ctx.contextRequestId
|
|
558
|
+
? { contextRequestId: ctx.contextRequestId }
|
|
559
|
+
: {}),
|
|
560
|
+
...(ctx.operationId ? { operationId: ctx.operationId } : {}),
|
|
561
|
+
});
|
|
562
|
+
return ctx;
|
|
563
|
+
}
|
|
564
|
+
recordResponse(context, response, contentSize) {
|
|
565
|
+
const duration = performance.now() - context.startTime;
|
|
566
|
+
const durationLabel = `${Math.round(duration)}ms`;
|
|
567
|
+
this.publish({
|
|
568
|
+
v: 1,
|
|
569
|
+
type: 'end',
|
|
570
|
+
requestId: context.requestId,
|
|
571
|
+
status: response.status,
|
|
572
|
+
duration,
|
|
573
|
+
...(context.contextRequestId
|
|
574
|
+
? { contextRequestId: context.contextRequestId }
|
|
575
|
+
: {}),
|
|
576
|
+
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
577
|
+
});
|
|
578
|
+
const contentType = response.headers.get('content-type') ?? undefined;
|
|
579
|
+
const contentLengthHeader = response.headers.get('content-length');
|
|
580
|
+
const size = contentLengthHeader ??
|
|
581
|
+
(contentSize === undefined ? undefined : String(contentSize));
|
|
582
|
+
logDebug('HTTP Response', {
|
|
583
|
+
requestId: context.requestId,
|
|
584
|
+
status: response.status,
|
|
585
|
+
url: context.url,
|
|
586
|
+
duration: durationLabel,
|
|
587
|
+
...(context.contextRequestId
|
|
588
|
+
? { contextRequestId: context.contextRequestId }
|
|
589
|
+
: {}),
|
|
590
|
+
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
591
|
+
...(contentType ? { contentType } : {}),
|
|
592
|
+
...(size ? { size } : {}),
|
|
593
|
+
});
|
|
594
|
+
if (duration > SLOW_REQUEST_THRESHOLD_MS) {
|
|
595
|
+
logWarn('Slow HTTP request detected', {
|
|
596
|
+
requestId: context.requestId,
|
|
597
|
+
url: context.url,
|
|
598
|
+
duration: durationLabel,
|
|
599
|
+
...(context.contextRequestId
|
|
600
|
+
? { contextRequestId: context.contextRequestId }
|
|
601
|
+
: {}),
|
|
602
|
+
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
603
|
+
});
|
|
604
|
+
}
|
|
575
605
|
}
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
type: 'error',
|
|
607
|
-
requestId: context.requestId,
|
|
608
|
-
url: context.url,
|
|
609
|
-
error: err.message,
|
|
610
|
-
duration,
|
|
611
|
-
...contextFields,
|
|
612
|
-
};
|
|
613
|
-
if (code !== undefined) {
|
|
614
|
-
event.code = code;
|
|
606
|
+
recordError(context, error, status) {
|
|
607
|
+
const duration = performance.now() - context.startTime;
|
|
608
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
609
|
+
const code = isSystemError(err) ? err.code : undefined;
|
|
610
|
+
this.publish({
|
|
611
|
+
v: 1,
|
|
612
|
+
type: 'error',
|
|
613
|
+
requestId: context.requestId,
|
|
614
|
+
url: context.url,
|
|
615
|
+
error: err.message,
|
|
616
|
+
duration,
|
|
617
|
+
...(code !== undefined ? { code } : {}),
|
|
618
|
+
...(status !== undefined ? { status } : {}),
|
|
619
|
+
...(context.contextRequestId
|
|
620
|
+
? { contextRequestId: context.contextRequestId }
|
|
621
|
+
: {}),
|
|
622
|
+
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
623
|
+
});
|
|
624
|
+
const log = status === 429 ? logWarn : logError;
|
|
625
|
+
log('HTTP Request Error', {
|
|
626
|
+
requestId: context.requestId,
|
|
627
|
+
url: context.url,
|
|
628
|
+
status,
|
|
629
|
+
code,
|
|
630
|
+
error: err.message,
|
|
631
|
+
...(context.contextRequestId
|
|
632
|
+
? { contextRequestId: context.contextRequestId }
|
|
633
|
+
: {}),
|
|
634
|
+
...(context.operationId ? { operationId: context.operationId } : {}),
|
|
635
|
+
});
|
|
615
636
|
}
|
|
616
|
-
|
|
617
|
-
|
|
637
|
+
publish(event) {
|
|
638
|
+
if (!fetchChannel.hasSubscribers)
|
|
639
|
+
return;
|
|
640
|
+
try {
|
|
641
|
+
fetchChannel.publish(event);
|
|
642
|
+
}
|
|
643
|
+
catch {
|
|
644
|
+
// Best-effort; subscriber failures must not crash request path.
|
|
645
|
+
}
|
|
618
646
|
}
|
|
619
|
-
return event;
|
|
620
|
-
}
|
|
621
|
-
function createTelemetryContext(url, method) {
|
|
622
|
-
const safeUrl = redactUrl(url);
|
|
623
|
-
const contextRequestId = getRequestId();
|
|
624
|
-
const operationId = getOperationId();
|
|
625
|
-
return {
|
|
626
|
-
requestId: randomUUID(),
|
|
627
|
-
startTime: performance.now(),
|
|
628
|
-
url: safeUrl,
|
|
629
|
-
method: method.toUpperCase(),
|
|
630
|
-
...(contextRequestId ? { contextRequestId } : {}),
|
|
631
|
-
...(operationId ? { operationId } : {}),
|
|
632
|
-
};
|
|
633
647
|
}
|
|
648
|
+
const telemetry = new FetchTelemetry();
|
|
649
|
+
/** Backwards-compatible exports */
|
|
634
650
|
export function startFetchTelemetry(url, method) {
|
|
635
|
-
|
|
636
|
-
const contextFields = buildContextFields(context);
|
|
637
|
-
publishFetchEvent({
|
|
638
|
-
v: 1,
|
|
639
|
-
type: 'start',
|
|
640
|
-
requestId: context.requestId,
|
|
641
|
-
method: context.method,
|
|
642
|
-
url: context.url,
|
|
643
|
-
...contextFields,
|
|
644
|
-
});
|
|
645
|
-
logDebug('HTTP Request', {
|
|
646
|
-
requestId: context.requestId,
|
|
647
|
-
method: context.method,
|
|
648
|
-
url: context.url,
|
|
649
|
-
...contextFields,
|
|
650
|
-
});
|
|
651
|
-
return context;
|
|
651
|
+
return telemetry.start(url, method);
|
|
652
652
|
}
|
|
653
653
|
export function recordFetchResponse(context, response, contentSize) {
|
|
654
|
-
|
|
655
|
-
const durationLabel = `${Math.round(duration)}ms`;
|
|
656
|
-
const contextFields = buildContextFields(context);
|
|
657
|
-
const responseMetadata = buildResponseMetadata(response, contentSize);
|
|
658
|
-
publishFetchEvent({
|
|
659
|
-
v: 1,
|
|
660
|
-
type: 'end',
|
|
661
|
-
requestId: context.requestId,
|
|
662
|
-
status: response.status,
|
|
663
|
-
duration,
|
|
664
|
-
...contextFields,
|
|
665
|
-
});
|
|
666
|
-
logDebug('HTTP Response', {
|
|
667
|
-
requestId: context.requestId,
|
|
668
|
-
status: response.status,
|
|
669
|
-
url: context.url,
|
|
670
|
-
duration: durationLabel,
|
|
671
|
-
...contextFields,
|
|
672
|
-
...responseMetadata,
|
|
673
|
-
});
|
|
674
|
-
logSlowRequest(context, duration, durationLabel, contextFields);
|
|
654
|
+
telemetry.recordResponse(context, response, contentSize);
|
|
675
655
|
}
|
|
676
656
|
export function recordFetchError(context, error, status) {
|
|
677
|
-
|
|
678
|
-
const err = error instanceof Error ? error : new Error(String(error));
|
|
679
|
-
const contextFields = buildContextFields(context);
|
|
680
|
-
const code = resolveSystemErrorCode(err);
|
|
681
|
-
publishFetchEvent(buildFetchErrorEvent(context, err, duration, contextFields, status, code));
|
|
682
|
-
const log = status === 429 ? logWarn : logError;
|
|
683
|
-
log('HTTP Request Error', {
|
|
684
|
-
requestId: context.requestId,
|
|
685
|
-
url: context.url,
|
|
686
|
-
status,
|
|
687
|
-
code,
|
|
688
|
-
error: err.message,
|
|
689
|
-
...contextFields,
|
|
690
|
-
});
|
|
657
|
+
telemetry.recordError(context, error, status);
|
|
691
658
|
}
|
|
659
|
+
/* -------------------------------------------------------------------------------------------------
|
|
660
|
+
* Redirect handling
|
|
661
|
+
* ------------------------------------------------------------------------------------------------- */
|
|
692
662
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
693
663
|
function isRedirectStatus(status) {
|
|
694
664
|
return REDIRECT_STATUSES.has(status);
|
|
695
665
|
}
|
|
696
666
|
function cancelResponseBody(response) {
|
|
697
667
|
const cancelPromise = response.body?.cancel();
|
|
698
|
-
if (cancelPromise)
|
|
668
|
+
if (cancelPromise)
|
|
699
669
|
cancelPromise.catch(() => {
|
|
700
|
-
|
|
670
|
+
/* ignore */
|
|
701
671
|
});
|
|
702
|
-
}
|
|
703
672
|
}
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
673
|
+
class RedirectFollower {
|
|
674
|
+
async fetchWithRedirects(url, init, maxRedirects) {
|
|
675
|
+
let currentUrl = url;
|
|
676
|
+
const redirectLimit = Math.max(0, maxRedirects);
|
|
677
|
+
for (let redirectCount = 0; redirectCount <= redirectLimit; redirectCount += 1) {
|
|
678
|
+
const { response, nextUrl } = await this.withRedirectErrorContext(currentUrl, async () => this.performFetchCycle(currentUrl, init, redirectLimit, redirectCount));
|
|
679
|
+
if (!nextUrl)
|
|
680
|
+
return { response, url: currentUrl };
|
|
681
|
+
currentUrl = nextUrl;
|
|
682
|
+
}
|
|
683
|
+
throw fetchErrors.tooManyRedirects(currentUrl);
|
|
684
|
+
}
|
|
685
|
+
async performFetchCycle(currentUrl, init, redirectLimit, redirectCount) {
|
|
686
|
+
const response = await fetch(currentUrl, { ...init, redirect: 'manual' });
|
|
687
|
+
if (!isRedirectStatus(response.status))
|
|
688
|
+
return { response };
|
|
689
|
+
this.assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount);
|
|
690
|
+
const location = this.getRedirectLocation(response, currentUrl);
|
|
691
|
+
cancelResponseBody(response);
|
|
692
|
+
return {
|
|
693
|
+
response,
|
|
694
|
+
nextUrl: this.resolveRedirectTarget(currentUrl, location),
|
|
695
|
+
};
|
|
708
696
|
}
|
|
709
|
-
assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount)
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
nextUrl: resolveRedirectTarget(currentUrl, location),
|
|
715
|
-
};
|
|
716
|
-
}
|
|
717
|
-
function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
|
|
718
|
-
if (redirectCount < redirectLimit)
|
|
719
|
-
return;
|
|
720
|
-
cancelResponseBody(response);
|
|
721
|
-
throw createTooManyRedirectsError(currentUrl);
|
|
722
|
-
}
|
|
723
|
-
function getRedirectLocation(response, currentUrl) {
|
|
724
|
-
const location = response.headers.get('location');
|
|
725
|
-
if (location)
|
|
726
|
-
return location;
|
|
727
|
-
cancelResponseBody(response);
|
|
728
|
-
throw createMissingRedirectLocationError(currentUrl);
|
|
729
|
-
}
|
|
730
|
-
function annotateRedirectError(error, url) {
|
|
731
|
-
if (!isObject(error))
|
|
732
|
-
return;
|
|
733
|
-
error.requestUrl = url;
|
|
734
|
-
}
|
|
735
|
-
function resolveRedirectTarget(baseUrl, location) {
|
|
736
|
-
if (!URL.canParse(location, baseUrl)) {
|
|
737
|
-
throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
|
|
697
|
+
assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
|
|
698
|
+
if (redirectCount < redirectLimit)
|
|
699
|
+
return;
|
|
700
|
+
cancelResponseBody(response);
|
|
701
|
+
throw fetchErrors.tooManyRedirects(currentUrl);
|
|
738
702
|
}
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
703
|
+
getRedirectLocation(response, currentUrl) {
|
|
704
|
+
const location = response.headers.get('location');
|
|
705
|
+
if (location)
|
|
706
|
+
return location;
|
|
707
|
+
cancelResponseBody(response);
|
|
708
|
+
throw fetchErrors.missingRedirectLocation(currentUrl);
|
|
709
|
+
}
|
|
710
|
+
resolveRedirectTarget(baseUrl, location) {
|
|
711
|
+
if (!URL.canParse(location, baseUrl))
|
|
712
|
+
throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
|
|
713
|
+
const resolved = new URL(location, baseUrl);
|
|
714
|
+
if (resolved.username || resolved.password) {
|
|
715
|
+
throw createErrorWithCode('Redirect target includes credentials', 'EBADREDIRECT');
|
|
716
|
+
}
|
|
717
|
+
return validateAndNormalizeUrl(resolved.href);
|
|
742
718
|
}
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
return await fn();
|
|
719
|
+
annotateRedirectError(error, url) {
|
|
720
|
+
if (!isObject(error))
|
|
721
|
+
return;
|
|
722
|
+
error.requestUrl = url;
|
|
748
723
|
}
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
724
|
+
async withRedirectErrorContext(url, fn) {
|
|
725
|
+
try {
|
|
726
|
+
return await fn();
|
|
727
|
+
}
|
|
728
|
+
catch (error) {
|
|
729
|
+
this.annotateRedirectError(error, url);
|
|
730
|
+
throw error;
|
|
731
|
+
}
|
|
752
732
|
}
|
|
753
733
|
}
|
|
734
|
+
const redirectFollower = new RedirectFollower();
|
|
735
|
+
/** Backwards-compatible export */
|
|
754
736
|
export async function fetchWithRedirects(url, init, maxRedirects) {
|
|
755
|
-
|
|
756
|
-
const redirectLimit = Math.max(0, maxRedirects);
|
|
757
|
-
for (let redirectCount = 0; redirectCount <= redirectLimit; redirectCount += 1) {
|
|
758
|
-
const { response, nextUrl } = await withRedirectErrorContext(currentUrl, () => performFetchCycle(currentUrl, init, redirectLimit, redirectCount));
|
|
759
|
-
if (!nextUrl) {
|
|
760
|
-
return { response, url: currentUrl };
|
|
761
|
-
}
|
|
762
|
-
currentUrl = nextUrl;
|
|
763
|
-
}
|
|
764
|
-
throw createTooManyRedirectsError(currentUrl);
|
|
737
|
+
return redirectFollower.fetchWithRedirects(url, init, maxRedirects);
|
|
765
738
|
}
|
|
739
|
+
/* -------------------------------------------------------------------------------------------------
|
|
740
|
+
* Response reading (max size + abort-aware streaming)
|
|
741
|
+
* ------------------------------------------------------------------------------------------------- */
|
|
766
742
|
function assertContentLengthWithinLimit(response, url, maxBytes) {
|
|
767
|
-
const
|
|
768
|
-
if (!
|
|
743
|
+
const header = response.headers.get('content-length');
|
|
744
|
+
if (!header)
|
|
769
745
|
return;
|
|
770
|
-
const contentLength = Number.parseInt(
|
|
771
|
-
if (Number.isNaN(contentLength) || contentLength <= maxBytes)
|
|
746
|
+
const contentLength = Number.parseInt(header, 10);
|
|
747
|
+
if (Number.isNaN(contentLength) || contentLength <= maxBytes)
|
|
772
748
|
return;
|
|
773
|
-
}
|
|
774
749
|
cancelResponseBody(response);
|
|
775
|
-
throw
|
|
776
|
-
}
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
}
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
throw createAbortError(url);
|
|
822
|
-
}
|
|
823
|
-
throw error;
|
|
824
|
-
}
|
|
825
|
-
async function readAllChunks(reader, state, url, maxBytes, signal) {
|
|
826
|
-
await throwIfAborted(signal, url, reader);
|
|
827
|
-
let result = await reader.read();
|
|
828
|
-
while (!result.done) {
|
|
829
|
-
appendChunk(state, result.value, maxBytes, url);
|
|
830
|
-
await throwIfAborted(signal, url, reader);
|
|
831
|
-
result = await reader.read();
|
|
832
|
-
}
|
|
833
|
-
}
|
|
834
|
-
async function readStreamWithLimit(stream, url, maxBytes, signal) {
|
|
835
|
-
const state = createReadState();
|
|
836
|
-
const reader = stream.getReader();
|
|
837
|
-
try {
|
|
838
|
-
await readAllChunks(reader, state, url, maxBytes, signal);
|
|
839
|
-
}
|
|
840
|
-
catch (error) {
|
|
841
|
-
await handleReadFailure(error, signal, url, reader);
|
|
750
|
+
throw fetchErrors.sizeLimit(url, maxBytes);
|
|
751
|
+
}
|
|
752
|
+
class ResponseTextReader {
|
|
753
|
+
async read(response, url, maxBytes, signal) {
|
|
754
|
+
assertContentLengthWithinLimit(response, url, maxBytes);
|
|
755
|
+
if (!response.body) {
|
|
756
|
+
const text = await response.text();
|
|
757
|
+
const size = Buffer.byteLength(text);
|
|
758
|
+
if (size > maxBytes)
|
|
759
|
+
throw fetchErrors.sizeLimit(url, maxBytes);
|
|
760
|
+
return { text, size };
|
|
761
|
+
}
|
|
762
|
+
return this.readStreamWithLimit(response.body, url, maxBytes, signal);
|
|
763
|
+
}
|
|
764
|
+
async readStreamWithLimit(stream, url, maxBytes, signal) {
|
|
765
|
+
const decoder = new TextDecoder();
|
|
766
|
+
const parts = [];
|
|
767
|
+
let total = 0;
|
|
768
|
+
const reader = stream.getReader();
|
|
769
|
+
try {
|
|
770
|
+
await this.throwIfAborted(signal, url, reader);
|
|
771
|
+
let result = await reader.read();
|
|
772
|
+
while (!result.done) {
|
|
773
|
+
total += result.value.byteLength;
|
|
774
|
+
if (total > maxBytes)
|
|
775
|
+
throw fetchErrors.sizeLimit(url, maxBytes);
|
|
776
|
+
const decoded = decoder.decode(result.value, { stream: true });
|
|
777
|
+
if (decoded)
|
|
778
|
+
parts.push(decoded);
|
|
779
|
+
await this.throwIfAborted(signal, url, reader);
|
|
780
|
+
result = await reader.read();
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
catch (error) {
|
|
784
|
+
await this.cancelReaderQuietly(reader);
|
|
785
|
+
if (signal?.aborted)
|
|
786
|
+
throw new FetchError('Request was aborted during response read', url, 499, { reason: 'aborted' });
|
|
787
|
+
throw error;
|
|
788
|
+
}
|
|
789
|
+
finally {
|
|
790
|
+
reader.releaseLock();
|
|
791
|
+
}
|
|
792
|
+
const final = decoder.decode();
|
|
793
|
+
if (final)
|
|
794
|
+
parts.push(final);
|
|
795
|
+
return { text: parts.join(''), size: total };
|
|
842
796
|
}
|
|
843
|
-
|
|
844
|
-
|
|
797
|
+
async throwIfAborted(signal, url, reader) {
|
|
798
|
+
if (!signal?.aborted)
|
|
799
|
+
return;
|
|
800
|
+
await this.cancelReaderQuietly(reader);
|
|
801
|
+
throw new FetchError('Request was aborted during response read', url, 499, {
|
|
802
|
+
reason: 'aborted',
|
|
803
|
+
});
|
|
845
804
|
}
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
throw createSizeLimitError(url, maxBytes);
|
|
805
|
+
async cancelReaderQuietly(reader) {
|
|
806
|
+
try {
|
|
807
|
+
await reader.cancel();
|
|
808
|
+
}
|
|
809
|
+
catch {
|
|
810
|
+
// ignore
|
|
811
|
+
}
|
|
854
812
|
}
|
|
855
|
-
return { text, size };
|
|
856
813
|
}
|
|
814
|
+
const responseReader = new ResponseTextReader();
|
|
815
|
+
/** Backwards-compatible export */
|
|
857
816
|
export async function readResponseText(response, url, maxBytes, signal) {
|
|
858
|
-
|
|
859
|
-
if (!response.body) {
|
|
860
|
-
return readResponseTextFallback(response, url, maxBytes);
|
|
861
|
-
}
|
|
862
|
-
return readStreamWithLimit(response.body, url, maxBytes, signal);
|
|
817
|
+
return responseReader.read(response, url, maxBytes, signal);
|
|
863
818
|
}
|
|
819
|
+
/* -------------------------------------------------------------------------------------------------
|
|
820
|
+
* HTTP fetcher (headers, signals, response handling)
|
|
821
|
+
* ------------------------------------------------------------------------------------------------- */
|
|
864
822
|
const DEFAULT_HEADERS = {
|
|
865
823
|
'User-Agent': config.fetcher.userAgent,
|
|
866
824
|
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
@@ -873,63 +831,51 @@ function buildHeaders() {
|
|
|
873
831
|
}
|
|
874
832
|
function buildRequestSignal(timeoutMs, external) {
|
|
875
833
|
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
876
|
-
|
|
877
|
-
return timeoutSignal;
|
|
878
|
-
return AbortSignal.any([external, timeoutSignal]);
|
|
834
|
+
return external ? AbortSignal.any([external, timeoutSignal]) : timeoutSignal;
|
|
879
835
|
}
|
|
880
836
|
function buildRequestInit(headers, signal) {
|
|
881
|
-
return {
|
|
882
|
-
method: 'GET',
|
|
883
|
-
headers,
|
|
884
|
-
signal,
|
|
885
|
-
dispatcher,
|
|
886
|
-
};
|
|
837
|
+
return { method: 'GET', headers, signal, dispatcher };
|
|
887
838
|
}
|
|
888
839
|
function resolveResponseError(response, finalUrl) {
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
}
|
|
892
|
-
function resolveRateLimitError(response, finalUrl) {
|
|
893
|
-
return response.status === 429
|
|
894
|
-
? createRateLimitError(finalUrl, response.headers.get('retry-after'))
|
|
895
|
-
: null;
|
|
896
|
-
}
|
|
897
|
-
function resolveHttpError(response, finalUrl) {
|
|
840
|
+
if (response.status === 429) {
|
|
841
|
+
return fetchErrors.rateLimited(finalUrl, response.headers.get('retry-after'));
|
|
842
|
+
}
|
|
898
843
|
return response.ok
|
|
899
844
|
? null
|
|
900
|
-
:
|
|
845
|
+
: fetchErrors.http(finalUrl, response.status, response.statusText);
|
|
901
846
|
}
|
|
902
|
-
async function handleFetchResponse(response, finalUrl,
|
|
847
|
+
async function handleFetchResponse(response, finalUrl, ctx, signal) {
|
|
903
848
|
const responseError = resolveResponseError(response, finalUrl);
|
|
904
849
|
if (responseError) {
|
|
905
850
|
cancelResponseBody(response);
|
|
906
851
|
throw responseError;
|
|
907
852
|
}
|
|
908
|
-
const { text, size } = await
|
|
909
|
-
|
|
853
|
+
const { text, size } = await responseReader.read(response, finalUrl, config.fetcher.maxContentLength, signal);
|
|
854
|
+
telemetry.recordResponse(ctx, response, size);
|
|
910
855
|
return text;
|
|
911
856
|
}
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
const
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
857
|
+
class HttpFetcher {
|
|
858
|
+
async fetchNormalizedUrl(normalizedUrl, options) {
|
|
859
|
+
const timeoutMs = config.fetcher.timeout;
|
|
860
|
+
const headers = buildHeaders();
|
|
861
|
+
const signal = buildRequestSignal(timeoutMs, options?.signal);
|
|
862
|
+
const init = buildRequestInit(headers, signal);
|
|
863
|
+
const ctx = telemetry.start(normalizedUrl, 'GET');
|
|
864
|
+
try {
|
|
865
|
+
const { response, url: finalUrl } = await redirectFollower.fetchWithRedirects(normalizedUrl, init, config.fetcher.maxRedirects);
|
|
866
|
+
ctx.url = finalUrl;
|
|
867
|
+
return await handleFetchResponse(response, finalUrl, ctx, init.signal ?? undefined);
|
|
868
|
+
}
|
|
869
|
+
catch (error) {
|
|
870
|
+
const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
|
|
871
|
+
ctx.url = mapped.url;
|
|
872
|
+
telemetry.recordError(ctx, mapped, mapped.statusCode);
|
|
873
|
+
throw mapped;
|
|
874
|
+
}
|
|
922
875
|
}
|
|
923
876
|
}
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
telemetry.url = finalUrl;
|
|
927
|
-
return handleFetchResponse(response, finalUrl, telemetry, requestInit.signal ?? undefined);
|
|
928
|
-
}
|
|
877
|
+
const httpFetcher = new HttpFetcher();
|
|
878
|
+
/** Backwards-compatible export */
|
|
929
879
|
export async function fetchNormalizedUrl(normalizedUrl, options) {
|
|
930
|
-
|
|
931
|
-
const headers = buildHeaders();
|
|
932
|
-
const signal = buildRequestSignal(timeoutMs, options?.signal);
|
|
933
|
-
const requestInit = buildRequestInit(headers, signal);
|
|
934
|
-
return fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs);
|
|
880
|
+
return httpFetcher.fetchNormalizedUrl(normalizedUrl, options);
|
|
935
881
|
}
|