@j0hanz/superfetch 2.4.3 → 2.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/fetch.js CHANGED
@@ -46,129 +46,135 @@ const BLOCKED_IPV6_SUBNETS = [
46
46
  { subnet: IPV6_FE80, prefix: 10 },
47
47
  { subnet: IPV6_FF00, prefix: 8 },
48
48
  ];
49
- let cachedBlockList;
50
- function getBlockList() {
51
- if (!cachedBlockList) {
52
- cachedBlockList = new BlockList();
53
- for (const entry of BLOCKED_IPV4_SUBNETS) {
54
- cachedBlockList.addSubnet(entry.subnet, entry.prefix, 'ipv4');
55
- }
56
- for (const entry of BLOCKED_IPV6_SUBNETS) {
57
- cachedBlockList.addSubnet(entry.subnet, entry.prefix, 'ipv6');
49
+ class IpBlocker {
50
+ cachedBlockList;
51
+ isBlockedIp(candidate) {
52
+ if (config.security.blockedHosts.has(candidate))
53
+ return true;
54
+ const ipType = this.resolveIpType(candidate);
55
+ if (!ipType)
56
+ return false;
57
+ const normalized = candidate.toLowerCase();
58
+ if (this.isBlockedBySubnetList(normalized, ipType))
59
+ return true;
60
+ return (config.security.blockedIpPattern.test(normalized) ||
61
+ config.security.blockedIpv4MappedPattern.test(normalized));
62
+ }
63
+ resolveIpType(ip) {
64
+ const ipType = isIP(ip);
65
+ return ipType === 4 || ipType === 6 ? ipType : null;
66
+ }
67
+ isBlockedBySubnetList(ip, ipType) {
68
+ const list = this.getBlockList();
69
+ return ipType === 4 ? list.check(ip, 'ipv4') : list.check(ip, 'ipv6');
70
+ }
71
+ getBlockList() {
72
+ if (!this.cachedBlockList) {
73
+ const list = new BlockList();
74
+ for (const entry of BLOCKED_IPV4_SUBNETS)
75
+ list.addSubnet(entry.subnet, entry.prefix, 'ipv4');
76
+ for (const entry of BLOCKED_IPV6_SUBNETS)
77
+ list.addSubnet(entry.subnet, entry.prefix, 'ipv6');
78
+ this.cachedBlockList = list;
58
79
  }
80
+ return this.cachedBlockList;
59
81
  }
60
- return cachedBlockList;
61
- }
62
- function matchesBlockedIpPatterns(resolvedIp) {
63
- return (config.security.blockedIpPattern.test(resolvedIp) ||
64
- config.security.blockedIpv4MappedPattern.test(resolvedIp));
65
82
  }
83
+ const ipBlocker = new IpBlocker();
84
+ /** Backwards-compatible export */
66
85
  export function isBlockedIp(ip) {
67
- if (config.security.blockedHosts.has(ip)) {
68
- return true;
69
- }
70
- const ipType = resolveIpType(ip);
71
- if (!ipType)
72
- return false;
73
- const normalizedIp = ip.toLowerCase();
74
- if (isBlockedByList(normalizedIp, ipType))
75
- return true;
76
- return matchesBlockedIpPatterns(normalizedIp);
77
- }
78
- function resolveIpType(ip) {
79
- const ipType = isIP(ip);
80
- return ipType === 4 || ipType === 6 ? ipType : null;
81
- }
82
- function isBlockedByList(ip, ipType) {
83
- const blockList = getBlockList();
84
- if (ipType === 4) {
85
- return blockList.check(ip, 'ipv4');
86
- }
87
- return blockList.check(ip, 'ipv6');
88
- }
89
- export function normalizeUrl(urlString) {
90
- const trimmedUrl = requireTrimmedUrl(urlString);
91
- assertUrlLength(trimmedUrl);
92
- const url = parseUrl(trimmedUrl);
93
- assertHttpProtocol(url);
94
- assertNoCredentials(url);
95
- const hostname = normalizeHostname(url);
96
- assertHostnameAllowed(hostname);
97
- // Canonicalize hostname to avoid trailing-dot variants and keep url.href consistent.
98
- url.hostname = hostname;
99
- return { normalizedUrl: url.href, hostname };
100
- }
101
- export function validateAndNormalizeUrl(urlString) {
102
- return normalizeUrl(urlString).normalizedUrl;
86
+ return ipBlocker.isBlockedIp(ip);
103
87
  }
88
+ /* -------------------------------------------------------------------------------------------------
89
+ * URL normalization & hostname policy
90
+ * ------------------------------------------------------------------------------------------------- */
104
91
  const VALIDATION_ERROR_CODE = 'VALIDATION_ERROR';
105
92
  function createValidationError(message) {
106
93
  return createErrorWithCode(message, VALIDATION_ERROR_CODE);
107
94
  }
108
- function requireTrimmedUrl(urlString) {
109
- if (!urlString || typeof urlString !== 'string') {
110
- throw createValidationError('URL is required');
95
+ const BLOCKED_HOST_SUFFIXES = ['.local', '.internal'];
96
+ class UrlNormalizer {
97
+ normalize(urlString) {
98
+ const trimmedUrl = this.requireTrimmedUrl(urlString);
99
+ this.assertUrlLength(trimmedUrl);
100
+ const url = this.parseUrl(trimmedUrl);
101
+ this.assertHttpProtocol(url);
102
+ this.assertNoCredentials(url);
103
+ const hostname = this.normalizeHostname(url);
104
+ this.assertHostnameAllowed(hostname);
105
+ // Canonicalize hostname to avoid trailing-dot variants and keep url.href consistent.
106
+ url.hostname = hostname;
107
+ return { normalizedUrl: url.href, hostname };
108
+ }
109
+ validateAndNormalize(urlString) {
110
+ return this.normalize(urlString).normalizedUrl;
111
+ }
112
+ requireTrimmedUrl(urlString) {
113
+ if (!urlString || typeof urlString !== 'string') {
114
+ throw createValidationError('URL is required');
115
+ }
116
+ const trimmed = urlString.trim();
117
+ if (!trimmed)
118
+ throw createValidationError('URL cannot be empty');
119
+ return trimmed;
111
120
  }
112
- const trimmedUrl = urlString.trim();
113
- if (!trimmedUrl) {
114
- throw createValidationError('URL cannot be empty');
121
+ assertUrlLength(url) {
122
+ if (url.length <= config.constants.maxUrlLength)
123
+ return;
124
+ throw createValidationError(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`);
115
125
  }
116
- return trimmedUrl;
117
- }
118
- function assertUrlLength(url) {
119
- if (url.length <= config.constants.maxUrlLength)
120
- return;
121
- throw createValidationError(`URL exceeds maximum length of ${config.constants.maxUrlLength} characters`);
122
- }
123
- function parseUrl(urlString) {
124
- if (!URL.canParse(urlString)) {
125
- throw createValidationError('Invalid URL format');
126
+ parseUrl(urlString) {
127
+ if (!URL.canParse(urlString))
128
+ throw createValidationError('Invalid URL format');
129
+ return new URL(urlString);
126
130
  }
127
- return new URL(urlString);
128
- }
129
- function assertHttpProtocol(url) {
130
- if (url.protocol === 'http:' || url.protocol === 'https:')
131
- return;
132
- throw createValidationError(`Invalid protocol: ${url.protocol}. Only http: and https: are allowed`);
133
- }
134
- function assertNoCredentials(url) {
135
- if (!url.username && !url.password)
136
- return;
137
- throw createValidationError('URLs with embedded credentials are not allowed');
138
- }
139
- function normalizeHostname(url) {
140
- let hostname = url.hostname.toLowerCase();
141
- while (hostname.endsWith('.')) {
142
- hostname = hostname.slice(0, -1);
131
+ assertHttpProtocol(url) {
132
+ if (url.protocol === 'http:' || url.protocol === 'https:')
133
+ return;
134
+ throw createValidationError(`Invalid protocol: ${url.protocol}. Only http: and https: are allowed`);
143
135
  }
144
- if (!hostname) {
145
- throw createValidationError('URL must have a valid hostname');
136
+ assertNoCredentials(url) {
137
+ if (!url.username && !url.password)
138
+ return;
139
+ throw createValidationError('URLs with embedded credentials are not allowed');
140
+ }
141
+ normalizeHostname(url) {
142
+ let hostname = url.hostname.toLowerCase();
143
+ while (hostname.endsWith('.'))
144
+ hostname = hostname.slice(0, -1);
145
+ if (!hostname)
146
+ throw createValidationError('URL must have a valid hostname');
147
+ return hostname;
148
+ }
149
+ assertHostnameAllowed(hostname) {
150
+ this.assertNotBlockedHost(hostname);
151
+ this.assertNotBlockedIp(hostname);
152
+ this.assertNotBlockedHostnameSuffix(hostname);
153
+ }
154
+ assertNotBlockedHost(hostname) {
155
+ if (!config.security.blockedHosts.has(hostname))
156
+ return;
157
+ throw createValidationError(`Blocked host: ${hostname}. Internal hosts are not allowed`);
158
+ }
159
+ assertNotBlockedIp(hostname) {
160
+ if (!ipBlocker.isBlockedIp(hostname))
161
+ return;
162
+ throw createValidationError(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
163
+ }
164
+ assertNotBlockedHostnameSuffix(hostname) {
165
+ const blocked = BLOCKED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix));
166
+ if (!blocked)
167
+ return;
168
+ throw createValidationError(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
146
169
  }
147
- return hostname;
148
- }
149
- const BLOCKED_HOST_SUFFIXES = ['.local', '.internal'];
150
- function assertHostnameAllowed(hostname) {
151
- assertNotBlockedHost(hostname);
152
- assertNotBlockedIp(hostname);
153
- assertNotBlockedHostnameSuffix(hostname);
154
- }
155
- function assertNotBlockedHost(hostname) {
156
- if (!config.security.blockedHosts.has(hostname))
157
- return;
158
- throw createValidationError(`Blocked host: ${hostname}. Internal hosts are not allowed`);
159
- }
160
- function assertNotBlockedIp(hostname) {
161
- if (!isBlockedIp(hostname))
162
- return;
163
- throw createValidationError(`Blocked IP range: ${hostname}. Private IPs are not allowed`);
164
170
  }
165
- function assertNotBlockedHostnameSuffix(hostname) {
166
- if (!matchesBlockedSuffix(hostname))
167
- return;
168
- throw createValidationError(`Blocked hostname pattern: ${hostname}. Internal domain suffixes are not allowed`);
171
+ const urlNormalizer = new UrlNormalizer();
172
+ /** Backwards-compatible exports */
173
+ export function normalizeUrl(urlString) {
174
+ return urlNormalizer.normalize(urlString);
169
175
  }
170
- function matchesBlockedSuffix(hostname) {
171
- return BLOCKED_HOST_SUFFIXES.some((suffix) => hostname.endsWith(suffix));
176
+ export function validateAndNormalizeUrl(urlString) {
177
+ return urlNormalizer.validateAndNormalize(urlString);
172
178
  }
173
179
  const GITHUB_BLOB_RULE = {
174
180
  name: 'github',
@@ -221,61 +227,6 @@ const TRANSFORM_RULES = [
221
227
  BITBUCKET_SRC_RULE,
222
228
  ];
223
229
  const BITBUCKET_RAW_RE = /bitbucket\.org\/[^/]+\/[^/]+\/raw\//;
224
- function isRawUrl(url) {
225
- const lowerUrl = url.toLowerCase();
226
- return (lowerUrl.includes('raw.githubusercontent.com') ||
227
- lowerUrl.includes('gist.githubusercontent.com') ||
228
- lowerUrl.includes('/-/raw/') ||
229
- BITBUCKET_RAW_RE.test(lowerUrl));
230
- }
231
- function getUrlWithoutParams(url) {
232
- const hashIndex = url.indexOf('#');
233
- const queryIndex = url.indexOf('?');
234
- const endIndex = Math.min(queryIndex === -1 ? url.length : queryIndex, hashIndex === -1 ? url.length : hashIndex);
235
- const hash = hashIndex !== -1 ? url.slice(hashIndex) : '';
236
- return {
237
- base: url.slice(0, endIndex),
238
- hash,
239
- };
240
- }
241
- function resolveUrlToMatch(rule, base, hash) {
242
- if (rule.name !== 'github-gist')
243
- return base;
244
- if (!hash.startsWith('#file-'))
245
- return base;
246
- return base + hash;
247
- }
248
- function applyTransformRules(base, hash) {
249
- for (const rule of TRANSFORM_RULES) {
250
- const urlToMatch = resolveUrlToMatch(rule, base, hash);
251
- const match = rule.pattern.exec(urlToMatch);
252
- if (match) {
253
- return { url: rule.transform(match), platform: rule.name };
254
- }
255
- }
256
- return null;
257
- }
258
- export function transformToRawUrl(url) {
259
- if (!url)
260
- return { url, transformed: false };
261
- if (isRawUrl(url)) {
262
- return { url, transformed: false };
263
- }
264
- const { base, hash } = getUrlWithoutParams(url);
265
- const result = applyTransformRules(base, hash);
266
- if (!result)
267
- return { url, transformed: false };
268
- logDebug('URL transformed to raw content URL', {
269
- platform: result.platform,
270
- original: url.substring(0, 100),
271
- transformed: result.url.substring(0, 100),
272
- });
273
- return {
274
- url: result.url,
275
- transformed: true,
276
- platform: result.platform,
277
- };
278
- }
279
230
  const RAW_TEXT_EXTENSIONS = new Set([
280
231
  '.md',
281
232
  '.markdown',
@@ -290,226 +241,261 @@ const RAW_TEXT_EXTENSIONS = new Set([
290
241
  '.adoc',
291
242
  '.org',
292
243
  ]);
293
- export function isRawTextContentUrl(url) {
294
- if (!url)
295
- return false;
296
- if (isRawUrl(url))
297
- return true;
298
- const { base } = getUrlWithoutParams(url);
299
- const lowerBase = base.toLowerCase();
300
- return hasKnownRawTextExtension(lowerBase);
301
- }
302
- function hasKnownRawTextExtension(urlBaseLower) {
303
- const lastDot = urlBaseLower.lastIndexOf('.');
304
- if (lastDot === -1)
305
- return false;
306
- const ext = urlBaseLower.slice(lastDot);
307
- return RAW_TEXT_EXTENSIONS.has(ext);
308
- }
309
- const DNS_LOOKUP_TIMEOUT_MS = 5000;
310
- const SLOW_REQUEST_THRESHOLD_MS = 5000;
311
- function normalizeLookupResults(addresses, family) {
312
- if (Array.isArray(addresses)) {
313
- return addresses;
314
- }
315
- return [{ address: addresses, family: family ?? 4 }];
316
- }
317
- function createNoDnsResultsError(hostname) {
318
- return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
319
- }
320
- function createEmptySelection(hostname) {
321
- return {
322
- error: createNoDnsResultsError(hostname),
323
- fallback: [],
324
- address: [],
325
- };
326
- }
327
- function selectLookupResult(list, useAll, hostname) {
328
- if (list.length === 0)
329
- return createEmptySelection(hostname);
330
- if (useAll)
331
- return { address: list, fallback: list };
332
- const first = list.at(0);
333
- if (!first)
334
- return createEmptySelection(hostname);
335
- return {
336
- address: first.address,
337
- family: first.family,
338
- fallback: list,
339
- };
340
- }
341
- function findLookupError(list, hostname) {
342
- for (const addr of list) {
343
- const family = typeof addr === 'string' ? 0 : addr.family;
344
- if (family !== 4 && family !== 6) {
345
- return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
346
- }
347
- const ip = typeof addr === 'string' ? addr : addr.address;
348
- if (isBlockedIp(ip)) {
349
- return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
244
+ class RawUrlTransformer {
245
+ transformToRawUrl(url) {
246
+ if (!url)
247
+ return { url, transformed: false };
248
+ if (this.isRawUrl(url))
249
+ return { url, transformed: false };
250
+ const { base, hash } = this.splitParams(url);
251
+ const result = this.applyRules(base, hash);
252
+ if (!result)
253
+ return { url, transformed: false };
254
+ logDebug('URL transformed to raw content URL', {
255
+ platform: result.platform,
256
+ original: url.substring(0, 100),
257
+ transformed: result.url.substring(0, 100),
258
+ });
259
+ return { url: result.url, transformed: true, platform: result.platform };
260
+ }
261
+ isRawTextContentUrl(url) {
262
+ if (!url)
263
+ return false;
264
+ if (this.isRawUrl(url))
265
+ return true;
266
+ const { base } = this.splitParams(url);
267
+ const lowerBase = base.toLowerCase();
268
+ const lastDot = lowerBase.lastIndexOf('.');
269
+ if (lastDot === -1)
270
+ return false;
271
+ return RAW_TEXT_EXTENSIONS.has(lowerBase.slice(lastDot));
272
+ }
273
+ isRawUrl(url) {
274
+ const lower = url.toLowerCase();
275
+ return (lower.includes('raw.githubusercontent.com') ||
276
+ lower.includes('gist.githubusercontent.com') ||
277
+ lower.includes('/-/raw/') ||
278
+ BITBUCKET_RAW_RE.test(lower));
279
+ }
280
+ splitParams(url) {
281
+ const hashIndex = url.indexOf('#');
282
+ const queryIndex = url.indexOf('?');
283
+ const endIndex = Math.min(queryIndex === -1 ? url.length : queryIndex, hashIndex === -1 ? url.length : hashIndex);
284
+ const hash = hashIndex !== -1 ? url.slice(hashIndex) : '';
285
+ return { base: url.slice(0, endIndex), hash };
286
+ }
287
+ applyRules(base, hash) {
288
+ for (const rule of TRANSFORM_RULES) {
289
+ const urlToMatch = rule.name === 'github-gist' && hash.startsWith('#file-')
290
+ ? base + hash
291
+ : base;
292
+ const match = rule.pattern.exec(urlToMatch);
293
+ if (match)
294
+ return { url: rule.transform(match), platform: rule.name };
350
295
  }
296
+ return null;
351
297
  }
352
- return null;
353
298
  }
354
- function normalizeAndValidateLookupResults(addresses, resolvedFamily, hostname) {
355
- const list = normalizeLookupResults(addresses, resolvedFamily);
356
- const error = findLookupError(list, hostname);
357
- return { list, error };
299
+ const rawUrlTransformer = new RawUrlTransformer();
300
+ /** Backwards-compatible exports */
301
+ export function transformToRawUrl(url) {
302
+ return rawUrlTransformer.transformToRawUrl(url);
358
303
  }
359
- function respondLookupError(callback, error, addresses) {
360
- callback(error, addresses);
304
+ export function isRawTextContentUrl(url) {
305
+ return rawUrlTransformer.isRawTextContentUrl(url);
361
306
  }
362
- function respondLookupSelection(callback, selection) {
363
- if (selection.error) {
364
- callback(selection.error, selection.fallback);
365
- return;
307
+ const DNS_LOOKUP_TIMEOUT_MS = 5000;
308
+ class SafeDnsLookup {
309
+ lookup(hostname, options, callback) {
310
+ const normalizedOptions = this.normalizeOptions(options);
311
+ const useAll = Boolean(normalizedOptions.all);
312
+ const resolvedFamily = this.resolveFamily(normalizedOptions.family);
313
+ const lookupOptions = {
314
+ family: normalizedOptions.family,
315
+ hints: normalizedOptions.hints,
316
+ all: true, // Always request all results; we select based on caller preference.
317
+ order: this.resolveOrder(normalizedOptions),
318
+ };
319
+ const timeout = this.createTimeout(hostname, callback);
320
+ const safeCallback = (err, address, family) => {
321
+ if (timeout.isDone())
322
+ return;
323
+ timeout.markDone();
324
+ callback(err, address, family);
325
+ };
326
+ (async () => {
327
+ try {
328
+ const result = await dns.promises.lookup(hostname, lookupOptions);
329
+ const addresses = Array.isArray(result) ? result : [result];
330
+ this.handleLookupResult(null, addresses, hostname, resolvedFamily, useAll, safeCallback);
331
+ }
332
+ catch (error) {
333
+ this.handleLookupResult(error, [], hostname, resolvedFamily, useAll, safeCallback);
334
+ }
335
+ })().catch((error) => {
336
+ if (!timeout.isDone()) {
337
+ safeCallback(error, []);
338
+ }
339
+ });
366
340
  }
367
- callback(null, selection.address, selection.family);
368
- }
369
- function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
370
- if (error) {
371
- respondLookupError(callback, error, addresses);
372
- return;
341
+ normalizeOptions(options) {
342
+ return typeof options === 'number' ? { family: options } : options;
343
+ }
344
+ resolveFamily(family) {
345
+ if (family === 'IPv4')
346
+ return 4;
347
+ if (family === 'IPv6')
348
+ return 6;
349
+ return family;
350
+ }
351
+ resolveOrder(options) {
352
+ if (options.order)
353
+ return options.order;
354
+ // legacy `verbatim` option support
355
+ if (isObject(options)) {
356
+ const legacy = options.verbatim;
357
+ if (typeof legacy === 'boolean')
358
+ return legacy ? 'verbatim' : 'ipv4first';
359
+ }
360
+ return 'verbatim';
373
361
  }
374
- const { list, error: lookupError } = normalizeAndValidateLookupResults(addresses, resolvedFamily, hostname);
375
- if (lookupError) {
376
- respondLookupError(callback, lookupError, list);
377
- return;
362
+ handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
363
+ if (error) {
364
+ callback(error, addresses);
365
+ return;
366
+ }
367
+ const list = this.normalizeResults(addresses, resolvedFamily);
368
+ const validationError = this.validateResults(list, hostname);
369
+ if (validationError) {
370
+ callback(validationError, list);
371
+ return;
372
+ }
373
+ const selection = this.selectResult(list, useAll, hostname);
374
+ if (selection.error) {
375
+ callback(selection.error, selection.fallback);
376
+ return;
377
+ }
378
+ callback(null, selection.address, selection.family);
378
379
  }
379
- respondLookupSelection(callback, selectLookupResult(list, useAll, hostname));
380
- }
381
- function resolveDns(hostname, options, callback) {
382
- const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
383
- const lookupOptions = buildLookupOptions(normalizedOptions);
384
- const timeout = createLookupTimeout(hostname, callback);
385
- const safeCallback = wrapLookupCallback(callback, timeout);
386
- dns.lookup(hostname, lookupOptions, createLookupCallback(hostname, resolvedFamily, useAll, safeCallback));
387
- }
388
- function normalizeLookupOptions(options) {
389
- return typeof options === 'number' ? { family: options } : options;
390
- }
391
- function buildLookupContext(options) {
392
- const normalizedOptions = normalizeLookupOptions(options);
393
- return {
394
- normalizedOptions,
395
- useAll: Boolean(normalizedOptions.all),
396
- resolvedFamily: resolveFamily(normalizedOptions.family),
397
- };
398
- }
399
- const DEFAULT_DNS_ORDER = 'verbatim';
400
- function resolveResultOrder(options) {
401
- if (options.order)
402
- return options.order;
403
- const legacyVerbatim = getLegacyVerbatim(options);
404
- if (legacyVerbatim !== undefined) {
405
- return legacyVerbatim ? 'verbatim' : 'ipv4first';
380
+ normalizeResults(addresses, family) {
381
+ if (Array.isArray(addresses))
382
+ return addresses;
383
+ return [{ address: addresses, family: family ?? 4 }];
406
384
  }
407
- return DEFAULT_DNS_ORDER;
408
- }
409
- function getLegacyVerbatim(options) {
410
- if (isObject(options)) {
411
- const { verbatim } = options;
412
- return typeof verbatim === 'boolean' ? verbatim : undefined;
385
+ validateResults(list, hostname) {
386
+ if (list.length === 0) {
387
+ return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
388
+ }
389
+ for (const addr of list) {
390
+ if (addr.family !== 4 && addr.family !== 6) {
391
+ return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
392
+ }
393
+ if (ipBlocker.isBlockedIp(addr.address)) {
394
+ return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
395
+ }
396
+ }
397
+ return null;
413
398
  }
414
- return undefined;
415
- }
416
- function buildLookupOptions(normalizedOptions) {
417
- return {
418
- family: normalizedOptions.family,
419
- hints: normalizedOptions.hints,
420
- all: true,
421
- order: resolveResultOrder(normalizedOptions),
422
- };
423
- }
424
- function createLookupCallback(hostname, resolvedFamily, useAll, callback) {
425
- return (err, addresses) => {
426
- handleLookupResult(err, addresses, hostname, resolvedFamily, useAll, callback);
427
- };
428
- }
429
- function resolveFamily(family) {
430
- if (family === 'IPv4')
431
- return 4;
432
- if (family === 'IPv6')
433
- return 6;
434
- return family;
435
- }
436
- function createLookupTimeout(hostname, callback) {
437
- let done = false;
438
- const timer = setTimeout(() => {
439
- if (done)
440
- return;
441
- done = true;
442
- callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
443
- }, DNS_LOOKUP_TIMEOUT_MS);
444
- timer.unref();
445
- return {
446
- isDone: () => done,
447
- markDone: () => {
399
+ selectResult(list, useAll, hostname) {
400
+ if (list.length === 0) {
401
+ return {
402
+ error: createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA'),
403
+ fallback: [],
404
+ address: [],
405
+ };
406
+ }
407
+ if (useAll)
408
+ return { address: list, fallback: list };
409
+ const first = list.at(0);
410
+ if (!first) {
411
+ return {
412
+ error: createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA'),
413
+ fallback: [],
414
+ address: [],
415
+ };
416
+ }
417
+ return { address: first.address, family: first.family, fallback: list };
418
+ }
419
+ createTimeout(hostname, callback) {
420
+ let done = false;
421
+ const timer = setTimeout(() => {
422
+ if (done)
423
+ return;
448
424
  done = true;
449
- clearTimeout(timer);
450
- },
451
- };
452
- }
453
- function wrapLookupCallback(callback, timeout) {
454
- return (err, address, family) => {
455
- if (timeout.isDone())
456
- return;
457
- timeout.markDone();
458
- callback(err, address, family);
459
- };
460
- }
425
+ callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
426
+ }, DNS_LOOKUP_TIMEOUT_MS);
427
+ timer.unref();
428
+ return {
429
+ isDone: () => done,
430
+ markDone: () => {
431
+ done = true;
432
+ clearTimeout(timer);
433
+ },
434
+ };
435
+ }
436
+ }
437
+ const safeDns = new SafeDnsLookup();
438
+ /* -------------------------------------------------------------------------------------------------
439
+ * Dispatcher / Agent lifecycle
440
+ * ------------------------------------------------------------------------------------------------- */
461
441
  function getAgentOptions() {
462
442
  const cpuCount = os.availableParallelism();
463
443
  return {
464
444
  keepAliveTimeout: 60000,
465
445
  connections: Math.max(cpuCount * 2, 25),
466
446
  pipelining: 1,
467
- connect: { lookup: resolveDns },
447
+ connect: { lookup: safeDns.lookup.bind(safeDns) },
468
448
  };
469
449
  }
470
450
  export const dispatcher = new Agent(getAgentOptions());
471
451
  export function destroyAgents() {
472
452
  void dispatcher.close();
473
453
  }
454
+ /* -------------------------------------------------------------------------------------------------
455
+ * Fetch error mapping (request-level)
456
+ * ------------------------------------------------------------------------------------------------- */
474
457
  function parseRetryAfter(header) {
475
458
  if (!header)
476
459
  return 60;
477
- const parsed = parseInt(header, 10);
460
+ const parsed = Number.parseInt(header, 10);
478
461
  return Number.isNaN(parsed) ? 60 : parsed;
479
462
  }
480
- function createCanceledError(url) {
481
- return new FetchError('Request was canceled', url, 499, {
482
- reason: 'aborted',
483
- });
484
- }
485
- function createTimeoutError(url, timeoutMs) {
486
- return new FetchError(`Request timeout after ${timeoutMs}ms`, url, 504, {
487
- timeout: timeoutMs,
488
- });
489
- }
490
- function createRateLimitError(url, headerValue) {
491
- const retryAfter = parseRetryAfter(headerValue);
492
- return new FetchError('Too many requests', url, 429, { retryAfter });
493
- }
494
- function createHttpError(url, status, statusText) {
495
- return new FetchError(`HTTP ${status}: ${statusText}`, url, status);
496
- }
497
- function createTooManyRedirectsError(url) {
498
- return new FetchError('Too many redirects', url);
499
- }
500
- function createMissingRedirectLocationError(url) {
501
- return new FetchError('Redirect response missing Location header', url);
502
- }
503
- function createSizeLimitError(url, maxBytes) {
504
- return new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
505
- }
506
- function createNetworkError(url, message) {
507
- const details = message ? { message } : undefined;
508
- return new FetchError(`Network error: Could not reach ${url}`, url, undefined, details ?? {});
509
- }
510
- function createUnknownError(url, message) {
511
- return new FetchError(message, url);
463
+ class FetchErrorFactory {
464
+ canceled(url) {
465
+ return new FetchError('Request was canceled', url, 499, {
466
+ reason: 'aborted',
467
+ });
468
+ }
469
+ timeout(url, timeoutMs) {
470
+ return new FetchError(`Request timeout after ${timeoutMs}ms`, url, 504, {
471
+ timeout: timeoutMs,
472
+ });
473
+ }
474
+ rateLimited(url, retryAfterHeader) {
475
+ return new FetchError('Too many requests', url, 429, {
476
+ retryAfter: parseRetryAfter(retryAfterHeader),
477
+ });
478
+ }
479
+ http(url, status, statusText) {
480
+ return new FetchError(`HTTP ${status}: ${statusText}`, url, status);
481
+ }
482
+ tooManyRedirects(url) {
483
+ return new FetchError('Too many redirects', url);
484
+ }
485
+ missingRedirectLocation(url) {
486
+ return new FetchError('Redirect response missing Location header', url);
487
+ }
488
+ sizeLimit(url, maxBytes) {
489
+ return new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
490
+ }
491
+ network(url, message) {
492
+ return new FetchError(`Network error: Could not reach ${url}`, url, undefined, message ? { message } : {});
493
+ }
494
+ unknown(url, message) {
495
+ return new FetchError(message, url);
496
+ }
512
497
  }
498
+ const fetchErrors = new FetchErrorFactory();
513
499
  function isAbortError(error) {
514
500
  return (error instanceof Error &&
515
501
  (error.name === 'AbortError' || error.name === 'TimeoutError'));
@@ -517,350 +503,322 @@ function isAbortError(error) {
517
503
  function isTimeoutError(error) {
518
504
  return error instanceof Error && error.name === 'TimeoutError';
519
505
  }
520
- function getRequestUrl(record) {
521
- const value = record.requestUrl;
522
- return typeof value === 'string' ? value : null;
523
- }
524
506
  function resolveErrorUrl(error, fallback) {
525
507
  if (error instanceof FetchError)
526
508
  return error.url;
527
509
  if (!isObject(error))
528
510
  return fallback;
529
- const requestUrl = getRequestUrl(error);
530
- if (requestUrl)
531
- return requestUrl;
532
- return fallback;
533
- }
534
- function resolveAbortFetchError(error, url, timeoutMs) {
535
- if (!isAbortError(error))
536
- return null;
537
- if (isTimeoutError(error)) {
538
- return createTimeoutError(url, timeoutMs);
539
- }
540
- return createCanceledError(url);
541
- }
542
- function resolveUnexpectedFetchError(error, url) {
543
- if (error instanceof Error) {
544
- return createNetworkError(url, error.message);
545
- }
546
- return createUnknownError(url, 'Unexpected error');
511
+ const { requestUrl } = error;
512
+ return typeof requestUrl === 'string' ? requestUrl : fallback;
547
513
  }
548
514
  function mapFetchError(error, fallbackUrl, timeoutMs) {
549
515
  if (error instanceof FetchError)
550
516
  return error;
551
517
  const url = resolveErrorUrl(error, fallbackUrl);
552
- const abortError = resolveAbortFetchError(error, url, timeoutMs);
553
- if (abortError)
554
- return abortError;
555
- return resolveUnexpectedFetchError(error, url);
556
- }
557
- const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
558
- function publishFetchEvent(event) {
559
- if (!fetchChannel.hasSubscribers)
560
- return;
561
- try {
562
- fetchChannel.publish(event);
563
- }
564
- catch {
565
- // Avoid crashing the publisher if a subscriber throws.
518
+ if (isAbortError(error)) {
519
+ return isTimeoutError(error)
520
+ ? fetchErrors.timeout(url, timeoutMs)
521
+ : fetchErrors.canceled(url);
566
522
  }
523
+ if (error instanceof Error)
524
+ return fetchErrors.network(url, error.message);
525
+ return fetchErrors.unknown(url, 'Unexpected error');
567
526
  }
568
- function buildContextFields(context) {
569
- const fields = {};
570
- if (context.contextRequestId) {
571
- fields.contextRequestId = context.contextRequestId;
572
- }
573
- if (context.operationId) {
574
- fields.operationId = context.operationId;
527
+ const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
528
+ const SLOW_REQUEST_THRESHOLD_MS = 5000;
529
+ class FetchTelemetry {
530
+ start(url, method) {
531
+ const safeUrl = redactUrl(url);
532
+ const contextRequestId = getRequestId();
533
+ const operationId = getOperationId();
534
+ const ctx = {
535
+ requestId: randomUUID(),
536
+ startTime: performance.now(),
537
+ url: safeUrl,
538
+ method: method.toUpperCase(),
539
+ ...(contextRequestId ? { contextRequestId } : {}),
540
+ ...(operationId ? { operationId } : {}),
541
+ };
542
+ this.publish({
543
+ v: 1,
544
+ type: 'start',
545
+ requestId: ctx.requestId,
546
+ method: ctx.method,
547
+ url: ctx.url,
548
+ ...(ctx.contextRequestId
549
+ ? { contextRequestId: ctx.contextRequestId }
550
+ : {}),
551
+ ...(ctx.operationId ? { operationId: ctx.operationId } : {}),
552
+ });
553
+ logDebug('HTTP Request', {
554
+ requestId: ctx.requestId,
555
+ method: ctx.method,
556
+ url: ctx.url,
557
+ ...(ctx.contextRequestId
558
+ ? { contextRequestId: ctx.contextRequestId }
559
+ : {}),
560
+ ...(ctx.operationId ? { operationId: ctx.operationId } : {}),
561
+ });
562
+ return ctx;
563
+ }
564
+ recordResponse(context, response, contentSize) {
565
+ const duration = performance.now() - context.startTime;
566
+ const durationLabel = `${Math.round(duration)}ms`;
567
+ this.publish({
568
+ v: 1,
569
+ type: 'end',
570
+ requestId: context.requestId,
571
+ status: response.status,
572
+ duration,
573
+ ...(context.contextRequestId
574
+ ? { contextRequestId: context.contextRequestId }
575
+ : {}),
576
+ ...(context.operationId ? { operationId: context.operationId } : {}),
577
+ });
578
+ const contentType = response.headers.get('content-type') ?? undefined;
579
+ const contentLengthHeader = response.headers.get('content-length');
580
+ const size = contentLengthHeader ??
581
+ (contentSize === undefined ? undefined : String(contentSize));
582
+ logDebug('HTTP Response', {
583
+ requestId: context.requestId,
584
+ status: response.status,
585
+ url: context.url,
586
+ duration: durationLabel,
587
+ ...(context.contextRequestId
588
+ ? { contextRequestId: context.contextRequestId }
589
+ : {}),
590
+ ...(context.operationId ? { operationId: context.operationId } : {}),
591
+ ...(contentType ? { contentType } : {}),
592
+ ...(size ? { size } : {}),
593
+ });
594
+ if (duration > SLOW_REQUEST_THRESHOLD_MS) {
595
+ logWarn('Slow HTTP request detected', {
596
+ requestId: context.requestId,
597
+ url: context.url,
598
+ duration: durationLabel,
599
+ ...(context.contextRequestId
600
+ ? { contextRequestId: context.contextRequestId }
601
+ : {}),
602
+ ...(context.operationId ? { operationId: context.operationId } : {}),
603
+ });
604
+ }
575
605
  }
576
- return fields;
577
- }
578
- function buildResponseMetadata(response, contentSize) {
579
- const contentType = response.headers.get('content-type') ?? undefined;
580
- const contentLengthHeader = response.headers.get('content-length');
581
- const size = contentLengthHeader ??
582
- (contentSize === undefined ? undefined : String(contentSize));
583
- const metadata = {};
584
- if (contentType)
585
- metadata.contentType = contentType;
586
- if (size)
587
- metadata.size = size;
588
- return metadata;
589
- }
590
- function logSlowRequest(context, duration, durationLabel, contextFields) {
591
- if (duration <= SLOW_REQUEST_THRESHOLD_MS)
592
- return;
593
- logWarn('Slow HTTP request detected', {
594
- requestId: context.requestId,
595
- url: context.url,
596
- duration: durationLabel,
597
- ...contextFields,
598
- });
599
- }
600
- function resolveSystemErrorCode(error) {
601
- return isSystemError(error) ? error.code : undefined;
602
- }
603
- function buildFetchErrorEvent(context, err, duration, contextFields, status, code) {
604
- const event = {
605
- v: 1,
606
- type: 'error',
607
- requestId: context.requestId,
608
- url: context.url,
609
- error: err.message,
610
- duration,
611
- ...contextFields,
612
- };
613
- if (code !== undefined) {
614
- event.code = code;
606
+ recordError(context, error, status) {
607
+ const duration = performance.now() - context.startTime;
608
+ const err = error instanceof Error ? error : new Error(String(error));
609
+ const code = isSystemError(err) ? err.code : undefined;
610
+ this.publish({
611
+ v: 1,
612
+ type: 'error',
613
+ requestId: context.requestId,
614
+ url: context.url,
615
+ error: err.message,
616
+ duration,
617
+ ...(code !== undefined ? { code } : {}),
618
+ ...(status !== undefined ? { status } : {}),
619
+ ...(context.contextRequestId
620
+ ? { contextRequestId: context.contextRequestId }
621
+ : {}),
622
+ ...(context.operationId ? { operationId: context.operationId } : {}),
623
+ });
624
+ const log = status === 429 ? logWarn : logError;
625
+ log('HTTP Request Error', {
626
+ requestId: context.requestId,
627
+ url: context.url,
628
+ status,
629
+ code,
630
+ error: err.message,
631
+ ...(context.contextRequestId
632
+ ? { contextRequestId: context.contextRequestId }
633
+ : {}),
634
+ ...(context.operationId ? { operationId: context.operationId } : {}),
635
+ });
615
636
  }
616
- if (status !== undefined) {
617
- event.status = status;
637
+ publish(event) {
638
+ if (!fetchChannel.hasSubscribers)
639
+ return;
640
+ try {
641
+ fetchChannel.publish(event);
642
+ }
643
+ catch {
644
+ // Best-effort; subscriber failures must not crash request path.
645
+ }
618
646
  }
619
- return event;
620
- }
621
- function createTelemetryContext(url, method) {
622
- const safeUrl = redactUrl(url);
623
- const contextRequestId = getRequestId();
624
- const operationId = getOperationId();
625
- return {
626
- requestId: randomUUID(),
627
- startTime: performance.now(),
628
- url: safeUrl,
629
- method: method.toUpperCase(),
630
- ...(contextRequestId ? { contextRequestId } : {}),
631
- ...(operationId ? { operationId } : {}),
632
- };
633
647
  }
648
+ const telemetry = new FetchTelemetry();
649
+ /** Backwards-compatible exports */
634
650
  export function startFetchTelemetry(url, method) {
635
- const context = createTelemetryContext(url, method);
636
- const contextFields = buildContextFields(context);
637
- publishFetchEvent({
638
- v: 1,
639
- type: 'start',
640
- requestId: context.requestId,
641
- method: context.method,
642
- url: context.url,
643
- ...contextFields,
644
- });
645
- logDebug('HTTP Request', {
646
- requestId: context.requestId,
647
- method: context.method,
648
- url: context.url,
649
- ...contextFields,
650
- });
651
- return context;
651
+ return telemetry.start(url, method);
652
652
  }
653
653
  export function recordFetchResponse(context, response, contentSize) {
654
- const duration = performance.now() - context.startTime;
655
- const durationLabel = `${Math.round(duration)}ms`;
656
- const contextFields = buildContextFields(context);
657
- const responseMetadata = buildResponseMetadata(response, contentSize);
658
- publishFetchEvent({
659
- v: 1,
660
- type: 'end',
661
- requestId: context.requestId,
662
- status: response.status,
663
- duration,
664
- ...contextFields,
665
- });
666
- logDebug('HTTP Response', {
667
- requestId: context.requestId,
668
- status: response.status,
669
- url: context.url,
670
- duration: durationLabel,
671
- ...contextFields,
672
- ...responseMetadata,
673
- });
674
- logSlowRequest(context, duration, durationLabel, contextFields);
654
+ telemetry.recordResponse(context, response, contentSize);
675
655
  }
676
656
  export function recordFetchError(context, error, status) {
677
- const duration = performance.now() - context.startTime;
678
- const err = error instanceof Error ? error : new Error(String(error));
679
- const contextFields = buildContextFields(context);
680
- const code = resolveSystemErrorCode(err);
681
- publishFetchEvent(buildFetchErrorEvent(context, err, duration, contextFields, status, code));
682
- const log = status === 429 ? logWarn : logError;
683
- log('HTTP Request Error', {
684
- requestId: context.requestId,
685
- url: context.url,
686
- status,
687
- code,
688
- error: err.message,
689
- ...contextFields,
690
- });
657
+ telemetry.recordError(context, error, status);
691
658
  }
659
+ /* -------------------------------------------------------------------------------------------------
660
+ * Redirect handling
661
+ * ------------------------------------------------------------------------------------------------- */
692
662
  const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
693
663
  function isRedirectStatus(status) {
694
664
  return REDIRECT_STATUSES.has(status);
695
665
  }
696
666
  function cancelResponseBody(response) {
697
667
  const cancelPromise = response.body?.cancel();
698
- if (cancelPromise) {
668
+ if (cancelPromise)
699
669
  cancelPromise.catch(() => {
700
- // Best-effort cancellation; ignore failures.
670
+ /* ignore */
701
671
  });
702
- }
703
672
  }
704
- async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount) {
705
- const response = await fetch(currentUrl, { ...init, redirect: 'manual' });
706
- if (!isRedirectStatus(response.status)) {
707
- return { response };
673
+ class RedirectFollower {
674
+ async fetchWithRedirects(url, init, maxRedirects) {
675
+ let currentUrl = url;
676
+ const redirectLimit = Math.max(0, maxRedirects);
677
+ for (let redirectCount = 0; redirectCount <= redirectLimit; redirectCount += 1) {
678
+ const { response, nextUrl } = await this.withRedirectErrorContext(currentUrl, async () => this.performFetchCycle(currentUrl, init, redirectLimit, redirectCount));
679
+ if (!nextUrl)
680
+ return { response, url: currentUrl };
681
+ currentUrl = nextUrl;
682
+ }
683
+ throw fetchErrors.tooManyRedirects(currentUrl);
684
+ }
685
+ async performFetchCycle(currentUrl, init, redirectLimit, redirectCount) {
686
+ const response = await fetch(currentUrl, { ...init, redirect: 'manual' });
687
+ if (!isRedirectStatus(response.status))
688
+ return { response };
689
+ this.assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount);
690
+ const location = this.getRedirectLocation(response, currentUrl);
691
+ cancelResponseBody(response);
692
+ return {
693
+ response,
694
+ nextUrl: this.resolveRedirectTarget(currentUrl, location),
695
+ };
708
696
  }
709
- assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount);
710
- const location = getRedirectLocation(response, currentUrl);
711
- cancelResponseBody(response);
712
- return {
713
- response,
714
- nextUrl: resolveRedirectTarget(currentUrl, location),
715
- };
716
- }
717
- function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
718
- if (redirectCount < redirectLimit)
719
- return;
720
- cancelResponseBody(response);
721
- throw createTooManyRedirectsError(currentUrl);
722
- }
723
- function getRedirectLocation(response, currentUrl) {
724
- const location = response.headers.get('location');
725
- if (location)
726
- return location;
727
- cancelResponseBody(response);
728
- throw createMissingRedirectLocationError(currentUrl);
729
- }
730
- function annotateRedirectError(error, url) {
731
- if (!isObject(error))
732
- return;
733
- error.requestUrl = url;
734
- }
735
- function resolveRedirectTarget(baseUrl, location) {
736
- if (!URL.canParse(location, baseUrl)) {
737
- throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
697
+ assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
698
+ if (redirectCount < redirectLimit)
699
+ return;
700
+ cancelResponseBody(response);
701
+ throw fetchErrors.tooManyRedirects(currentUrl);
738
702
  }
739
- const resolved = new URL(location, baseUrl);
740
- if (resolved.username || resolved.password) {
741
- throw createErrorWithCode('Redirect target includes credentials', 'EBADREDIRECT');
703
+ getRedirectLocation(response, currentUrl) {
704
+ const location = response.headers.get('location');
705
+ if (location)
706
+ return location;
707
+ cancelResponseBody(response);
708
+ throw fetchErrors.missingRedirectLocation(currentUrl);
709
+ }
710
+ resolveRedirectTarget(baseUrl, location) {
711
+ if (!URL.canParse(location, baseUrl))
712
+ throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
713
+ const resolved = new URL(location, baseUrl);
714
+ if (resolved.username || resolved.password) {
715
+ throw createErrorWithCode('Redirect target includes credentials', 'EBADREDIRECT');
716
+ }
717
+ return validateAndNormalizeUrl(resolved.href);
742
718
  }
743
- return validateAndNormalizeUrl(resolved.href);
744
- }
745
- async function withRedirectErrorContext(url, fn) {
746
- try {
747
- return await fn();
719
+ annotateRedirectError(error, url) {
720
+ if (!isObject(error))
721
+ return;
722
+ error.requestUrl = url;
748
723
  }
749
- catch (error) {
750
- annotateRedirectError(error, url);
751
- throw error;
724
+ async withRedirectErrorContext(url, fn) {
725
+ try {
726
+ return await fn();
727
+ }
728
+ catch (error) {
729
+ this.annotateRedirectError(error, url);
730
+ throw error;
731
+ }
752
732
  }
753
733
  }
734
+ const redirectFollower = new RedirectFollower();
735
+ /** Backwards-compatible export */
754
736
  export async function fetchWithRedirects(url, init, maxRedirects) {
755
- let currentUrl = url;
756
- const redirectLimit = Math.max(0, maxRedirects);
757
- for (let redirectCount = 0; redirectCount <= redirectLimit; redirectCount += 1) {
758
- const { response, nextUrl } = await withRedirectErrorContext(currentUrl, () => performFetchCycle(currentUrl, init, redirectLimit, redirectCount));
759
- if (!nextUrl) {
760
- return { response, url: currentUrl };
761
- }
762
- currentUrl = nextUrl;
763
- }
764
- throw createTooManyRedirectsError(currentUrl);
737
+ return redirectFollower.fetchWithRedirects(url, init, maxRedirects);
765
738
  }
739
+ /* -------------------------------------------------------------------------------------------------
740
+ * Response reading (max size + abort-aware streaming)
741
+ * ------------------------------------------------------------------------------------------------- */
766
742
  function assertContentLengthWithinLimit(response, url, maxBytes) {
767
- const contentLengthHeader = response.headers.get('content-length');
768
- if (!contentLengthHeader)
743
+ const header = response.headers.get('content-length');
744
+ if (!header)
769
745
  return;
770
- const contentLength = Number.parseInt(contentLengthHeader, 10);
771
- if (Number.isNaN(contentLength) || contentLength <= maxBytes) {
746
+ const contentLength = Number.parseInt(header, 10);
747
+ if (Number.isNaN(contentLength) || contentLength <= maxBytes)
772
748
  return;
773
- }
774
749
  cancelResponseBody(response);
775
- throw createSizeLimitError(url, maxBytes);
776
- }
777
- function createReadState() {
778
- return {
779
- decoder: new TextDecoder(),
780
- parts: [],
781
- total: 0,
782
- };
783
- }
784
- function appendChunk(state, chunk, maxBytes, url) {
785
- state.total += chunk.byteLength;
786
- if (state.total > maxBytes) {
787
- throw createSizeLimitError(url, maxBytes);
788
- }
789
- const decoded = state.decoder.decode(chunk, { stream: true });
790
- if (decoded)
791
- state.parts.push(decoded);
792
- }
793
- function finalizeRead(state) {
794
- const decoded = state.decoder.decode();
795
- if (decoded)
796
- state.parts.push(decoded);
797
- }
798
- function createAbortError(url) {
799
- return new FetchError('Request was aborted during response read', url, 499, {
800
- reason: 'aborted',
801
- });
802
- }
803
- async function cancelReaderQuietly(reader) {
804
- try {
805
- await reader.cancel();
806
- }
807
- catch {
808
- // Ignore cancel errors; we're already failing this read.
809
- }
810
- }
811
- async function throwIfAborted(signal, url, reader) {
812
- if (!signal?.aborted)
813
- return;
814
- await cancelReaderQuietly(reader);
815
- throw createAbortError(url);
816
- }
817
- async function handleReadFailure(error, signal, url, reader) {
818
- const aborted = signal?.aborted ?? false;
819
- await cancelReaderQuietly(reader);
820
- if (aborted) {
821
- throw createAbortError(url);
822
- }
823
- throw error;
824
- }
825
- async function readAllChunks(reader, state, url, maxBytes, signal) {
826
- await throwIfAborted(signal, url, reader);
827
- let result = await reader.read();
828
- while (!result.done) {
829
- appendChunk(state, result.value, maxBytes, url);
830
- await throwIfAborted(signal, url, reader);
831
- result = await reader.read();
832
- }
833
- }
834
- async function readStreamWithLimit(stream, url, maxBytes, signal) {
835
- const state = createReadState();
836
- const reader = stream.getReader();
837
- try {
838
- await readAllChunks(reader, state, url, maxBytes, signal);
839
- }
840
- catch (error) {
841
- await handleReadFailure(error, signal, url, reader);
750
+ throw fetchErrors.sizeLimit(url, maxBytes);
751
+ }
752
+ class ResponseTextReader {
753
+ async read(response, url, maxBytes, signal) {
754
+ assertContentLengthWithinLimit(response, url, maxBytes);
755
+ if (!response.body) {
756
+ const text = await response.text();
757
+ const size = Buffer.byteLength(text);
758
+ if (size > maxBytes)
759
+ throw fetchErrors.sizeLimit(url, maxBytes);
760
+ return { text, size };
761
+ }
762
+ return this.readStreamWithLimit(response.body, url, maxBytes, signal);
763
+ }
764
+ async readStreamWithLimit(stream, url, maxBytes, signal) {
765
+ const decoder = new TextDecoder();
766
+ const parts = [];
767
+ let total = 0;
768
+ const reader = stream.getReader();
769
+ try {
770
+ await this.throwIfAborted(signal, url, reader);
771
+ let result = await reader.read();
772
+ while (!result.done) {
773
+ total += result.value.byteLength;
774
+ if (total > maxBytes)
775
+ throw fetchErrors.sizeLimit(url, maxBytes);
776
+ const decoded = decoder.decode(result.value, { stream: true });
777
+ if (decoded)
778
+ parts.push(decoded);
779
+ await this.throwIfAborted(signal, url, reader);
780
+ result = await reader.read();
781
+ }
782
+ }
783
+ catch (error) {
784
+ await this.cancelReaderQuietly(reader);
785
+ if (signal?.aborted)
786
+ throw new FetchError('Request was aborted during response read', url, 499, { reason: 'aborted' });
787
+ throw error;
788
+ }
789
+ finally {
790
+ reader.releaseLock();
791
+ }
792
+ const final = decoder.decode();
793
+ if (final)
794
+ parts.push(final);
795
+ return { text: parts.join(''), size: total };
842
796
  }
843
- finally {
844
- reader.releaseLock();
797
+ async throwIfAborted(signal, url, reader) {
798
+ if (!signal?.aborted)
799
+ return;
800
+ await this.cancelReaderQuietly(reader);
801
+ throw new FetchError('Request was aborted during response read', url, 499, {
802
+ reason: 'aborted',
803
+ });
845
804
  }
846
- finalizeRead(state);
847
- return { text: state.parts.join(''), size: state.total };
848
- }
849
- async function readResponseTextFallback(response, url, maxBytes) {
850
- const text = await response.text();
851
- const size = Buffer.byteLength(text);
852
- if (size > maxBytes) {
853
- throw createSizeLimitError(url, maxBytes);
805
+ async cancelReaderQuietly(reader) {
806
+ try {
807
+ await reader.cancel();
808
+ }
809
+ catch {
810
+ // ignore
811
+ }
854
812
  }
855
- return { text, size };
856
813
  }
814
+ const responseReader = new ResponseTextReader();
815
+ /** Backwards-compatible export */
857
816
  export async function readResponseText(response, url, maxBytes, signal) {
858
- assertContentLengthWithinLimit(response, url, maxBytes);
859
- if (!response.body) {
860
- return readResponseTextFallback(response, url, maxBytes);
861
- }
862
- return readStreamWithLimit(response.body, url, maxBytes, signal);
817
+ return responseReader.read(response, url, maxBytes, signal);
863
818
  }
819
+ /* -------------------------------------------------------------------------------------------------
820
+ * HTTP fetcher (headers, signals, response handling)
821
+ * ------------------------------------------------------------------------------------------------- */
864
822
  const DEFAULT_HEADERS = {
865
823
  'User-Agent': config.fetcher.userAgent,
866
824
  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
@@ -873,63 +831,51 @@ function buildHeaders() {
873
831
  }
874
832
  function buildRequestSignal(timeoutMs, external) {
875
833
  const timeoutSignal = AbortSignal.timeout(timeoutMs);
876
- if (!external)
877
- return timeoutSignal;
878
- return AbortSignal.any([external, timeoutSignal]);
834
+ return external ? AbortSignal.any([external, timeoutSignal]) : timeoutSignal;
879
835
  }
880
836
  function buildRequestInit(headers, signal) {
881
- return {
882
- method: 'GET',
883
- headers,
884
- signal,
885
- dispatcher,
886
- };
837
+ return { method: 'GET', headers, signal, dispatcher };
887
838
  }
888
839
  function resolveResponseError(response, finalUrl) {
889
- return (resolveRateLimitError(response, finalUrl) ??
890
- resolveHttpError(response, finalUrl));
891
- }
892
- function resolveRateLimitError(response, finalUrl) {
893
- return response.status === 429
894
- ? createRateLimitError(finalUrl, response.headers.get('retry-after'))
895
- : null;
896
- }
897
- function resolveHttpError(response, finalUrl) {
840
+ if (response.status === 429) {
841
+ return fetchErrors.rateLimited(finalUrl, response.headers.get('retry-after'));
842
+ }
898
843
  return response.ok
899
844
  ? null
900
- : createHttpError(finalUrl, response.status, response.statusText);
845
+ : fetchErrors.http(finalUrl, response.status, response.statusText);
901
846
  }
902
- async function handleFetchResponse(response, finalUrl, telemetry, signal) {
847
+ async function handleFetchResponse(response, finalUrl, ctx, signal) {
903
848
  const responseError = resolveResponseError(response, finalUrl);
904
849
  if (responseError) {
905
850
  cancelResponseBody(response);
906
851
  throw responseError;
907
852
  }
908
- const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength, signal);
909
- recordFetchResponse(telemetry, response, size);
853
+ const { text, size } = await responseReader.read(response, finalUrl, config.fetcher.maxContentLength, signal);
854
+ telemetry.recordResponse(ctx, response, size);
910
855
  return text;
911
856
  }
912
- async function fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs) {
913
- const telemetry = startFetchTelemetry(normalizedUrl, 'GET');
914
- try {
915
- return await fetchAndHandle(normalizedUrl, requestInit, telemetry);
916
- }
917
- catch (error) {
918
- const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
919
- telemetry.url = mapped.url;
920
- recordFetchError(telemetry, mapped, mapped.statusCode);
921
- throw mapped;
857
+ class HttpFetcher {
858
+ async fetchNormalizedUrl(normalizedUrl, options) {
859
+ const timeoutMs = config.fetcher.timeout;
860
+ const headers = buildHeaders();
861
+ const signal = buildRequestSignal(timeoutMs, options?.signal);
862
+ const init = buildRequestInit(headers, signal);
863
+ const ctx = telemetry.start(normalizedUrl, 'GET');
864
+ try {
865
+ const { response, url: finalUrl } = await redirectFollower.fetchWithRedirects(normalizedUrl, init, config.fetcher.maxRedirects);
866
+ ctx.url = finalUrl;
867
+ return await handleFetchResponse(response, finalUrl, ctx, init.signal ?? undefined);
868
+ }
869
+ catch (error) {
870
+ const mapped = mapFetchError(error, normalizedUrl, timeoutMs);
871
+ ctx.url = mapped.url;
872
+ telemetry.recordError(ctx, mapped, mapped.statusCode);
873
+ throw mapped;
874
+ }
922
875
  }
923
876
  }
924
- async function fetchAndHandle(normalizedUrl, requestInit, telemetry) {
925
- const { response, url: finalUrl } = await fetchWithRedirects(normalizedUrl, requestInit, config.fetcher.maxRedirects);
926
- telemetry.url = finalUrl;
927
- return handleFetchResponse(response, finalUrl, telemetry, requestInit.signal ?? undefined);
928
- }
877
+ const httpFetcher = new HttpFetcher();
878
+ /** Backwards-compatible export */
929
879
  export async function fetchNormalizedUrl(normalizedUrl, options) {
930
- const timeoutMs = config.fetcher.timeout;
931
- const headers = buildHeaders();
932
- const signal = buildRequestSignal(timeoutMs, options?.signal);
933
- const requestInit = buildRequestInit(headers, signal);
934
- return fetchWithTelemetry(normalizedUrl, requestInit, timeoutMs);
880
+ return httpFetcher.fetchNormalizedUrl(normalizedUrl, options);
935
881
  }