@j0hanz/superfetch 1.2.5 → 2.0.1

package/dist/services/fetcher/agents.js

@@ -1,27 +1,15 @@
 import dns from 'node:dns';
 import os from 'node:os';
 import { Agent } from 'undici';
-import { createErrorWithCode } from '../../utils/error-utils.js';
-import { isBlockedIp } from '../../utils/url-validator.js';
+import { createErrorWithCode } from '../../utils/error-details.js';
+import { isRecord } from '../../utils/guards.js';
+import { handleLookupResult } from './dns-selection.js';
 const DNS_LOOKUP_TIMEOUT_MS = 5000;
 function resolveDns(hostname, options, callback) {
     const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
     const lookupOptions = buildLookupOptions(normalizedOptions);
-    let done = false;
-    const timer = setTimeout(() => {
-        if (done)
-            return;
-        done = true;
-        callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
-    }, DNS_LOOKUP_TIMEOUT_MS);
-    timer.unref();
-    const safeCallback = (err, address, family) => {
-        if (done)
-            return;
-        done = true;
-        clearTimeout(timer);
-        callback(err, address, family);
-    };
+    const timeout = createLookupTimeout(hostname, callback);
+    const safeCallback = wrapLookupCallback(callback, timeout);
     dns.lookup(hostname, lookupOptions, createLookupCallback(hostname, resolvedFamily, useAll, safeCallback));
 }
 function normalizeLookupOptions(options) {
@@ -46,17 +34,19 @@ function resolveResultOrder(options) {
     return DEFAULT_DNS_ORDER;
 }
 function getLegacyVerbatim(options) {
-    const legacy = options.verbatim;
-    return typeof legacy === 'boolean' ? legacy : undefined;
+    if (isRecord(options)) {
+        const { verbatim } = options;
+        return typeof verbatim === 'boolean' ? verbatim : undefined;
+    }
+    return undefined;
 }
 function buildLookupOptions(normalizedOptions) {
-    const options = {
-        ...normalizedOptions,
-        order: resolveResultOrder(normalizedOptions),
+    return {
+        family: normalizedOptions.family,
+        hints: normalizedOptions.hints,
         all: true,
+        order: resolveResultOrder(normalizedOptions),
     };
-    delete options.verbatim;
-    return options;
 }
 function createLookupCallback(hostname, resolvedFamily, useAll, callback) {
     return (err, addresses) => {
@@ -70,81 +60,30 @@ function resolveFamily(family) {
         return 6;
     return family;
 }
-function normalizeLookupResults(addresses, family) {
-    if (Array.isArray(addresses)) {
-        return addresses;
-    }
-    return [{ address: addresses, family: family ?? 4 }];
-}
-function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
-    if (error) {
-        callback(error, addresses);
-        return;
-    }
-    const list = normalizeLookupResults(addresses, resolvedFamily);
-    const invalidFamilyError = findInvalidFamilyError(list, hostname);
-    if (invalidFamilyError) {
-        callback(invalidFamilyError, list);
-        return;
-    }
-    const blockedError = findBlockedIpError(list, hostname);
-    if (blockedError) {
-        callback(blockedError, list);
-        return;
-    }
-    const selection = selectLookupResult(list, useAll, hostname);
-    if (selection.error) {
-        callback(selection.error, selection.fallback);
-        return;
-    }
-    callback(null, selection.address, selection.family);
-}
-function selectLookupResult(list, useAll, hostname) {
-    if (list.length === 0) {
-        return {
-            error: createNoDnsResultsError(hostname),
-            fallback: [],
-            address: [],
-        };
-    }
-    if (useAll) {
-        return { address: list, fallback: list };
-    }
-    const first = list.at(0);
-    if (!first) {
-        return {
-            error: createNoDnsResultsError(hostname),
-            fallback: [],
-            address: [],
-        };
-    }
+function createLookupTimeout(hostname, callback) {
+    let done = false;
+    const timer = setTimeout(() => {
+        if (done)
+            return;
+        done = true;
+        callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
+    }, DNS_LOOKUP_TIMEOUT_MS);
+    timer.unref();
     return {
-        address: first.address,
-        family: first.family,
-        fallback: list,
+        isDone: () => done,
+        markDone: () => {
+            done = true;
+            clearTimeout(timer);
+        },
     };
 }
-function findBlockedIpError(list, hostname) {
-    for (const addr of list) {
-        const ip = typeof addr === 'string' ? addr : addr.address;
-        if (!isBlockedIp(ip)) {
-            continue;
-        }
-        return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
-    }
-    return null;
-}
-function findInvalidFamilyError(list, hostname) {
-    for (const addr of list) {
-        const family = typeof addr === 'string' ? 0 : addr.family;
-        if (family === 4 || family === 6)
-            continue;
-        return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
-    }
-    return null;
-}
-function createNoDnsResultsError(hostname) {
-    return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
+function wrapLookupCallback(callback, timeout) {
+    return (err, address, family) => {
+        if (timeout.isDone())
+            return;
+        timeout.markDone();
+        callback(err, address, family);
+    };
 }
 function getAgentOptions() {
     const cpuCount = os.availableParallelism();

package/dist/services/fetcher/dns-selection.d.ts

@@ -0,0 +1,2 @@
+import type { LookupAddress } from 'node:dns';
+export declare function handleLookupResult(error: NodeJS.ErrnoException | null, addresses: string | LookupAddress[], hostname: string, resolvedFamily: number | undefined, useAll: boolean, callback: (err: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void): void;

package/dist/services/fetcher/dns-selection.js

@@ -0,0 +1,72 @@
+import { createErrorWithCode } from '../../utils/error-details.js';
+import { isBlockedIp } from '../../utils/url-validator.js';
+function normalizeLookupResults(addresses, family) {
+    if (Array.isArray(addresses)) {
+        return addresses;
+    }
+    return [{ address: addresses, family: family ?? 4 }];
+}
+function findBlockedIpError(list, hostname) {
+    for (const addr of list) {
+        const ip = typeof addr === 'string' ? addr : addr.address;
+        if (!isBlockedIp(ip)) {
+            continue;
+        }
+        return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
+    }
+    return null;
+}
+function findInvalidFamilyError(list, hostname) {
+    for (const addr of list) {
+        const family = typeof addr === 'string' ? 0 : addr.family;
+        if (family === 4 || family === 6)
+            continue;
+        return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
+    }
+    return null;
+}
+function createNoDnsResultsError(hostname) {
+    return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
+}
+function createEmptySelection(hostname) {
+    return {
+        error: createNoDnsResultsError(hostname),
+        fallback: [],
+        address: [],
+    };
+}
+function selectLookupResult(list, useAll, hostname) {
+    if (list.length === 0)
+        return createEmptySelection(hostname);
+    if (useAll)
+        return { address: list, fallback: list };
+    const first = list.at(0);
+    if (!first)
+        return createEmptySelection(hostname);
+    return {
+        address: first.address,
+        family: first.family,
+        fallback: list,
+    };
+}
+function findLookupError(list, hostname) {
+    return (findInvalidFamilyError(list, hostname) ?? findBlockedIpError(list, hostname));
+}
+export function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
+    if (error) {
+        callback(error, addresses);
+        return;
+    }
+    const list = normalizeLookupResults(addresses, resolvedFamily);
+    const lookupError = findLookupError(list, hostname);
+    if (lookupError) {
+        callback(lookupError, list);
+        return;
+    }
+    const selection = selectLookupResult(list, useAll, hostname);
+    if (selection.error) {
+        callback(selection.error, selection.fallback);
+        return;
+    }
+    callback(null, selection.address, selection.family);
+}

package/dist/services/fetcher/interceptors.d.ts

@@ -1,25 +1,3 @@
-export type FetchChannelEvent = {
-    v: 1;
-    type: 'start';
-    requestId: string;
-    method: string;
-    url: string;
-} | {
-    v: 1;
-    type: 'end';
-    requestId: string;
-    status: number;
-    duration: number;
-} | {
-    v: 1;
-    type: 'error';
-    requestId: string;
-    url: string;
-    error: string;
-    code?: string;
-    status?: number;
-    duration: number;
-};
 interface FetchTelemetryContext {
     requestId: string;
     startTime: number;

package/dist/services/fetcher/interceptors.js

@@ -1,7 +1,7 @@
 import { randomUUID } from 'node:crypto';
 import diagnosticsChannel from 'node:diagnostics_channel';
 import { performance } from 'node:perf_hooks';
-import { isSystemError } from '../../utils/error-utils.js';
+import { isSystemError } from '../../utils/error-details.js';
 import { logDebug, logError, logWarn } from '../logger.js';
 const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
 function redactUrl(rawUrl) {
@@ -51,51 +51,36 @@ export function startFetchTelemetry(url, method) {
 }
 export function recordFetchResponse(context, response, contentSize) {
     const duration = performance.now() - context.startTime;
-    publishFetchEnd(context, response.status, duration);
-    logDebug('HTTP Response', {
-        requestId: context.requestId,
-        status: response.status,
-        url: context.url,
-        ...buildResponseMeta(response, contentSize, duration),
-    });
-    logSlowRequestIfNeeded(context, duration);
-}
-function publishFetchEnd(context, status, duration) {
+    const durationLabel = `${Math.round(duration)}ms`;
     publishFetchEvent({
         v: 1,
         type: 'end',
         requestId: context.requestId,
-        status,
+        status: response.status,
         duration,
     });
-}
-function buildResponseMeta(response, contentSize, duration) {
-    const contentLength = response.headers.get('content-length') ?? contentSize?.toString();
-    const meta = {
-        duration: `${Math.round(duration)}ms`,
-    };
     const contentType = response.headers.get('content-type');
-    if (contentType !== null) {
-        meta.contentType = contentType;
-    }
-    if (contentLength !== undefined) {
-        meta.size = contentLength;
-    }
-    return meta;
-}
-function logSlowRequestIfNeeded(context, duration) {
-    if (duration <= 5000)
-        return;
-    logWarn('Slow HTTP request detected', {
+    const contentLength = response.headers.get('content-length') ??
+        (contentSize === undefined ? undefined : String(contentSize));
+    logDebug('HTTP Response', {
         requestId: context.requestId,
+        status: response.status,
         url: context.url,
-        duration: `${Math.round(duration)}ms`,
+        duration: durationLabel,
+        ...(contentType ? { contentType } : {}),
+        ...(contentLength ? { size: contentLength } : {}),
     });
+    if (duration > 5000) {
+        logWarn('Slow HTTP request detected', {
+            requestId: context.requestId,
+            url: context.url,
+            duration: durationLabel,
+        });
+    }
 }
 export function recordFetchError(context, error, status) {
     const duration = performance.now() - context.startTime;
     const err = error instanceof Error ? error : new Error(String(error));
-    const code = isSystemError(err) ? err.code : undefined;
     const event = {
         v: 1,
         type: 'error',
@@ -104,6 +89,7 @@ export function recordFetchError(context, error, status) {
         error: err.message,
         duration,
     };
+    const code = isSystemError(err) ? err.code : undefined;
     if (code !== undefined) {
         event.code = code;
     }

package/dist/services/fetcher/redirects.js

@@ -1,10 +1,19 @@
 import { FetchError } from '../../errors/app-error.js';
-import { createErrorWithCode } from '../../utils/error-utils.js';
+import { createErrorWithCode } from '../../utils/error-details.js';
+import { isRecord } from '../../utils/guards.js';
 import { validateAndNormalizeUrl } from '../../utils/url-validator.js';
 const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 function isRedirectStatus(status) {
     return REDIRECT_STATUSES.has(status);
 }
+function cancelResponseBody(response) {
+    const cancelPromise = response.body?.cancel();
+    if (cancelPromise) {
+        cancelPromise.catch(() => {
+            // Best-effort cancellation; ignore failures.
+        });
+    }
+}
 async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount) {
     const response = await fetch(currentUrl, { ...init, redirect: 'manual' });
     if (!isRedirectStatus(response.status)) {
@@ -12,31 +21,31 @@ async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount)
     }
     assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount);
     const location = getRedirectLocation(response, currentUrl);
-    void response.body?.cancel();
+    cancelResponseBody(response);
     return {
         response,
-        nextUrl: await resolveRedirectTarget(currentUrl, location),
+        nextUrl: resolveRedirectTarget(currentUrl, location),
     };
 }
 function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
     if (redirectCount < redirectLimit)
         return;
-    void response.body?.cancel();
+    cancelResponseBody(response);
     throw new FetchError('Too many redirects', currentUrl);
 }
 function getRedirectLocation(response, currentUrl) {
     const location = response.headers.get('location');
     if (location)
         return location;
-    void response.body?.cancel();
+    cancelResponseBody(response);
     throw new FetchError('Redirect response missing Location header', currentUrl);
 }
 function annotateRedirectError(error, url) {
-    if (!error || typeof error !== 'object')
+    if (!isRecord(error))
         return;
     error.requestUrl = url;
 }
-async function resolveRedirectTarget(baseUrl, location) {
+function resolveRedirectTarget(baseUrl, location) {
     if (!URL.canParse(location, baseUrl)) {
         throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
     }

package/dist/services/fetcher/response.js

@@ -1,5 +1,3 @@
-import { Readable, Writable } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
 import { FetchError } from '../../errors/app-error.js';
 function assertContentLengthWithinLimit(response, url, maxBytes) {
     const contentLengthHeader = response.headers.get('content-length');
@@ -9,51 +7,98 @@ function assertContentLengthWithinLimit(response, url, maxBytes) {
     if (Number.isNaN(contentLength) || contentLength <= maxBytes) {
         return;
     }
+    cancelResponseBody(response);
     throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
 }
-async function readStreamWithLimit(stream, url, maxBytes, signal) {
-    const decoder = new TextDecoder();
-    let total = 0;
-    let text = '';
-    const toBuffer = (chunk) => {
-        if (typeof chunk === 'string') {
-            return Buffer.from(chunk);
-        }
-        return Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+function createReadState() {
+    return {
+        decoder: new TextDecoder(),
+        parts: [],
+        total: 0,
     };
-    const sink = new Writable({
-        write(chunk, _encoding, callback) {
-            const buffer = toBuffer(chunk);
-            total += buffer.length;
-            if (total > maxBytes) {
-                callback(new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url));
-                return;
-            }
-            text += decoder.decode(buffer, { stream: true });
-            callback();
-        },
-        final(callback) {
-            text += decoder.decode();
-            callback();
-        },
+}
+function appendChunk(state, chunk, maxBytes, url) {
+    state.total += chunk.byteLength;
+    if (state.total > maxBytes) {
+        throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+    }
+    const decoded = state.decoder.decode(chunk, { stream: true });
+    if (decoded)
+        state.parts.push(decoded);
+}
+function finalizeRead(state) {
+    const decoded = state.decoder.decode();
+    if (decoded)
+        state.parts.push(decoded);
+}
+function createAbortError(url) {
+    return new FetchError('Request was aborted during response read', url, 499, {
+        reason: 'aborted',
     });
+}
+function cancelResponseBody(response) {
+    const cancelPromise = response.body?.cancel();
+    if (cancelPromise) {
+        cancelPromise.catch(() => {
+            // Best-effort cancellation; ignore failures.
+        });
+    }
+}
+async function cancelReaderQuietly(reader) {
     try {
-        const readable = Readable.fromWeb(stream, { signal });
-        await pipeline(readable, sink, { signal });
+        await reader.cancel();
+    }
+    catch {
+        // Ignore cancel errors; we're already failing this read.
+    }
+}
+async function throwIfAborted(signal, url, reader) {
+    if (!signal?.aborted)
+        return;
+    await cancelReaderQuietly(reader);
+    throw createAbortError(url);
+}
+async function handleReadFailure(error, signal, url, reader) {
+    const aborted = signal?.aborted ?? false;
+    await cancelReaderQuietly(reader);
+    if (aborted) {
+        throw createAbortError(url);
+    }
+    throw error;
+}
+async function readAllChunks(reader, state, url, maxBytes, signal) {
+    await throwIfAborted(signal, url, reader);
+    let result = await reader.read();
+    while (!result.done) {
+        appendChunk(state, result.value, maxBytes, url);
+        await throwIfAborted(signal, url, reader);
+        result = await reader.read();
+    }
+}
+async function readStreamWithLimit(stream, url, maxBytes, signal) {
+    const state = createReadState();
+    const reader = stream.getReader();
+    try {
+        await readAllChunks(reader, state, url, maxBytes, signal);
     }
     catch (error) {
-        if (signal?.aborted) {
-            throw new FetchError('Request was aborted during response read', url, 499, { reason: 'aborted' });
-        }
-        throw error;
+        await handleReadFailure(error, signal, url, reader);
     }
-    return { text, size: total };
+    finally {
+        reader.releaseLock();
+    }
+    finalizeRead(state);
+    return { text: state.parts.join(''), size: state.total };
 }
 export async function readResponseText(response, url, maxBytes, signal) {
     assertContentLengthWithinLimit(response, url, maxBytes);
     if (!response.body) {
         const text = await response.text();
-        return { text, size: Buffer.byteLength(text) };
+        const size = Buffer.byteLength(text);
+        if (size > maxBytes) {
+            throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+        }
+        return { text, size };
     }
     return readStreamWithLimit(response.body, url, maxBytes, signal);
 }

package/dist/services/fetcher.d.ts

@@ -1,4 +1,23 @@
+import type { Dispatcher } from 'undici';
 import type { FetchOptions } from '../config/types/runtime.js';
-import { destroyAgents } from './fetcher/agents.js';
-export { destroyAgents };
-export declare function fetchNormalizedUrlWithRetry(normalizedUrl: string, options?: FetchOptions, maxRetries?: number): Promise<string>;
+export declare const dispatcher: Dispatcher;
+export declare function destroyAgents(): void;
+interface FetchTelemetryContext {
+    requestId: string;
+    startTime: number;
+    url: string;
+    method: string;
+}
+export declare function startFetchTelemetry(url: string, method: string): FetchTelemetryContext;
+export declare function recordFetchResponse(context: FetchTelemetryContext, response: Response, contentSize?: number): void;
+export declare function recordFetchError(context: FetchTelemetryContext, error: unknown, status?: number): void;
+export declare function fetchWithRedirects(url: string, init: RequestInit, maxRedirects: number): Promise<{
+    response: Response;
+    url: string;
+}>;
+export declare function readResponseText(response: Response, url: string, maxBytes: number, signal?: AbortSignal): Promise<{
+    text: string;
+    size: number;
+}>;
+export declare function fetchNormalizedUrl(normalizedUrl: string, options?: FetchOptions): Promise<string>;
+export {};