@j0hanz/superfetch 2.0.0 → 2.1.0

package/dist/services/fetcher.js

@@ -1,9 +1,538 @@
+import { randomUUID } from 'node:crypto';
+import diagnosticsChannel from 'node:diagnostics_channel';
+import dns from 'node:dns';
+import os from 'node:os';
+import { performance } from 'node:perf_hooks';
+import { Agent } from 'undici';
 import { config } from '../config/index.js';
-import { dispatcher } from './fetcher/agents.js';
-import { createHttpError, createRateLimitError, mapFetchError, } from './fetcher/errors.js';
-import { recordFetchError, recordFetchResponse, startFetchTelemetry, } from './fetcher/interceptors.js';
-import { fetchWithRedirects } from './fetcher/redirects.js';
-import { readResponseText } from './fetcher/response.js';
+import { FetchError } from '../errors/app-error.js';
+import { createErrorWithCode, isSystemError } from '../utils/error-details.js';
+import { isRecord } from '../utils/guards.js';
+import { redactUrl } from '../utils/url-redactor.js';
+import { isBlockedIp, validateAndNormalizeUrl, } from '../utils/url-validator.js';
+import { getOperationId, getRequestId } from './context.js';
+import { logDebug, logError, logWarn } from './logger.js';
+const DNS_LOOKUP_TIMEOUT_MS = 5000;
+function normalizeLookupResults(addresses, family) {
+    if (Array.isArray(addresses)) {
+        return addresses;
+    }
+    return [{ address: addresses, family: family ?? 4 }];
+}
+function findBlockedIpError(list, hostname) {
+    for (const addr of list) {
+        const ip = typeof addr === 'string' ? addr : addr.address;
+        if (!isBlockedIp(ip)) {
+            continue;
+        }
+        return createErrorWithCode(`Blocked IP detected for ${hostname}`, 'EBLOCKED');
+    }
+    return null;
+}
+function findInvalidFamilyError(list, hostname) {
+    for (const addr of list) {
+        const family = typeof addr === 'string' ? 0 : addr.family;
+        if (family === 4 || family === 6)
+            continue;
+        return createErrorWithCode(`Invalid address family returned for ${hostname}`, 'EINVAL');
+    }
+    return null;
+}
+function createNoDnsResultsError(hostname) {
+    return createErrorWithCode(`No DNS results returned for ${hostname}`, 'ENODATA');
+}
+function createEmptySelection(hostname) {
+    return {
+        error: createNoDnsResultsError(hostname),
+        fallback: [],
+        address: [],
+    };
+}
+function selectLookupResult(list, useAll, hostname) {
+    if (list.length === 0)
+        return createEmptySelection(hostname);
+    if (useAll)
+        return { address: list, fallback: list };
+    const first = list.at(0);
+    if (!first)
+        return createEmptySelection(hostname);
+    return {
+        address: first.address,
+        family: first.family,
+        fallback: list,
+    };
+}
+function findLookupError(list, hostname) {
+    return (findInvalidFamilyError(list, hostname) ?? findBlockedIpError(list, hostname));
+}
+function handleLookupResult(error, addresses, hostname, resolvedFamily, useAll, callback) {
+    if (error) {
+        callback(error, addresses);
+        return;
+    }
+    const list = normalizeLookupResults(addresses, resolvedFamily);
+    const lookupError = findLookupError(list, hostname);
+    if (lookupError) {
+        callback(lookupError, list);
+        return;
+    }
+    const selection = selectLookupResult(list, useAll, hostname);
+    if (selection.error) {
+        callback(selection.error, selection.fallback);
+        return;
+    }
+    callback(null, selection.address, selection.family);
+}
+function resolveDns(hostname, options, callback) {
+    const { normalizedOptions, useAll, resolvedFamily } = buildLookupContext(options);
+    const lookupOptions = buildLookupOptions(normalizedOptions);
+    const timeout = createLookupTimeout(hostname, callback);
+    const safeCallback = wrapLookupCallback(callback, timeout);
+    dns.lookup(hostname, lookupOptions, createLookupCallback(hostname, resolvedFamily, useAll, safeCallback));
+}
+function normalizeLookupOptions(options) {
+    return typeof options === 'number' ? { family: options } : options;
+}
+function buildLookupContext(options) {
+    const normalizedOptions = normalizeLookupOptions(options);
+    return {
+        normalizedOptions,
+        useAll: Boolean(normalizedOptions.all),
+        resolvedFamily: resolveFamily(normalizedOptions.family),
+    };
+}
+const DEFAULT_DNS_ORDER = 'verbatim';
+function resolveResultOrder(options) {
+    if (options.order)
+        return options.order;
+    const legacyVerbatim = getLegacyVerbatim(options);
+    if (legacyVerbatim !== undefined) {
+        return legacyVerbatim ? 'verbatim' : 'ipv4first';
+    }
+    return DEFAULT_DNS_ORDER;
+}
+function getLegacyVerbatim(options) {
+    if (isRecord(options)) {
+        const { verbatim } = options;
+        return typeof verbatim === 'boolean' ? verbatim : undefined;
+    }
+    return undefined;
+}
+function buildLookupOptions(normalizedOptions) {
+    return {
+        family: normalizedOptions.family,
+        hints: normalizedOptions.hints,
+        all: true,
+        order: resolveResultOrder(normalizedOptions),
+    };
+}
+function createLookupCallback(hostname, resolvedFamily, useAll, callback) {
+    return (err, addresses) => {
+        handleLookupResult(err, addresses, hostname, resolvedFamily, useAll, callback);
+    };
+}
+function resolveFamily(family) {
+    if (family === 'IPv4')
+        return 4;
+    if (family === 'IPv6')
+        return 6;
+    return family;
+}
+function createLookupTimeout(hostname, callback) {
+    let done = false;
+    const timer = setTimeout(() => {
+        if (done)
+            return;
+        done = true;
+        callback(createErrorWithCode(`DNS lookup timed out for ${hostname}`, 'ETIMEOUT'), []);
+    }, DNS_LOOKUP_TIMEOUT_MS);
+    timer.unref();
+    return {
+        isDone: () => done,
+        markDone: () => {
+            done = true;
+            clearTimeout(timer);
+        },
+    };
+}
+function wrapLookupCallback(callback, timeout) {
+    return (err, address, family) => {
+        if (timeout.isDone())
+            return;
+        timeout.markDone();
+        callback(err, address, family);
+    };
+}
+function getAgentOptions() {
+    const cpuCount = os.availableParallelism();
+    return {
+        keepAliveTimeout: 60000,
+        connections: Math.max(cpuCount * 2, 25),
+        pipelining: 1,
+        connect: { lookup: resolveDns },
+    };
+}
+export const dispatcher = new Agent(getAgentOptions());
+export function destroyAgents() {
+    void dispatcher.close();
+}
+function parseRetryAfter(header) {
+    if (!header)
+        return 60;
+    const parsed = parseInt(header, 10);
+    return Number.isNaN(parsed) ? 60 : parsed;
+}
+function createCanceledError(url) {
+    return new FetchError('Request was canceled', url, 499, {
+        reason: 'aborted',
+    });
+}
+function createTimeoutError(url, timeoutMs) {
+    return new FetchError(`Request timeout after ${timeoutMs}ms`, url, 504, {
+        timeout: timeoutMs,
+    });
+}
+function createRateLimitError(url, headerValue) {
+    const retryAfter = parseRetryAfter(headerValue);
+    return new FetchError('Too many requests', url, 429, { retryAfter });
+}
+function createHttpError(url, status, statusText) {
+    return new FetchError(`HTTP ${status}: ${statusText}`, url, status);
+}
+function createNetworkError(url, message) {
+    const details = message ? { message } : undefined;
+    return new FetchError(`Network error: Could not reach ${url}`, url, undefined, details ?? {});
+}
+function createUnknownError(url, message) {
+    return new FetchError(message, url);
+}
+function isAbortError(error) {
+    return (error instanceof Error &&
+        (error.name === 'AbortError' || error.name === 'TimeoutError'));
+}
+function isTimeoutError(error) {
+    return error instanceof Error && error.name === 'TimeoutError';
+}
+function getRequestUrl(record) {
+    const value = record.requestUrl;
+    return typeof value === 'string' ? value : null;
+}
+function resolveErrorUrl(error, fallback) {
+    if (error instanceof FetchError)
+        return error.url;
+    if (!isRecord(error))
+        return fallback;
+    const requestUrl = getRequestUrl(error);
+    if (requestUrl)
+        return requestUrl;
+    return fallback;
+}
+function mapFetchError(error, fallbackUrl, timeoutMs) {
+    if (error instanceof FetchError)
+        return error;
+    const url = resolveErrorUrl(error, fallbackUrl);
+    if (isAbortError(error)) {
+        if (isTimeoutError(error)) {
+            return createTimeoutError(url, timeoutMs);
+        }
+        return createCanceledError(url);
+    }
+    if (error instanceof Error) {
+        return createNetworkError(url, error.message);
+    }
+    return createUnknownError(url, 'Unexpected error');
+}
+const fetchChannel = diagnosticsChannel.channel('superfetch.fetch');
+function publishFetchEvent(event) {
+    if (!fetchChannel.hasSubscribers)
+        return;
+    try {
+        fetchChannel.publish(event);
+    }
+    catch {
+        // Avoid crashing the publisher if a subscriber throws.
+    }
+}
+export function startFetchTelemetry(url, method) {
+    const safeUrl = redactUrl(url);
+    const contextRequestId = getRequestId();
+    const operationId = getOperationId();
+    const context = {
+        requestId: randomUUID(),
+        startTime: performance.now(),
+        url: safeUrl,
+        method: method.toUpperCase(),
+        ...(contextRequestId ? { contextRequestId } : {}),
+        ...(operationId ? { operationId } : {}),
+    };
+    publishFetchEvent({
+        v: 1,
+        type: 'start',
+        requestId: context.requestId,
+        method: context.method,
+        url: context.url,
+        ...(context.contextRequestId
+            ? { contextRequestId: context.contextRequestId }
+            : {}),
+        ...(context.operationId ? { operationId: context.operationId } : {}),
+    });
+    logDebug('HTTP Request', {
+        requestId: context.requestId,
+        method: context.method,
+        url: context.url,
+        ...(context.contextRequestId
+            ? { contextRequestId: context.contextRequestId }
+            : {}),
+        ...(context.operationId ? { operationId: context.operationId } : {}),
+    });
+    return context;
+}
+export function recordFetchResponse(context, response, contentSize) {
+    const duration = performance.now() - context.startTime;
+    const durationLabel = `${Math.round(duration)}ms`;
+    publishFetchEvent({
+        v: 1,
+        type: 'end',
+        requestId: context.requestId,
+        status: response.status,
+        duration,
+        ...(context.contextRequestId
+            ? { contextRequestId: context.contextRequestId }
+            : {}),
+        ...(context.operationId ? { operationId: context.operationId } : {}),
+    });
+    const contentType = response.headers.get('content-type');
+    const contentLength = response.headers.get('content-length') ??
+        (contentSize === undefined ? undefined : String(contentSize));
+    logDebug('HTTP Response', {
+        requestId: context.requestId,
+        status: response.status,
+        url: context.url,
+        duration: durationLabel,
+        ...(context.contextRequestId
+            ? { contextRequestId: context.contextRequestId }
+            : {}),
+        ...(context.operationId ? { operationId: context.operationId } : {}),
+        ...(contentType ? { contentType } : {}),
+        ...(contentLength ? { size: contentLength } : {}),
+    });
+    if (duration > 5000) {
+        logWarn('Slow HTTP request detected', {
+            requestId: context.requestId,
+            url: context.url,
+            duration: durationLabel,
+            ...(context.contextRequestId
+                ? { contextRequestId: context.contextRequestId }
+                : {}),
+            ...(context.operationId ? { operationId: context.operationId } : {}),
+        });
+    }
+}
+export function recordFetchError(context, error, status) {
+    const duration = performance.now() - context.startTime;
+    const err = error instanceof Error ? error : new Error(String(error));
+    const event = {
+        v: 1,
+        type: 'error',
+        requestId: context.requestId,
+        url: context.url,
+        error: err.message,
+        duration,
+        ...(context.contextRequestId
+            ? { contextRequestId: context.contextRequestId }
+            : {}),
+        ...(context.operationId ? { operationId: context.operationId } : {}),
+    };
+    const code = isSystemError(err) ? err.code : undefined;
+    if (code !== undefined) {
+        event.code = code;
+    }
+    if (status !== undefined) {
+        event.status = status;
+    }
+    publishFetchEvent(event);
+    const log = status === 429 ? logWarn : logError;
+    log('HTTP Request Error', {
+        requestId: context.requestId,
+        url: context.url,
+        status,
+        code,
+        error: err.message,
+        ...(context.contextRequestId
+            ? { contextRequestId: context.contextRequestId }
+            : {}),
+        ...(context.operationId ? { operationId: context.operationId } : {}),
+    });
+}
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+function isRedirectStatus(status) {
+    return REDIRECT_STATUSES.has(status);
+}
+function cancelResponseBody(response) {
+    const cancelPromise = response.body?.cancel();
+    if (cancelPromise) {
+        cancelPromise.catch(() => {
+            // Best-effort cancellation; ignore failures.
+        });
+    }
+}
+async function performFetchCycle(currentUrl, init, redirectLimit, redirectCount) {
+    const response = await fetch(currentUrl, { ...init, redirect: 'manual' });
+    if (!isRedirectStatus(response.status)) {
+        return { response };
+    }
+    assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount);
+    const location = getRedirectLocation(response, currentUrl);
+    cancelResponseBody(response);
+    return {
+        response,
+        nextUrl: resolveRedirectTarget(currentUrl, location),
+    };
+}
+function assertRedirectWithinLimit(response, currentUrl, redirectLimit, redirectCount) {
+    if (redirectCount < redirectLimit)
+        return;
+    cancelResponseBody(response);
+    throw new FetchError('Too many redirects', currentUrl);
+}
+function getRedirectLocation(response, currentUrl) {
+    const location = response.headers.get('location');
+    if (location)
+        return location;
+    cancelResponseBody(response);
+    throw new FetchError('Redirect response missing Location header', currentUrl);
+}
+function annotateRedirectError(error, url) {
+    if (!isRecord(error))
+        return;
+    error.requestUrl = url;
+}
+function resolveRedirectTarget(baseUrl, location) {
+    if (!URL.canParse(location, baseUrl)) {
+        throw createErrorWithCode('Invalid redirect target', 'EBADREDIRECT');
+    }
+    const resolved = new URL(location, baseUrl);
+    if (resolved.username || resolved.password) {
+        throw createErrorWithCode('Redirect target includes credentials', 'EBADREDIRECT');
+    }
+    return validateAndNormalizeUrl(resolved.href);
+}
+export async function fetchWithRedirects(url, init, maxRedirects) {
+    let currentUrl = url;
+    const redirectLimit = Math.max(0, maxRedirects);
+    for (let redirectCount = 0; redirectCount <= redirectLimit; redirectCount += 1) {
+        const { response, nextUrl } = await performFetchCycleSafely(currentUrl, init, redirectLimit, redirectCount);
+        if (!nextUrl) {
+            return { response, url: currentUrl };
+        }
+        currentUrl = nextUrl;
+    }
+    throw new FetchError('Too many redirects', currentUrl);
+}
+async function performFetchCycleSafely(currentUrl, init, redirectLimit, redirectCount) {
+    try {
+        return await performFetchCycle(currentUrl, init, redirectLimit, redirectCount);
+    }
+    catch (error) {
+        annotateRedirectError(error, currentUrl);
+        throw error;
+    }
+}
+function assertContentLengthWithinLimit(response, url, maxBytes) {
+    const contentLengthHeader = response.headers.get('content-length');
+    if (!contentLengthHeader)
+        return;
+    const contentLength = Number.parseInt(contentLengthHeader, 10);
+    if (Number.isNaN(contentLength) || contentLength <= maxBytes) {
+        return;
+    }
+    cancelResponseBody(response);
+    throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+}
+function createReadState() {
+    return {
+        decoder: new TextDecoder(),
+        parts: [],
+        total: 0,
+    };
+}
+function appendChunk(state, chunk, maxBytes, url) {
+    state.total += chunk.byteLength;
+    if (state.total > maxBytes) {
+        throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+    }
+    const decoded = state.decoder.decode(chunk, { stream: true });
+    if (decoded)
+        state.parts.push(decoded);
+}
+function finalizeRead(state) {
+    const decoded = state.decoder.decode();
+    if (decoded)
+        state.parts.push(decoded);
+}
+function createAbortError(url) {
+    return new FetchError('Request was aborted during response read', url, 499, {
+        reason: 'aborted',
+    });
+}
+async function cancelReaderQuietly(reader) {
+    try {
+        await reader.cancel();
+    }
+    catch {
+        // Ignore cancel errors; we're already failing this read.
+    }
+}
+async function throwIfAborted(signal, url, reader) {
+    if (!signal?.aborted)
+        return;
+    await cancelReaderQuietly(reader);
+    throw createAbortError(url);
+}
+async function handleReadFailure(error, signal, url, reader) {
+    const aborted = signal?.aborted ?? false;
+    await cancelReaderQuietly(reader);
+    if (aborted) {
+        throw createAbortError(url);
+    }
+    throw error;
+}
+async function readAllChunks(reader, state, url, maxBytes, signal) {
+    await throwIfAborted(signal, url, reader);
+    let result = await reader.read();
+    while (!result.done) {
+        appendChunk(state, result.value, maxBytes, url);
+        await throwIfAborted(signal, url, reader);
+        result = await reader.read();
+    }
+}
+async function readStreamWithLimit(stream, url, maxBytes, signal) {
+    const state = createReadState();
+    const reader = stream.getReader();
+    try {
+        await readAllChunks(reader, state, url, maxBytes, signal);
+    }
+    catch (error) {
+        await handleReadFailure(error, signal, url, reader);
+    }
+    finally {
+        reader.releaseLock();
+    }
+    finalizeRead(state);
+    return { text: state.parts.join(''), size: state.total };
+}
+export async function readResponseText(response, url, maxBytes, signal) {
+    assertContentLengthWithinLimit(response, url, maxBytes);
+    if (!response.body) {
+        const text = await response.text();
+        const size = Buffer.byteLength(text);
+        if (size > maxBytes) {
+            throw new FetchError(`Response exceeds maximum size of ${maxBytes} bytes`, url);
+        }
+        return { text, size };
+    }
+    return readStreamWithLimit(response.body, url, maxBytes, signal);
+}
 const DEFAULT_HEADERS = {
     'User-Agent': config.fetcher.userAgent,
     Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
@@ -12,7 +541,7 @@ const DEFAULT_HEADERS = {
     Connection: 'keep-alive',
 };
 function buildHeaders() {
-    return new Headers(DEFAULT_HEADERS);
+    return { ...DEFAULT_HEADERS };
 }
 function buildRequestSignal(timeoutMs, external) {
     const timeoutSignal = AbortSignal.timeout(timeoutMs);
@@ -28,14 +557,25 @@ function buildRequestInit(headers, signal) {
         dispatcher,
     };
 }
+function resolveResponseError(response, finalUrl) {
+    return (resolveRateLimitError(response, finalUrl) ??
+        resolveHttpError(response, finalUrl));
+}
+function resolveRateLimitError(response, finalUrl) {
+    return response.status === 429
+        ? createRateLimitError(finalUrl, response.headers.get('retry-after'))
+        : null;
+}
+function resolveHttpError(response, finalUrl) {
+    return response.ok
+        ? null
+        : createHttpError(finalUrl, response.status, response.statusText);
+}
 async function handleFetchResponse(response, finalUrl, telemetry, signal) {
-    if (response.status === 429) {
-        void response.body?.cancel();
-        throw createRateLimitError(finalUrl, response.headers.get('retry-after'));
-    }
-    if (!response.ok) {
-        void response.body?.cancel();
-        throw createHttpError(finalUrl, response.status, response.statusText);
+    const responseError = resolveResponseError(response, finalUrl);
+    if (responseError) {
+        cancelResponseBody(response);
+        throw responseError;
     }
     const { text, size } = await readResponseText(response, finalUrl, config.fetcher.maxContentLength, signal);
     recordFetchResponse(telemetry, response, size);

package/dist/services/logger.js

@@ -1,13 +1,16 @@
 import { config } from '../config/index.js';
-import { getRequestId, getSessionId } from './context.js';
+import { getOperationId, getRequestId, getSessionId } from './context.js';
 function formatMetadata(meta) {
     const requestId = getRequestId();
     const sessionId = getSessionId();
+    const operationId = getOperationId();
     const contextMeta = {};
     if (requestId)
         contextMeta.requestId = requestId;
     if (sessionId)
         contextMeta.sessionId = sessionId;
+    if (operationId)
+        contextMeta.operationId = operationId;
     const merged = { ...contextMeta, ...meta };
     return Object.keys(merged).length > 0 ? ` ${JSON.stringify(merged)}` : '';
 }

package/dist/services/telemetry.d.ts

@@ -0,0 +1,19 @@
+export interface TransformStageEvent {
+    v: 1;
+    type: 'stage';
+    stage: string;
+    durationMs: number;
+    url: string;
+    requestId?: string;
+    operationId?: string;
+    truncated?: boolean;
+}
+export interface TransformStageContext {
+    readonly stage: string;
+    readonly startTime: number;
+    readonly url: string;
+}
+export declare function startTransformStage(url: string, stage: string): TransformStageContext | null;
+export declare function endTransformStage(context: TransformStageContext | null, options?: {
+    truncated?: boolean;
+}): void;

package/dist/services/telemetry.js

@@ -0,0 +1,43 @@
+import diagnosticsChannel from 'node:diagnostics_channel';
+import { performance } from 'node:perf_hooks';
+import { redactUrl } from '../utils/url-redactor.js';
+import { getOperationId, getRequestId } from './context.js';
+const transformChannel = diagnosticsChannel.channel('superfetch.transform');
+function publishTransformEvent(event) {
+    if (!transformChannel.hasSubscribers)
+        return;
+    try {
+        transformChannel.publish(event);
+    }
+    catch {
+        // Avoid crashing the publisher if a subscriber throws.
+    }
+}
+export function startTransformStage(url, stage) {
+    if (!transformChannel.hasSubscribers)
+        return null;
+    return {
+        stage,
+        startTime: performance.now(),
+        url: redactUrl(url),
+    };
+}
+export function endTransformStage(context, options) {
+    if (!context)
+        return;
+    const requestId = getRequestId();
+    const operationId = getOperationId();
+    const event = {
+        v: 1,
+        type: 'stage',
+        stage: context.stage,
+        durationMs: performance.now() - context.startTime,
+        url: context.url,
+        ...(requestId ? { requestId } : {}),
+        ...(operationId ? { operationId } : {}),
+        ...(options?.truncated !== undefined
+            ? { truncated: options.truncated }
+            : {}),
+    };
+    publishTransformEvent(event);
+}

package/dist/services/transform-worker-pool.d.ts

@@ -1,4 +1,11 @@
 import type { MarkdownTransformResult } from '../config/types/content.js';
-import type { WorkerTransformRequest } from './transform-worker-types.js';
-export declare function transformInWorker(request: Omit<WorkerTransformRequest, 'id'>, signal?: AbortSignal): Promise<MarkdownTransformResult>;
-export declare function destroyTransformWorkers(): Promise<void>;
+interface TransformWorkerPool {
+    transform(html: string, url: string, options: {
+        includeMetadata: boolean;
+        signal?: AbortSignal;
+    }): Promise<MarkdownTransformResult>;
+    close(): Promise<void>;
+}
+export declare function getOrCreateTransformWorkerPool(): TransformWorkerPool;
+export declare function shutdownTransformWorkerPool(): Promise<void>;
+export {};