nstarter-http-request 0.1.12 → 0.1.13

package/src/client.ts

@@ -32,6 +32,7 @@ import { EClientAdapter } from './const';
 import { HttpError } from './types';
 import {
     DomainUtils,
+    RedirectUtils,
     TraceContextUtils
 } from './utils';
 import {
@@ -39,6 +40,7 @@ import {
     pipeline,
     Readable
 } from 'node:stream';
+import { finished } from 'node:stream/promises';
 import { MetricsTransform } from './stream';
 
 type ForwardProxyDecision = {
@@ -189,84 +191,120 @@ export class NsHttpClient implements IHttpClient {
         const startTime = Date.now();
 
         try {
-            const forwardProxyDecision = this.resolveForwardProxyDecision(options.url);
-
-            // 安全验证
-            const { safeUrl, result: ssrfResult } = await this.ssrfGuard.validateAndGetSafeUrl(
-                options.url,
-                {
-                    skipDnsValidation: forwardProxyDecision.shouldProxy,
-                    skipDnsValidationReason: forwardProxyDecision.shouldProxy ? 'forward_proxy' : undefined
-                }
-            );
+            return await this.safeRequest<T>(options, this.config.request.maxRedirects);
+        } catch (error) {
+            // 记录错误日志
+            this.logError(options, error, Date.now() - startTime);
+            throw error;
+        }
+    }
 
-            this.logger.debug('URL validation passed', {
-                originalUrl: options.url,
-                safeUrl,
-                resolvedIps: ssrfResult.resolvedIps,
-                dnsValidationSkipped: ssrfResult.dnsValidationSkipped,
-                dnsValidationSkipReason: ssrfResult.dnsValidationSkipReason,
-                scenario: options.scenario
-            });
+    /**
+     * 
+     * @param options 请求选项
+     * @param remainedRedirects 剩余可重定向次数
+     * @returns 响应结果
+     */
+    private async safeRequest<T>(options: HttpRequestOptions, remainedRedirects: number): Promise<HttpResponse<T>> {
+        const startTime = Date.now();
+        const forwardProxyDecision = this.resolveForwardProxyDecision(options.url);
 
-            if (ssrfResult.dnsValidationSkipped && forwardProxyDecision.shouldProxy) {
-                this.logger.debug('Skipped target DNS validation because request uses forward proxy', {
-                    hostname: forwardProxyDecision.hostname,
-                    proxyUrl: forwardProxyDecision.proxyUrl
-                });
+        // 安全验证
+        const { safeUrl, result: ssrfResult } = await this.ssrfGuard.validateAndGetSafeUrl(
+            options.url,
+            {
+                skipDnsValidation: forwardProxyDecision.shouldProxy,
+                skipDnsValidationReason: forwardProxyDecision.shouldProxy ? 'forward_proxy' : undefined
             }
+        );
 
-            // 使用安全的 URL 发起请求
-            const safeOptions: HttpRequestOptions = {
-                ...options,
-                url: safeUrl
-            };
+        this.logger.debug('URL validation passed', {
+            originalUrl: options.url,
+            safeUrl,
+            resolvedIps: ssrfResult.resolvedIps,
+            dnsValidationSkipped: ssrfResult.dnsValidationSkipped,
+            dnsValidationSkipReason: ssrfResult.dnsValidationSkipReason,
+            scenario: options.scenario
+        });
 
-            // 处理 W3C Trace Context traceparent 头
-            if (options.traceparent) {
-                // 验证 traceparent 格式
-                if (!TraceContextUtils.isValidTraceParent(options.traceparent)) {
-                    this.logger.warn('Invalid traceparent format, ignoring', {
-                        traceparent: options.traceparent
-                    });
-                } else {
-                    safeOptions.headers = {
-                        ...safeOptions.headers,
-                        traceparent: options.traceparent
-                    };
+        if (ssrfResult.dnsValidationSkipped && forwardProxyDecision.shouldProxy) {
+            this.logger.debug('Skipped target DNS validation because request uses forward proxy', {
+                hostname: forwardProxyDecision.hostname,
+                proxyUrl: forwardProxyDecision.proxyUrl
+            });
+        }
 
-                    this.logger.debug('Traceparent header added', {
-                        traceparent: options.traceparent,
-                        sampled: TraceContextUtils.isSampled(options.traceparent)
-                    });
-                }
-            }
+        // 使用安全的 URL 发起请求
+        const safeOptions: HttpRequestOptions = {
+            ...options,
+            url: safeUrl
+        };
 
-            // 应用正向代理配置
-            if (this.forwardProxy) {
-                this.applyForwardProxy(safeOptions, forwardProxyDecision);
+        // 处理 W3C Trace Context traceparent 头
+        if (options.traceparent) {
+            // 验证 traceparent 格式
+            if (!TraceContextUtils.isValidTraceParent(options.traceparent)) {
+                this.logger.warn('Invalid traceparent format, ignoring', {
+                    traceparent: options.traceparent
+                });
+            } else {
+                safeOptions.headers = {
+                    ...safeOptions.headers,
+                    traceparent: options.traceparent
+                };
+
+                this.logger.debug('Traceparent header added', {
+                    traceparent: options.traceparent,
+                    sampled: TraceContextUtils.isSampled(options.traceparent)
+                });
             }
+        }
 
-            // 发起请求
-            const response = await this.adapter.request<T>(safeOptions);
+        // 应用正向代理配置
+        if (this.forwardProxy) {
+            this.applyForwardProxy(safeOptions, forwardProxyDecision);
+        }
 
-            // 首次响应时间（TTFB）
-            const ttfb = Date.now() - startTime;
+        // 发起请求
+        const response = await this.adapter.request<T>(safeOptions);
 
-            // 如果是流式响应，包装流对象以追踪完整生命周期
-            if (options.responseType === 'stream' && this.isStreamResponse(response.data)) {
-                return this.wrapStreamResponse(safeOptions, response, startTime, ttfb);
+        // 需要重定向时，继续跟跳
+        if (RedirectUtils.isRedirectStatus(response.status)) {
+            try {
+                if (remainedRedirects === undefined || remainedRedirects <= 0) {
+                    throw new Error('max redirects');
+                }
+                const redirectOptions = RedirectUtils.buildRedirectOptions(
+                    safeOptions,
+                    response,
+                    this.adapter.name,
+                    this.config.request.timeout
+                );
+                if (redirectOptions) {
+                    RedirectUtils.consumeRedirectTimeout(redirectOptions, startTime);
+                    // 下一跳前，把中间响应的流先丢掉，以免占用连接
+                    await this.discardResponseStream(response);
+                    const nextRemainedRedirects = remainedRedirects - 1;
+                    return await this.safeRequest<T>(redirectOptions, nextRemainedRedirects);
+                }
+            } catch (error) {
+                await this.discardResponseStream(response);
+                throw error;
             }
+        }
 
-            // 记录非流式响应日志
-            this.logResponse(safeOptions, response, ttfb);
+        // 首次响应时间（TTFB）
+        const ttfb = Date.now() - startTime;
 
-            return response;
-        } catch (error) {
-            // 记录错误日志
-            this.logError(options, error, Date.now() - startTime);
-            throw error;
+        // 如果是流式响应，包装流对象以追踪完整生命周期
+        if (options.responseType === 'stream' && this.isStreamResponse(response.data)) {
+            return this.wrapStreamResponse(safeOptions, response, startTime, ttfb);
         }
+
+        // 记录非流式响应日志
+        this.logResponse(safeOptions, response, ttfb);
+
+        return response;
     }
 
     /**
@@ -354,6 +392,19 @@ export class NsHttpClient implements IHttpClient {
         });
     }
 
+    /**
+     * 销毁不再交给调用方的响应流
+     */
+    private async discardResponseStream(response: HttpResponse): Promise<void> {
+        const data = response.data;
+        if (!(data instanceof Readable) || data.destroyed) {
+            return;
+        }
+        const done = finished(data);
+        data.destroy();
+        await done.catch(() => undefined);
+    }
+
     /**
      * 判断响应数据是否为流对象（Node.js Readable）
      */

package/src/const/config.const.ts

@@ -48,6 +48,13 @@ export const DEFAULT_REQUEST_CONFIG: RequestConfig = {
     maxResponseBodySize: 50 * 1024 * 1024 // 50MB
 };
 
+/**
+ * 响应体大小预检的线上长度容差比例。
+ * axios 的 onDownloadProgress.total 是压缩后的 Content-Length，而 maxContentLength 按解压后计数；
+ * 不可压缩数据经 gzip 等压缩后可能略大于解压大小，预检仅在超过上限该比例时提前中止。
+ */
+export const MAX_RESPONSE_SIZE_TOLERANCE_RATIO = 0.01;
+
 /**
  * 默认 HTTP 客户端配置
  */

package/src/types/errors.ts

@@ -8,12 +8,21 @@ import type { HttpRequestOptions, HttpResponse } from './request_response';
  * HTTP 错误
  */
 export class HttpError extends Error {
+    /**
+     * @param message 错误信息
+     * @param code 错误码
+     * @param status HTTP 状态码
+     * @param response 响应（不应携带不再需要的大体积响应体）
+     * @param request 请求选项
+     * @param retriable 是否允许重试，默认 true；重试无法改变结果的确定性失败应传 false
+     */
     constructor(
         message: string,
         public readonly code: string,
         public readonly status?: number,
         public readonly response?: HttpResponse,
-        public readonly request?: HttpRequestOptions
+        public readonly request?: HttpRequestOptions,
+        public readonly retriable = true
     ) {
         super(message);
         this.name = 'HttpError';
@@ -21,6 +30,35 @@ export class HttpError extends Error {
     }
 }
 
+/**
+ * 响应体超过 maxResponseBodySize 限制的错误。
+ *
+ * - 错误码与文案对齐 axios 的 maxContentLength（ERR_BAD_RESPONSE）。
+ * - 重试不会改变响应体大小，因此标记为不可重试，避免每次重试都重复下载。
+ * - 不携带 response：仅保留状态码，防止超限数据随错误对象存活（与 axios 抛出该错误时一致）。
+ */
+export class ResponseTooLargeError extends HttpError {
+    /**
+     * @param maxResponseBodySize 响应体大小上限
+     * @param request 请求选项
+     * @param status 响应状态码（未知时不传）
+     */
+    constructor(
+        public readonly maxResponseBodySize: number,
+        request: HttpRequestOptions,
+        status?: number
+    ) {
+        super(
+            `maxContentLength size of ${ maxResponseBodySize } exceeded`,
+            'ERR_BAD_RESPONSE',
+            status,
+            undefined,
+            request,
+            false
+        );
+    }
+}
+
 /**
  * 安全验证错误
  */

package/src/utils/index.ts

@@ -1,6 +1,7 @@
 export * as IpRangeUtils from './ip.range';
 export * as CommonUtils from './common';
 export * as DomainUtils from './domain';
+export * as RedirectUtils from './redirect';
 export * as TraceContextUtils from './trace.context';
 
 // 导出 TraceContext 类供外部直接使用

package/src/utils/redirect.ts

@@ -0,0 +1,163 @@
+import type {
+    HttpRequestOptions,
+    HttpResponse
+} from '../types';
+import { HttpError } from '../types';
+import { Readable } from 'node:stream';
+import { EClientAdapter } from '../const';
+
+/**
+ * 交给调用方跟跳的状态码，与 undici 的 redirectableStatusCodes 一致。
+ * 304 表示未修改，305 表示改走代理，306 已废弃，这三类不跟。
+ */
+const REDIRECT_STATUS: ReadonlySet<number> = new Set([
+    300,
+    301,
+    302,
+    303,
+    307,
+    308
+]);
+
+/**
+ * 判断状态码是否需要跟跳。
+ * @param status HTTP 状态码
+ */
+export function isRedirectStatus(status: number): boolean {
+    return REDIRECT_STATUS.has(status);
+}
+
+/**
+ * 根据响应里的 Location 构造下一跳请求。
+ * 没有 Location 时返回 undefined，调用方把当前 3xx 原样返回。
+ * axios：POST 的 301/302，以及非 GET/HEAD 的 303，改成 GET 并去掉 body 和 content-*。
+ * undici：只有非 HEAD 的 303 改成 GET 并去掉 body；303 仍去掉 content-*。
+ * 两边都去掉 host；跨域去掉 authorization、cookie、proxy-authorization。
+ * @param options 当前这一跳的请求
+ * @param response 当前这一跳的响应
+ * @param adapter 当前适配器
+ * @param configuredTimeout 客户端配置中的超时。请求未单独设置时使用。
+ */
+export function buildRedirectOptions(
+    options: HttpRequestOptions,
+    response: HttpResponse,
+    adapter: EClientAdapter,
+    configuredTimeout?: number
+): HttpRequestOptions | undefined {
+    const location = response.headers?.location;
+    if (!location) {
+        return;
+    }
+    const status = response.status;
+    const currentUrl = new URL(options.url);
+    const nextUrl = new URL(location, currentUrl);
+    const headers = { ...options.headers };
+    const crossOrigin = currentUrl.origin !== nextUrl.origin;
+    const rewriteToGet = shouldRewriteToGet(adapter, status, options.method);
+    const removeContentHeaders = adapter === EClientAdapter.axios
+        ? rewriteToGet
+        : status === 303;
+        
+    // 清理请求头
+    for (const key of Object.keys(headers)) {
+        const name = key.toLowerCase();
+        if (
+            name === 'host'
+            || (removeContentHeaders && name.startsWith('content-'))
+            || (crossOrigin && (name === 'authorization' || name === 'cookie' || name === 'proxy-authorization'))
+        ) {
+            delete headers[key];
+        }
+    }
+    const nextOptions: HttpRequestOptions = {
+        ...options,
+        url: nextUrl.toString(),
+        params: undefined,
+        headers,
+        timeout: options.timeout ?? configuredTimeout,
+        meta: options.meta ? { ...options.meta } : undefined
+    };
+    // 清理 meta 中的正向代理信息
+    if (nextOptions.meta) {
+        delete nextOptions.meta.__forwardProxy;
+    }
+    if (rewriteToGet) {
+        nextOptions.method = 'GET';
+        nextOptions.body = undefined;
+    } else if (!isReplayable(options.body)) {
+        throw new Error('Request body is not replayable');
+    }
+    return nextOptions;
+}
+
+/**
+ * axios 的 301/302 只改 POST。undici 的 301/302 保持原方法。
+ * 303 时两边都把非 HEAD 改成 GET；axios 额外保留已经是 GET 的请求。
+ */
+function shouldRewriteToGet(adapter: EClientAdapter, status: number, method: HttpRequestOptions['method']): boolean {
+    if (adapter === EClientAdapter.axios) {
+        return ((status === 301 || status === 302) && method === 'POST')
+            || (status === 303 && method !== 'GET' && method !== 'HEAD');
+    }
+    return status === 303 && method !== 'HEAD';
+}
+
+/**
+ * 字符串、Buffer、普通对象每次都能重新序列化。
+ * 流和异步可迭代对象读过一次就不能再发。
+ */
+function isReplayable(body: unknown): boolean {
+    if (
+        body === null
+        || body === undefined
+        || typeof body === 'string'
+        || typeof body === 'number'
+        || typeof body === 'boolean'
+        || Buffer.isBuffer(body)
+        || body instanceof ArrayBuffer
+        || ArrayBuffer.isView(body)
+        || body instanceof URLSearchParams
+        || (typeof Blob !== 'undefined' && body instanceof Blob)
+        || (typeof FormData !== 'undefined' && body instanceof FormData)
+    ) {
+        return true;
+    }
+    if (body instanceof Readable) {
+        return false;
+    }
+    if (typeof body !== 'object') {
+        return true;
+    }
+    const candidate = body as {
+        getReader?: unknown,
+        [Symbol.asyncIterator]?: unknown
+    };
+    return typeof candidate.getReader !== 'function'
+        && typeof candidate[Symbol.asyncIterator] !== 'function';
+}
+
+/**
+ * 跟跳共用同一次超时。把本跳已经花掉的时间从 redirectOptions.timeout 扣掉。
+ * @param redirectOptions 下一跳请求
+ * @param startTime 本跳开始时间
+ */
+export function consumeRedirectTimeout(
+    redirectOptions: HttpRequestOptions,
+    startTime: number
+): void {
+    const timeout = redirectOptions.timeout;
+    if (typeof timeout !== 'number' || !Number.isFinite(timeout) || timeout <= 0) {
+        return;
+    }
+    const remaining = timeout - (Date.now() - startTime);
+    if (remaining <= 0) {
+        throw new HttpError(
+            `timeout of ${ timeout }ms exceeded`,
+            'ECONNABORTED',
+            undefined,
+            undefined,
+            redirectOptions
+        );
+    }
+    redirectOptions.timeout = remaining;
+}