nstarter-http-request 0.1.12 → 0.1.14

package/src/adapter/undici.adapter.ts

@@ -4,10 +4,13 @@
 
 import type {
     HttpRequestOptions,
-    HttpResponse,
-    ResponseType,
+    HttpResponse
+} from '../types';
+import {
+    HttpError,
+    RequestTimeoutError,
+    ResponseTooLargeError
 } from '../types';
-import { HttpError } from '../types';
 import { BaseHttpAdapter } from './base';
 import undici, {
     Agent,
@@ -16,6 +19,7 @@ import undici, {
 } from 'undici';
 import { STATUS_CODES } from 'http';
 import { EClientAdapter } from '../const';
+import { RedirectUtils } from '../utils';
 
 interface RequestTimeoutState {
     signal?: AbortSignal;
@@ -69,7 +73,8 @@ export class UndiciAdapter extends BaseHttpAdapter {
                 timeoutState.cleanup();
             }
 
-            const data = await this.readResponseBody<T>(response, options.responseType);
+            // 读取过程中会按 maxResponseBodySize 提前拒绝/中断，超限时不会把完整响应体读入内存
+            const data = await this.readResponseBody<T>(response, options);
 
             const httpResponse: HttpResponse<T> = {
                 status: response.statusCode,
@@ -79,23 +84,18 @@ export class UndiciAdapter extends BaseHttpAdapter {
                 config: options
             };
 
-            this.ensureResponseBodySize(httpResponse, options.responseType);
             this.ensureSuccessStatus(httpResponse);
 
             return httpResponse;
         } catch (error: unknown) {
             if (timeoutState.timedOut) {
-                throw new HttpError(
-                    `timeout of ${ timeoutState.timeout }ms exceeded`,
-                    'ECONNABORTED',
-                    undefined,
-                    undefined,
-                    options
-                );
+                throw new RequestTimeoutError(timeoutState.timeout, options, error);
             }
             throw this.handleUndiciError(error, options);
         } finally {
             timeoutState.cleanup();
+            // buildDispatcher 为每个请求单独创建 Agent/ProxyAgent，用完需要关闭，否则连接池与 keep-alive 连接会一直保留。
+            dispatcher?.close().catch(() => undefined);
         }
     }
 
@@ -199,7 +199,7 @@ export class UndiciAdapter extends BaseHttpAdapter {
         const agent = new ProxyAgent({
             uri: proxyUrl,
             connectTimeout: timeout,
-            connect: {
+            requestTls: {
                 rejectUnauthorized: this.securityConfig.rejectUnauthorized
             }
         });
@@ -223,7 +223,7 @@ export class UndiciAdapter extends BaseHttpAdapter {
         const agent = new ProxyAgent({
             uri: proxyUrl,
             connectTimeout: timeout,
-            connect: {
+            requestTls: {
                 rejectUnauthorized: this.securityConfig.rejectUnauthorized
             }
         });
@@ -338,10 +338,14 @@ export class UndiciAdapter extends BaseHttpAdapter {
     }
 
     /**
-     * 按 axios 默认行为处理非 2xx 状态码。
+     * 2xx 和 304 作为成功结果返回。undici 未跟跳的 3xx 交回调用方。
      */
     private ensureSuccessStatus(response: HttpResponse): void {
-        if (response.status >= 200 && response.status < 300) {
+        if (
+            (response.status >= 200 && response.status < 300)
+            || response.status === 304
+            || RedirectUtils.isRedirectStatus(response.status)
+        ) {
             return;
         }
 
@@ -355,119 +359,122 @@ export class UndiciAdapter extends BaseHttpAdapter {
     }
 
     /**
-     * 按 axios maxContentLength 行为限制非流式响应体大小。
+     * 解析 Content-Length 响应头。仅接受合法的非负整数，重复头/非法值视为未知。
      */
-    private ensureResponseBodySize(response: HttpResponse, responseType?: ResponseType): void {
-        if (responseType === 'stream') {
-            return;
-        }
-        const maxResponseBodySize = this.requestConfig.maxResponseBodySize;
-        if (
-            typeof maxResponseBodySize !== 'number' ||
-            !Number.isFinite(maxResponseBodySize) ||
-            maxResponseBodySize <= 0
-        ) {
-            // NsHttpClient 会先做配置合并和校验；这里防御直接实例化 Adapter 的异常配置。
-            return;
-        }
-        const bodySize = this.getBodySize(response.data);
-        if (bodySize <= maxResponseBodySize) {
-            return;
+    private parseContentLength(value: string | string[] | undefined): number | undefined {
+        if (typeof value !== 'string' || !/^\d+$/.test(value.trim())) {
+            return undefined;
         }
-        throw new HttpError(
-            `maxContentLength size of ${ maxResponseBodySize } exceeded`,
-            'ERR_BAD_RESPONSE',
-            response.status,
-            response,
-            response.config
-        );
+        const length = Number(value.trim());
+        return Number.isSafeInteger(length) ? length : undefined;
     }
 
     /**
-     * 计算已读取响应体的字节大小。
+     * 丢弃未读完的响应体并终止下载。
+     * undici 的 BodyReadable.destroy() 不带错误时会以 RequestAbortedError 触发 error 事件，
+     * 没有监听器会变成未捕获异常，因此先挂空监听器。
      */
-    private getBodySize(body: any): number {
-        if (body === undefined || body === null) {
-            return 0;
-        }
-        if (Buffer.isBuffer(body)) {
-            return body.byteLength;
-        }
-        if (body instanceof ArrayBuffer) {
-            return body.byteLength;
-        }
-        if (ArrayBuffer.isView(body)) {
-            return body.byteLength;
+    private discardBody(response: Dispatcher.ResponseData): void {
+        response.body.on('error', () => undefined);
+        response.body.destroy();
+    }
+
+    /**
+     * 带大小上限地读取非流式响应体。
+     *
+     * 1. Content-Length 预检：响应头已声明超限时，不读取响应体，直接销毁并拒绝。
+     * 2. 逐分块累计字节数，累计值超过上限（等于上限放行）立即销毁响应流并拒绝，后续数据不再下载；
+     *    该行为对齐 axios（Node http 适配器）的 maxContentLength。
+     */
+    private async readBodyWithLimit(response: Dispatcher.ResponseData, options: HttpRequestOptions): Promise<Buffer> {
+        const maxRespBodySize = this.resolveMaxResponseBodySize();
+
+        // HEAD / 204 / 304 没有响应体，其 Content-Length 描述的是对应实体而不是本次响应体：
+        // 不能用于预检；服务端携带时 undici 还会一直等待并不存在的响应体，因此不进入读取，直接丢弃。
+        if (options.method === 'HEAD' || response.statusCode === 204 || response.statusCode === 304) {
+            this.discardBody(response);
+            return Buffer.alloc(0);
         }
-        if (typeof Blob !== 'undefined' && body instanceof Blob) {
-            return body.size;
+
+        if (maxRespBodySize !== undefined) {
+            const contentLength = this.parseContentLength(response.headers['content-length']);
+            if (contentLength !== undefined && contentLength > maxRespBodySize) {
+                this.discardBody(response);
+                throw new ResponseTooLargeError(maxRespBodySize, options, response.statusCode);
+            }
         }
-        if (typeof body === 'string') {
-            return Buffer.byteLength(body);
+
+        const chunks: Buffer[] = [];
+        let total = 0;
+        for await (const chunk of response.body) {
+            const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            total += buffer.byteLength;
+            if (maxRespBodySize !== undefined && total > maxRespBodySize) {
+                this.discardBody(response);
+                throw new ResponseTooLargeError(maxRespBodySize, options, response.statusCode);
+            }
+            chunks.push(buffer);
         }
-        return Buffer.byteLength(JSON.stringify(body));
+        return Buffer.concat(chunks, total);
+    }
+
+    /**
+     * 基于已读取（已受大小限制）的字节构造 Response，复用运行时的 blob/formData 解析逻辑。
+     */
+    private toFetchResponse(buffer: Buffer, response: Dispatcher.ResponseData): Response {
+        const contentType = response.headers['content-type'];
+        return new Response(buffer, {
+            headers: typeof contentType === 'string' ? { 'content-type': contentType } : undefined
+        });
     }
 
     /**
      * 读取响应体
      * @param response Undici 响应对象
-     * @param responseType 响应类型
+     * @param options 请求选项
      */
-    private async readResponseBody<T>(response: Dispatcher.ResponseData, responseType?: ResponseType): Promise<T> {
+    private async readResponseBody<T>(response: Dispatcher.ResponseData, options: HttpRequestOptions): Promise<T> {
+        if (options.responseType === 'stream') {
+            // undici 返回 BodyReadable (继承自 Node.js Readable)，流式响应由调用方自行消费，不受 maxResponseBodySize 限制
+            return response.body as T;
+        }
+
         try {
-            switch (responseType) {
+            const buffer = await this.readBodyWithLimit(response, options);
+            const decoder = new TextDecoder();
+            switch (options.responseType) {
                 case 'text': {
-                    return (await response.body.text()) as T;
+                    return decoder.decode(buffer) as T;
                 }
                 case 'arraybuffer': {
-                    return Buffer.from(await response.body.arrayBuffer()) as T;
-                }
-                case 'stream': {
-                    // undici 返回 BodyReadable (继承自 Node.js Readable)
-                    return response.body as T;
+                    return buffer as T;
                 }
                 case 'blob': {
-                    return (await response.body.blob()) as T;
+                    return (await this.toFetchResponse(buffer, response).blob()) as T;
                 }
                 case 'formdata': {
-                    return (await response.body.formData()) as T;
-                }
-                case 'json': {
-                    return await this.readJsonResponseBody<T>(response);
+                    return (await this.toFetchResponse(buffer, response).formData()) as T;
                 }
+                case 'json':
                 default: {
-                    const text = await response.body.text();
+                    // 按 axios 的默认兼容行为：空响应体或非 JSON 文本不应让成功状态码请求因为解析失败变成异常
+                    const text = decoder.decode(buffer);
                     try {
                         return JSON.parse(text) as T;
                     } catch {
-                        // 如果不是有效的 JSON，返回原文本
                         return text as T;
                     }
                 }
-
             }
         } catch (error) {
+            // 响应体超限错误需要原样抛出，保留错误码、文案与不可重试标记
+            if (error instanceof ResponseTooLargeError) {
+                throw error;
+            }
             throw new Error(`Failed to read response body: ${ error instanceof Error ? error.message : String(error) }`);
         }
     }
 
-    /**
-     * 按 axios 的默认兼容行为读取 JSON 响应。
-     * 空响应体或非 JSON 文本不应让成功状态码请求因为解析失败变成异常。
-     */
-    private async readJsonResponseBody<T>(response: Dispatcher.ResponseData): Promise<T> {
-        const text = await response.body.text();
-        if (!text) {
-            return text as T;
-        }
-
-        try {
-            return JSON.parse(text) as T;
-        } catch {
-            return text as T;
-        }
-    }
-
     /**
      * 获取状态文本
      * @param statusCode 状态码

package/src/client.ts

@@ -32,6 +32,7 @@ import { EClientAdapter } from './const';
 import { HttpError } from './types';
 import {
     DomainUtils,
+    RedirectUtils,
     TraceContextUtils
 } from './utils';
 import {
@@ -39,6 +40,7 @@ import {
     pipeline,
     Readable
 } from 'node:stream';
+import { finished } from 'node:stream/promises';
 import { MetricsTransform } from './stream';
 
 type ForwardProxyDecision = {
@@ -189,84 +191,120 @@ export class NsHttpClient implements IHttpClient {
         const startTime = Date.now();
 
         try {
-            const forwardProxyDecision = this.resolveForwardProxyDecision(options.url);
-
-            // 安全验证
-            const { safeUrl, result: ssrfResult } = await this.ssrfGuard.validateAndGetSafeUrl(
-                options.url,
-                {
-                    skipDnsValidation: forwardProxyDecision.shouldProxy,
-                    skipDnsValidationReason: forwardProxyDecision.shouldProxy ? 'forward_proxy' : undefined
-                }
-            );
+            return await this.safeRequest<T>(options, this.config.request.maxRedirects);
+        } catch (error) {
+            // 记录错误日志
+            this.logError(options, error, Date.now() - startTime);
+            throw error;
+        }
+    }
 
-            this.logger.debug('URL validation passed', {
-                originalUrl: options.url,
-                safeUrl,
-                resolvedIps: ssrfResult.resolvedIps,
-                dnsValidationSkipped: ssrfResult.dnsValidationSkipped,
-                dnsValidationSkipReason: ssrfResult.dnsValidationSkipReason,
-                scenario: options.scenario
-            });
+    /**
+     * 
+     * @param options 请求选项
+     * @param remainedRedirects 剩余可重定向次数
+     * @returns 响应结果
+     */
+    private async safeRequest<T>(options: HttpRequestOptions, remainedRedirects: number): Promise<HttpResponse<T>> {
+        const startTime = Date.now();
+        const forwardProxyDecision = this.resolveForwardProxyDecision(options.url);
 
-            if (ssrfResult.dnsValidationSkipped && forwardProxyDecision.shouldProxy) {
-                this.logger.debug('Skipped target DNS validation because request uses forward proxy', {
-                    hostname: forwardProxyDecision.hostname,
-                    proxyUrl: forwardProxyDecision.proxyUrl
-                });
+        // 安全验证
+        const { safeUrl, result: ssrfResult } = await this.ssrfGuard.validateAndGetSafeUrl(
+            options.url,
+            {
+                skipDnsValidation: forwardProxyDecision.shouldProxy,
+                skipDnsValidationReason: forwardProxyDecision.shouldProxy ? 'forward_proxy' : undefined
             }
+        );
 
-            // 使用安全的 URL 发起请求
-            const safeOptions: HttpRequestOptions = {
-                ...options,
-                url: safeUrl
-            };
+        this.logger.debug('URL validation passed', {
+            originalUrl: options.url,
+            safeUrl,
+            resolvedIps: ssrfResult.resolvedIps,
+            dnsValidationSkipped: ssrfResult.dnsValidationSkipped,
+            dnsValidationSkipReason: ssrfResult.dnsValidationSkipReason,
+            scenario: options.scenario
+        });
 
-            // 处理 W3C Trace Context traceparent 头
-            if (options.traceparent) {
-                // 验证 traceparent 格式
-                if (!TraceContextUtils.isValidTraceParent(options.traceparent)) {
-                    this.logger.warn('Invalid traceparent format, ignoring', {
-                        traceparent: options.traceparent
-                    });
-                } else {
-                    safeOptions.headers = {
-                        ...safeOptions.headers,
-                        traceparent: options.traceparent
-                    };
+        if (ssrfResult.dnsValidationSkipped && forwardProxyDecision.shouldProxy) {
+            this.logger.debug('Skipped target DNS validation because request uses forward proxy', {
+                hostname: forwardProxyDecision.hostname,
+                proxyUrl: forwardProxyDecision.proxyUrl
+            });
+        }
 
-                    this.logger.debug('Traceparent header added', {
-                        traceparent: options.traceparent,
-                        sampled: TraceContextUtils.isSampled(options.traceparent)
-                    });
-                }
-            }
+        // 使用安全的 URL 发起请求
+        const safeOptions: HttpRequestOptions = {
+            ...options,
+            url: safeUrl
+        };
 
-            // 应用正向代理配置
-            if (this.forwardProxy) {
-                this.applyForwardProxy(safeOptions, forwardProxyDecision);
+        // 处理 W3C Trace Context traceparent 头
+        if (options.traceparent) {
+            // 验证 traceparent 格式
+            if (!TraceContextUtils.isValidTraceParent(options.traceparent)) {
+                this.logger.warn('Invalid traceparent format, ignoring', {
+                    traceparent: options.traceparent
+                });
+            } else {
+                safeOptions.headers = {
+                    ...safeOptions.headers,
+                    traceparent: options.traceparent
+                };
+
+                this.logger.debug('Traceparent header added', {
+                    traceparent: options.traceparent,
+                    sampled: TraceContextUtils.isSampled(options.traceparent)
+                });
             }
+        }
 
-            // 发起请求
-            const response = await this.adapter.request<T>(safeOptions);
+        // 应用正向代理配置
+        if (this.forwardProxy) {
+            this.applyForwardProxy(safeOptions, forwardProxyDecision);
+        }
 
-            // 首次响应时间（TTFB）
-            const ttfb = Date.now() - startTime;
+        // 发起请求
+        const response = await this.adapter.request<T>(safeOptions);
 
-            // 如果是流式响应，包装流对象以追踪完整生命周期
-            if (options.responseType === 'stream' && this.isStreamResponse(response.data)) {
-                return this.wrapStreamResponse(safeOptions, response, startTime, ttfb);
+        // 需要重定向时，继续跟跳
+        if (RedirectUtils.isRedirectStatus(response.status)) {
+            try {
+                if (remainedRedirects === undefined || remainedRedirects <= 0) {
+                    throw new Error('max redirects');
+                }
+                const redirectOptions = RedirectUtils.buildRedirectOptions(
+                    safeOptions,
+                    response,
+                    this.adapter.name,
+                    this.config.request.timeout
+                );
+                if (redirectOptions) {
+                    RedirectUtils.consumeRedirectTimeout(redirectOptions, startTime);
+                    // 下一跳前，把中间响应的流先丢掉，以免占用连接
+                    await this.discardResponseStream(response);
+                    const nextRemainedRedirects = remainedRedirects - 1;
+                    return await this.safeRequest<T>(redirectOptions, nextRemainedRedirects);
+                }
+            } catch (error) {
+                await this.discardResponseStream(response);
+                throw error;
             }
+        }
 
-            // 记录非流式响应日志
-            this.logResponse(safeOptions, response, ttfb);
+        // 首次响应时间（TTFB）
+        const ttfb = Date.now() - startTime;
 
-            return response;
-        } catch (error) {
-            // 记录错误日志
-            this.logError(options, error, Date.now() - startTime);
-            throw error;
+        // 如果是流式响应，包装流对象以追踪完整生命周期
+        if (options.responseType === 'stream' && this.isStreamResponse(response.data)) {
+            return this.wrapStreamResponse(safeOptions, response, startTime, ttfb);
         }
+
+        // 记录非流式响应日志
+        this.logResponse(safeOptions, response, ttfb);
+
+        return response;
     }
 
     /**
@@ -354,6 +392,19 @@ export class NsHttpClient implements IHttpClient {
         });
     }
 
+    /**
+     * 销毁不再交给调用方的响应流
+     */
+    private async discardResponseStream(response: HttpResponse): Promise<void> {
+        const data = response.data;
+        if (!(data instanceof Readable) || data.destroyed) {
+            return;
+        }
+        const done = finished(data);
+        data.destroy();
+        await done.catch(() => undefined);
+    }
+
     /**
      * 判断响应数据是否为流对象（Node.js Readable）
      */

package/src/const/config.const.ts

@@ -48,6 +48,13 @@ export const DEFAULT_REQUEST_CONFIG: RequestConfig = {
     maxResponseBodySize: 50 * 1024 * 1024 // 50MB
 };
 
+/**
+ * 响应体大小预检的线上长度容差比例。
+ * axios 的 onDownloadProgress.total 是压缩后的 Content-Length，而 maxContentLength 按解压后计数；
+ * 不可压缩数据经 gzip 等压缩后可能略大于解压大小，预检仅在超过上限该比例时提前中止。
+ */
+export const MAX_RESPONSE_SIZE_TOLERANCE_RATIO = 0.01;
+
 /**
  * 默认 HTTP 客户端配置
  */

package/src/types/errors.ts

@@ -8,12 +8,21 @@ import type { HttpRequestOptions, HttpResponse } from './request_response';
  * HTTP 错误
  */
 export class HttpError extends Error {
+    /**
+     * @param message 错误信息
+     * @param code 错误码
+     * @param status HTTP 状态码
+     * @param response 响应（不应携带不再需要的大体积响应体）
+     * @param request 请求选项
+     * @param retriable 是否允许重试，默认 true；重试无法改变结果的确定性失败应传 false
+     */
     constructor(
         message: string,
         public readonly code: string,
         public readonly status?: number,
         public readonly response?: HttpResponse,
-        public readonly request?: HttpRequestOptions
+        public readonly request?: HttpRequestOptions,
+        public readonly retriable = true
     ) {
         super(message);
         this.name = 'HttpError';
@@ -21,6 +30,60 @@ export class HttpError extends Error {
     }
 }
 
+/**
+ * 响应体超过 maxResponseBodySize 限制的错误。
+ *
+ * - 错误码与文案对齐 axios 的 maxContentLength（ERR_BAD_RESPONSE）。
+ * - 重试不会改变响应体大小，因此标记为不可重试，避免每次重试都重复下载。
+ * - 不携带 response：仅保留状态码，防止超限数据随错误对象存活（与 axios 抛出该错误时一致）。
+ */
+export class ResponseTooLargeError extends HttpError {
+    /**
+     * @param maxResponseBodySize 响应体大小上限
+     * @param request 请求选项
+     * @param status 响应状态码（未知时不传）
+     */
+    constructor(
+        public readonly maxResponseBodySize: number,
+        request: HttpRequestOptions,
+        status?: number
+    ) {
+        super(
+            `maxContentLength size of ${ maxResponseBodySize } exceeded`,
+            'ERR_BAD_RESPONSE',
+            status,
+            undefined,
+            request,
+            false
+        );
+    }
+}
+
+/**
+ * 整次请求超时错误
+ */
+export class RequestTimeoutError extends HttpError {
+    /**
+     * @param timeout 整次请求的超时时间（毫秒）
+     * @param request 请求选项
+     * @param cause 底层错误
+     */
+    constructor(
+        public readonly timeout: number | undefined,
+        request: HttpRequestOptions,
+        public readonly cause?: unknown
+    ) {
+        // cause 保留底层错误，调用方可以据此判断超时发生在哪个阶段, 比如TCP 建连阶段超时为 UND_ERR_CONNECT_TIMEOUT
+        super(
+            `timeout of ${ timeout }ms exceeded`,
+            'ECONNABORTED',
+            undefined,
+            undefined,
+            request
+        );
+    }
+}
+
 /**
  * 安全验证错误
  */

package/src/utils/index.ts

@@ -1,6 +1,7 @@
 export * as IpRangeUtils from './ip.range';
 export * as CommonUtils from './common';
 export * as DomainUtils from './domain';
+export * as RedirectUtils from './redirect';
 export * as TraceContextUtils from './trace.context';
 
 // 导出 TraceContext 类供外部直接使用