got 14.6.0 → 14.6.2

package/dist/source/core/index.js

@@ -1,7 +1,8 @@
 import process from 'node:process';
 import { Buffer } from 'node:buffer';
-import { Duplex, Transform } from 'node:stream';
+import { Duplex } from 'node:stream';
 import http, { ServerResponse } from 'node:http';
+import { byteLength } from 'byte-counter';
 import timer from '@szmarczak/http-timer';
 import CacheableRequest, { CacheError as CacheableCacheError, } from 'cacheable-request';
 import decompressResponse from 'decompress-response';
@@ -19,12 +20,16 @@ import { isResponseOk } from './response.js';
 import isClientRequest from './utils/is-client-request.js';
 import isUnixSocketURL, { getUnixSocketPath } from './utils/is-unix-socket-url.js';
 import { RequestError, ReadError, MaxRedirectsError, HTTPError, TimeoutError, UploadError, CacheError, AbortError, } from './errors.js';
+import { generateRequestId, publishRequestCreate, publishRequestStart, publishResponseStart, publishResponseEnd, publishRetry, publishError, publishRedirect, } from './diagnostics-channel.js';
 const supportsBrotli = is.string(process.versions.brotli);
+const supportsZstd = is.string(process.versions.zstd);
 const methodsWithoutBody = new Set(['GET', 'HEAD']);
 // Methods that should auto-end streams when no body is provided
 const methodsWithoutBodyStream = new Set(['OPTIONS', 'DELETE', 'PATCH']);
 const cacheableStore = new WeakableMap();
 const redirectCodes = new Set([300, 301, 302, 303, 304, 307, 308]);
+// Track errors that have been processed by beforeError hooks to preserve custom error types
+const errorsProcessedByHooks = new WeakSet();
 const proxiedRequestEvents = [
     'socket',
     'connect',
@@ -33,17 +38,6 @@ const proxiedRequestEvents = [
     'upgrade',
 ];
 const noop = () => { };
-/**
-Stream transform that counts bytes passing through.
-Used to track compressed bytes before decompression for content-length validation.
-*/
-class ByteCounter extends Transform {
-    count = 0;
-    _transform(chunk, _encoding, callback) {
-        this.count += chunk.length;
-        callback(null, chunk);
-    }
-}
 export default class Request extends Duplex {
     // @ts-expect-error - Ignoring for now.
     ['constructor'];
@@ -52,28 +46,30 @@ export default class Request extends Duplex {
     options;
     response;
     requestUrl;
-    redirectUrls;
-    retryCount;
-    _stopReading;
-    _stopRetry;
-    _downloadedSize;
-    _uploadedSize;
-    _pipedServerResponses;
+    redirectUrls = [];
+    retryCount = 0;
+    _stopReading = false;
+    _stopRetry = noop;
+    _downloadedSize = 0;
+    _uploadedSize = 0;
+    _pipedServerResponses = new Set();
     _request;
     _responseSize;
     _bodySize;
-    _unproxyEvents;
+    _unproxyEvents = noop;
     _isFromCache;
-    _triggerRead;
-    _cancelTimeouts;
-    _removeListeners;
+    _triggerRead = false;
+    _jobs = [];
+    _cancelTimeouts = noop;
+    _removeListeners = noop;
     _nativeResponse;
-    _flushed;
-    _aborted;
+    _flushed = false;
+    _aborted = false;
     _expectedContentLength;
-    _byteCounter;
+    _compressedBytesCount;
+    _requestId = generateRequestId();
     // We need this because `this._request` if `undefined` when using cache
-    _requestInitialized;
+    _requestInitialized = false;
     constructor(url, options, defaults) {
         super({
             // Don't destroy immediately, as the error may be emitted on unsuccessful retry
@@ -81,23 +77,8 @@ export default class Request extends Duplex {
             // It needs to be zero because we're just proxying the data to another stream
             highWaterMark: 0,
         });
-        this._downloadedSize = 0;
-        this._uploadedSize = 0;
-        this._stopReading = false;
-        this._pipedServerResponses = new Set();
-        this._unproxyEvents = noop;
-        this._triggerRead = false;
-        this._cancelTimeouts = noop;
-        this._removeListeners = noop;
-        this._jobs = [];
-        this._flushed = false;
-        this._requestInitialized = false;
-        this._aborted = false;
-        this.redirectUrls = [];
-        this.retryCount = 0;
-        this._stopRetry = noop;
         this.on('pipe', (source) => {
-            if (source?.headers) {
+            if (this.options.copyPipedHeaders && source?.headers) {
                 Object.assign(this.options.headers, source.headers);
             }
         });
@@ -115,6 +96,12 @@ export default class Request extends Duplex {
                 this.options.url = '';
             }
             this.requestUrl = this.options.url;
+            // Publish request creation event
+            publishRequestCreate({
+                requestId: this._requestId,
+                url: this.options.url?.toString() ?? '',
+                method: this.options.method,
+            });
         }
         catch (error) {
             const { options } = error;
@@ -285,6 +272,8 @@ export default class Request extends Duplex {
                     if (this.destroyed) {
                         return;
                     }
+                    // Capture body BEFORE hooks run to detect reassignment
+                    const bodyBeforeHooks = this.options.body;
                     try {
                         for (const hook of this.options.hooks.beforeRetry) {
                             // eslint-disable-next-line no-await-in-loop
@@ -292,14 +281,58 @@ export default class Request extends Duplex {
                         }
                     }
                     catch (error_) {
-                        void this._error(new RequestError(error_.message, error, this));
+                        void this._error(new RequestError(error_.message, error_, this));
                         return;
                     }
                     // Something forced us to abort the retry
                     if (this.destroyed) {
                         return;
                     }
-                    this.destroy();
+                    // Preserve stream body reassigned in beforeRetry hooks.
+                    const bodyAfterHooks = this.options.body;
+                    const bodyWasReassigned = bodyBeforeHooks !== bodyAfterHooks;
+                    // Resource cleanup and preservation logic for retry with body reassignment.
+                    // The Promise wrapper (as-promise/index.ts) compares body identity to detect consumed streams,
+                    // so we must preserve the body reference across destroy(). However, destroy() calls _destroy()
+                    // which destroys this.options.body, creating a complex dance of clear/restore operations.
+                    //
+                    // Key constraints:
+                    // 1. If body was reassigned, we must NOT destroy the NEW stream (it will be used for retry)
+                    // 2. If body was reassigned, we MUST destroy the OLD stream to prevent memory leaks
+                    // 3. We must restore the body reference after destroy() for identity checks in promise wrapper
+                    // 4. We cannot use the normal setter after destroy() because it validates stream readability
+                    if (bodyWasReassigned) {
+                        const oldBody = bodyBeforeHooks;
+                        // Temporarily clear body to prevent destroy() from destroying the new stream
+                        this.options.body = undefined;
+                        this.destroy();
+                        // Clean up the old stream resource if it's a stream and different from new body
+                        // (edge case: if old and new are same stream object, don't destroy it)
+                        if (is.nodeStream(oldBody) && oldBody !== bodyAfterHooks) {
+                            oldBody.destroy();
+                        }
+                        // Restore new body for promise wrapper's identity check
+                        // We bypass the setter because it validates stream.readable (which fails for destroyed request)
+                        // Type assertion is necessary here to access private _internals without exposing internal API
+                        if (is.nodeStream(bodyAfterHooks) && (bodyAfterHooks.readableEnded || bodyAfterHooks.destroyed)) {
+                            throw new TypeError('The reassigned stream body must be readable. Ensure you provide a fresh, readable stream in the beforeRetry hook.');
+                        }
+                        this.options._internals.body = bodyAfterHooks;
+                    }
+                    else {
+                        // Body wasn't reassigned - use normal destroy flow which handles body cleanup
+                        this.destroy();
+                        // Note: We do NOT restore the body reference here. The stream was destroyed by _destroy()
+                        // and should not be accessed. The promise wrapper will see that body identity hasn't changed
+                        // and will detect it's a consumed stream, which is the correct behavior.
+                    }
+                    // Publish retry event
+                    publishRetry({
+                        requestId: this._requestId,
+                        retryCount: this.retryCount + 1,
+                        error: typedError,
+                        delay: backoff,
+                    });
                     this.emit('retry', this.retryCount + 1, error, (updatedOptions) => {
                         const request = new Request(options.url, updatedOptions, options);
                         request.retryCount = this.retryCount + 1;
@@ -403,8 +436,15 @@ export default class Request extends Duplex {
                 timings.phases.total = timings.end - timings.start;
             }
         }
-        if (error !== null && !is.undefined(error) && !(error instanceof RequestError)) {
-            error = new RequestError(error.message, error, this);
+        // Preserve custom errors returned by beforeError hooks.
+        // For other errors, wrap non-RequestError instances for consistency.
+        if (error !== null && !is.undefined(error)) {
+            const processedByHooks = error instanceof Error && errorsProcessedByHooks.has(error);
+            if (!processedByHooks && !(error instanceof RequestError)) {
+                error = error instanceof Error
+                    ? new RequestError(error.message, error, this)
+                    : new RequestError(String(error), {}, this);
+            }
         }
         callback(error);
     }
@@ -423,9 +463,9 @@ export default class Request extends Duplex {
     }
     _checkContentLengthMismatch() {
         if (this.options.strictContentLength && this._expectedContentLength !== undefined) {
-            // Use ByteCounter's count when available (for compressed responses),
+            // Use compressed bytes count when available (for compressed responses),
             // otherwise use _downloadedSize (for uncompressed responses)
-            const actualSize = this._byteCounter?.count ?? this._downloadedSize;
+            const actualSize = this._compressedBytesCount ?? this._downloadedSize;
             if (actualSize !== this._expectedContentLength) {
                 this._beforeError(new ReadError({
                     message: `Content-Length mismatch: expected ${this._expectedContentLength} bytes, received ${actualSize} bytes`,
@@ -528,9 +568,9 @@ export default class Request extends Duplex {
             // When strictContentLength is enabled, track compressed bytes by listening to
             // the native response's data events before decompression
             if (options.strictContentLength) {
-                this._byteCounter = new ByteCounter();
+                this._compressedBytesCount = 0;
                 this._nativeResponse.on('data', (chunk) => {
-                    this._byteCounter.count += chunk.length;
+                    this._compressedBytesCount += byteLength(chunk);
                 });
             }
             response = decompressResponse(response);
@@ -548,6 +588,14 @@ export default class Request extends Duplex {
         this._isFromCache = typedResponse.isFromCache;
         this._responseSize = Number(response.headers['content-length']) || undefined;
         this.response = typedResponse;
+        // Publish response start event
+        publishResponseStart({
+            requestId: this._requestId,
+            url: typedResponse.url,
+            statusCode,
+            headers: response.headers,
+            isFromCache: typedResponse.isFromCache,
+        });
         // Workaround for http-timer bug: when connecting to an IP address (no DNS lookup),
         // http-timer sets lookup = connect instead of lookup = socket, resulting in
         // dns = lookup - socket being a small positive number instead of 0.
@@ -560,6 +608,15 @@ export default class Request extends Duplex {
             // Recalculate TCP time to be the full time from socket to connect
             timings.phases.tcp = timings.connect - timings.socket;
         }
+        // Workaround for http-timer limitation with HTTP/2:
+        // When using HTTP/2, the socket is a proxy that http-timer discards,
+        // so lookup, connect, and secureConnect events are never captured.
+        // This results in phases.request being NaN (undefined - undefined).
+        // Set it to undefined to be consistent with other unavailable timings.
+        // See https://github.com/sindresorhus/got/issues/1958
+        if (timings && Number.isNaN(timings.phases.request)) {
+            timings.phases.request = undefined;
+        }
         response.once('error', (error) => {
             this._aborted = true;
             // Force clean-up, because some packages don't do this.
@@ -662,12 +719,18 @@ export default class Request extends Duplex {
                         redirectUrl.password = updatedOptions.password;
                     }
                     this.redirectUrls.push(redirectUrl);
-                    updatedOptions.prefixUrl = '';
                     updatedOptions.url = redirectUrl;
                     for (const hook of updatedOptions.hooks.beforeRedirect) {
                         // eslint-disable-next-line no-await-in-loop
                         await hook(updatedOptions, typedResponse);
                     }
+                    // Publish redirect event
+                    publishRedirect({
+                        requestId: this._requestId,
+                        fromUrl: url.toString(),
+                        toUrl: redirectUrl.toString(),
+                        statusCode,
+                    });
                     this.emit('redirect', updatedOptions, typedResponse);
                     this.options = updatedOptions;
                     await this._makeRequest();
@@ -711,6 +774,14 @@ export default class Request extends Duplex {
             }
             this._responseSize = this._downloadedSize;
             this.emit('downloadProgress', this.downloadProgress);
+            // Publish response end event
+            publishResponseEnd({
+                requestId: this._requestId,
+                url: typedResponse.url,
+                statusCode,
+                bodySize: this._downloadedSize,
+                timings: this.timings,
+            });
             this.push(null);
         });
         this.emit('downloadProgress', this.downloadProgress);
@@ -788,6 +859,13 @@ export default class Request extends Duplex {
     _onRequest(request) {
         const { options } = this;
         const { timeout, url } = options;
+        // Publish request start event
+        publishRequestStart({
+            requestId: this._requestId,
+            url: url?.toString() ?? '',
+            method: options.method,
+            headers: options.headers,
+        });
         timer(request);
         this._cancelTimeouts = timedOut(request, timeout, url);
         if (this.options.http2) {
@@ -877,47 +955,111 @@ export default class Request extends Duplex {
         }
     }
     _prepareCache(cache) {
-        if (!cacheableStore.has(cache)) {
-            const cacheableRequest = new CacheableRequest(((requestOptions, handler) => {
-                const result = requestOptions._request(requestOptions, handler);
-                // TODO: remove this when `cacheable-request` supports async request functions.
-                if (is.promise(result)) {
-                    // We only need to implement the error handler in order to support HTTP2 caching.
-                    // The result will be a promise anyway.
-                    // @ts-expect-error ignore
-                    result.once = (event, handler) => {
-                        if (event === 'error') {
-                            (async () => {
-                                try {
-                                    await result;
-                                }
-                                catch (error) {
-                                    handler(error);
-                                }
-                            })();
+        if (cacheableStore.has(cache)) {
+            return;
+        }
+        const cacheableRequest = new CacheableRequest(((requestOptions, handler) => {
+            /**
+            Wraps the cacheable-request handler to run beforeCache hooks.
+            These hooks control caching behavior by:
+            - Directly mutating the response object (changes apply to what gets cached)
+            - Returning `false` to prevent caching
+            - Returning `void`/`undefined` to use default caching behavior
+
+            Hooks use direct mutation - they can modify response.headers, response.statusCode, etc.
+            Mutations take effect immediately and determine what gets cached.
+            */
+            const wrappedHandler = handler ? (response) => {
+                const { beforeCacheHooks, gotRequest } = requestOptions;
+                // Early return if no hooks - cache the original response
+                if (!beforeCacheHooks || beforeCacheHooks.length === 0) {
+                    handler(response);
+                    return;
+                }
+                try {
+                    // Call each beforeCache hook with the response
+                    // Hooks can directly mutate the response - mutations take effect immediately
+                    for (const hook of beforeCacheHooks) {
+                        const result = hook(response);
+                        if (result === false) {
+                            // Prevent caching by adding no-cache headers
+                            // Mutate the response directly to add headers
+                            response.headers['cache-control'] = 'no-cache, no-store, must-revalidate';
+                            response.headers.pragma = 'no-cache';
+                            response.headers.expires = '0';
+                            handler(response);
+                            // Don't call remaining hooks - we've decided not to cache
+                            return;
                         }
-                        else if (event === 'abort' || event === 'destroy') {
-                            // The empty catch is needed here in case when
-                            // it rejects before it's `await`ed in `_makeRequest`.
-                            (async () => {
-                                try {
-                                    const request = (await result);
-                                    request.once(event, handler);
-                                }
-                                catch { }
-                            })();
+                        if (is.promise(result)) {
+                            // BeforeCache hooks must be synchronous because cacheable-request's handler is synchronous
+                            throw new TypeError('beforeCache hooks must be synchronous. The hook returned a Promise, but this hook must return synchronously. If you need async logic, use beforeRequest hook instead.');
                         }
-                        else {
-                            /* istanbul ignore next: safety check */
-                            throw new Error(`Unknown HTTP2 promise event: ${event}`);
+                        if (result !== undefined) {
+                            // Hooks should return false or undefined only
+                            // Mutations work directly - no need to return the response
+                            throw new TypeError('beforeCache hook must return false or undefined. To modify the response, mutate it directly.');
                         }
-                        return result;
-                    };
+                        // Else: void/undefined = continue
+                    }
                 }
-                return result;
-            }), cache);
-            cacheableStore.set(cache, cacheableRequest.request());
-        }
+                catch (error) {
+                    // Convert hook errors to RequestError and propagate
+                    // This is consistent with how other hooks handle errors
+                    if (gotRequest) {
+                        gotRequest._beforeError(error instanceof RequestError ? error : new RequestError(error.message, error, gotRequest));
+                        // Don't call handler when error was propagated successfully
+                        return;
+                    }
+                    // If gotRequest is missing, log the error to aid debugging
+                    // We still call the handler to prevent the request from hanging
+                    console.error('Got: beforeCache hook error (request context unavailable):', error);
+                    // Call handler with response (potentially partially modified)
+                    handler(response);
+                    return;
+                }
+                // All hooks ran successfully
+                // Cache the response with any mutations applied
+                handler(response);
+            } : handler;
+            const result = requestOptions._request(requestOptions, wrappedHandler);
+            // TODO: remove this when `cacheable-request` supports async request functions.
+            if (is.promise(result)) {
+                // We only need to implement the error handler in order to support HTTP2 caching.
+                // The result will be a promise anyway.
+                // @ts-expect-error ignore
+                result.once = (event, handler) => {
+                    if (event === 'error') {
+                        (async () => {
+                            try {
+                                await result;
+                            }
+                            catch (error) {
+                                handler(error);
+                            }
+                        })();
+                    }
+                    else if (event === 'abort' || event === 'destroy') {
+                        // The empty catch is needed here in case when
+                        // it rejects before it's `await`ed in `_makeRequest`.
+                        (async () => {
+                            try {
+                                const request = (await result);
+                                request.once(event, handler);
+                            }
+                            catch { }
+                        })();
+                    }
+                    else {
+                        /* istanbul ignore next: safety check */
+                        throw new Error(`Unknown HTTP2 promise event: ${event}`);
+                    }
+                    return result;
+                };
+            }
+            return result;
+        }), cache);
+        cacheableStore.set(cache, cacheableRequest.request());
     }
     async _createCacheableRequest(url, options) {
         return new Promise((resolve, reject) => {
@@ -966,7 +1108,14 @@ export default class Request extends Duplex {
             }
         }
         if (options.decompress && is.undefined(headers['accept-encoding'])) {
-            headers['accept-encoding'] = supportsBrotli ? 'gzip, deflate, br' : 'gzip, deflate';
+            const encodings = ['gzip', 'deflate'];
+            if (supportsBrotli) {
+                encodings.push('br');
+            }
+            if (supportsZstd) {
+                encodings.push('zstd');
+            }
+            headers['accept-encoding'] = encodings.join(', ');
         }
         if (username || password) {
             const credentials = Buffer.from(`${username}:${password}`).toString('base64');
@@ -979,8 +1128,6 @@ export default class Request extends Duplex {
                 headers.cookie = cookieString;
             }
         }
-        // Reset `prefixUrl`
-        options.prefixUrl = '';
         let request;
         for (const hook of options.hooks.beforeRequest) {
             // eslint-disable-next-line no-await-in-loop
@@ -998,6 +1145,8 @@ export default class Request extends Duplex {
             this._requestOptions._request = request;
             this._requestOptions.cache = options.cache;
             this._requestOptions.body = options.body;
+            this._requestOptions.beforeCacheHooks = options.hooks.beforeCache;
+            this._requestOptions.gotRequest = this;
             try {
                 this._prepareCache(options.cache);
             }
@@ -1024,15 +1173,15 @@ export default class Request extends Duplex {
             if (isClientRequest(requestOrResponse)) {
                 this._onRequest(requestOrResponse);
             }
-            else if (this.writable) {
+            else if (this.writableEnded) {
+                void this._onResponse(requestOrResponse);
+            }
+            else {
                 this.once('finish', () => {
                     void this._onResponse(requestOrResponse);
                 });
                 this._sendBody();
             }
-            else {
-                void this._onResponse(requestOrResponse);
-            }
         }
         catch (error) {
             if (error instanceof CacheableCacheError) {
@@ -1049,15 +1198,35 @@ export default class Request extends Duplex {
                 // See https://github.com/sindresorhus/got/issues/2103
             }
             else if (this.options) {
-                for (const hook of this.options.hooks.beforeError) {
-                    // eslint-disable-next-line no-await-in-loop
-                    error = await hook(error);
+                const hooks = this.options.hooks.beforeError;
+                if (hooks.length > 0) {
+                    for (const hook of hooks) {
+                        // eslint-disable-next-line no-await-in-loop
+                        error = await hook(error);
+                        // Validate hook return value
+                        if (!(error instanceof Error)) {
+                            throw new TypeError(`The \`beforeError\` hook must return an Error instance. Received ${is.string(error) ? 'string' : String(typeof error)}.`);
+                        }
+                    }
+                    // Mark this error as processed by hooks so _destroy preserves custom error types.
+                    // Only mark non-RequestError errors, since RequestErrors are already preserved
+                    // by the instanceof check in _destroy (line 642).
+                    if (!(error instanceof RequestError)) {
+                        errorsProcessedByHooks.add(error);
+                    }
                 }
             }
         }
         catch (error_) {
             error = new RequestError(error_.message, error_, this);
         }
+        // Publish error event
+        publishError({
+            requestId: this._requestId,
+            url: this.options?.url?.toString() ?? '',
+            error,
+            timings: this.timings,
+        });
         this.destroy(error);
         // Manually emit error for Promise API to ensure it receives it.
         // Node.js streams may not re-emit if an error was already emitted during retry attempts.
@@ -1080,7 +1249,9 @@ export default class Request extends Duplex {
         this._request.write(chunk, encoding, (error) => {
             // The `!destroyed` check is required to prevent `uploadProgress` being emitted after the stream was destroyed
             if (!error && !this._request.destroyed) {
-                this._uploadedSize += Buffer.byteLength(chunk, encoding);
+                // For strings, encode them first to measure the actual bytes that will be sent
+                const bytes = typeof chunk === 'string' ? Buffer.from(chunk, encoding) : chunk;
+                this._uploadedSize += byteLength(bytes);
                 const progress = this.uploadProgress;
                 if (progress.percent < 1) {
                     this.emit('uploadProgress', progress);
@@ -1183,4 +1354,10 @@ export default class Request extends Duplex {
     get reusedSocket() {
         return this._request?.reusedSocket;
     }
+    /**
+    Whether the stream is read-only. Returns `true` when `body`, `json`, or `form` options are provided.
+    */
+    get isReadonly() {
+        return !is.undefined(this.options?.body) || !is.undefined(this.options?.json) || !is.undefined(this.options?.form);
+    }
 }