@clickhouse/client 1.18.0 → 1.18.2-head.084b623.1

package/dist/connection/node_base_connection.js

@@ -6,37 +6,20 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.NodeBaseConnection = void 0;
 const client_common_1 = require("@clickhouse/client-common");
 const crypto_1 = __importDefault(require("crypto"));
-const stream_1 = __importDefault(require("stream"));
-const zlib_1 = __importDefault(require("zlib"));
 const utils_1 = require("../utils");
-const compression_1 = require("./compression");
-const stream_2 = require("./stream");
+const stream_1 = require("./stream");
+const socket_pool_1 = require("./socket_pool");
 class NodeBaseConnection {
     params;
     agent;
     defaultAuthHeader;
     defaultHeaders;
-    jsonHandling;
-    knownSockets = new WeakMap();
-    idleSocketTTL;
     connectionId = crypto_1.default.randomUUID();
-    socketCounter = 0;
-    // For overflow concerns:
-    //   node -e 'console.log(Number.MAX_SAFE_INTEGER / (1_000_000 * 60 * 60 * 24 * 366))'
-    // gives 284 years of continuous operation at 1M requests per second
-    // before overflowing the 53-bit integer
-    requestCounter = 0;
-    getNewSocketId() {
-        this.socketCounter += 1;
-        return `${this.connectionId}:${this.socketCounter}`;
-    }
-    getNewRequestId() {
-        this.requestCounter += 1;
-        return `${this.connectionId}:${this.requestCounter}`;
-    }
+    socketPool;
     constructor(params, agent) {
         this.params = params;
         this.agent = agent;
+        this.socketPool = new socket_pool_1.SocketPool(this.connectionId, this.params, this.createClientRequest.bind(this), this.agent);
         if (params.auth.type === 'Credentials') {
             this.defaultAuthHeader = `Basic ${Buffer.from(`${params.auth.username}:${params.auth.password}`).toString('base64')}`;
         }
@@ -51,18 +34,13 @@ class NodeBaseConnection {
             Connection: this.params.keep_alive.enabled ? 'keep-alive' : 'close',
             'User-Agent': (0, utils_1.getUserAgent)(this.params.application_id),
         };
-        this.idleSocketTTL = params.keep_alive.idle_socket_ttl;
-        this.jsonHandling = params.json ?? {
-            parse: JSON.parse,
-            stringify: JSON.stringify,
-        };
     }
     async ping(params) {
         const { log_writer, log_level } = this.params;
         const query_id = this.getQueryId(params.query_id);
         const { controller, controllerCleanup } = this.getAbortController(params);
-        let result;
         try {
+            let result;
             if (params.select) {
                 const searchParams = (0, client_common_1.toSearchParams)({
                     database: undefined,
@@ -70,9 +48,9 @@ class NodeBaseConnection {
                     query_id,
                 });
                 result = await this.request({
+                    query: PingQuery,
                     method: 'GET',
                     url: (0, client_common_1.transformUrl)({ url: this.params.url, searchParams }),
-                    query: PingQuery,
                     abort_signal: controller.signal,
                     headers: this.buildRequestHeaders(),
                     query_id,
@@ -82,17 +60,17 @@ class NodeBaseConnection {
             }
             else {
                 result = await this.request({
+                    query: 'ping',
                     method: 'GET',
                     url: (0, client_common_1.transformUrl)({ url: this.params.url, pathname: '/ping' }),
                     abort_signal: controller.signal,
                     headers: this.buildRequestHeaders(),
-                    query: 'ping',
                     query_id,
                     log_writer,
                     log_level,
                 }, 'Ping');
             }
-            await (0, stream_2.drainStream)({
+            await (0, stream_1.drainStreamInternal)({
                 op: 'Ping',
                 log_writer,
                 query_id,
@@ -205,7 +183,7 @@ class NodeBaseConnection {
                 log_writer,
                 log_level,
             }, 'Insert');
-            await (0, stream_2.drainStream)({
+            await (0, stream_1.drainStreamInternal)({
                 op: 'Insert',
                 log_writer,
                 query_id,
@@ -250,9 +228,6 @@ class NodeBaseConnection {
                     operation: 'Command',
                     connection_id: this.connectionId,
                     query_id,
-                    query: this.params.unsafeLogUnredactedQueries
-                        ? params.query
-                        : undefined,
                 },
             });
         }
@@ -280,7 +255,7 @@ class NodeBaseConnection {
         }
         // ignore the response stream and release the socket immediately
         const drainStartTime = Date.now();
-        await (0, stream_2.drainStream)({
+        await (0, stream_1.drainStreamInternal)({
             op: 'Command',
             log_writer,
             query_id,
@@ -362,14 +337,8 @@ class NodeBaseConnection {
             },
         };
     }
-    logRequestError({ op, err, query_id, query_params, search_params, extra_args, }) {
+    logRequestError({ op, err, query_id, query_params, extra_args, }) {
         if (this.params.log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
-            // Redact query parameter from search params unless explicitly allowed
-            if (!this.params.unsafeLogUnredactedQueries && search_params) {
-                // Clone to avoid mutating the original search params
-                search_params = new URLSearchParams(search_params);
-                search_params.delete('query');
-            }
             this.params.log_writer.error({
                 message: this.httpRequestErrorMessage(op),
                 err: err,
@@ -377,10 +346,6 @@ class NodeBaseConnection {
                     operation: op,
                     connection_id: this.connectionId,
                     query_id,
-                    query: this.params.unsafeLogUnredactedQueries
-                        ? query_params.query
-                        : undefined,
-                    search_params: search_params?.toString(),
                     with_abort_signal: query_params.abort_signal !== undefined,
                     session_id: query_params.session_id,
                     ...extra_args,
@@ -391,27 +356,6 @@ class NodeBaseConnection {
     httpRequestErrorMessage(op) {
         return `${op}: HTTP request error.`;
     }
-    parseSummary(op, response) {
-        const summaryHeader = response.headers['x-clickhouse-summary'];
-        if (typeof summaryHeader === 'string') {
-            try {
-                return this.jsonHandling.parse(summaryHeader);
-            }
-            catch (err) {
-                if (this.params.log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
-                    this.params.log_writer.error({
-                        message: `${op}: failed to parse X-ClickHouse-Summary header.`,
-                        args: {
-                            operation: op,
-                            connection_id: this.connectionId,
-                            'X-ClickHouse-Summary': summaryHeader,
-                        },
-                        err: err,
-                    });
-                }
-            }
-        }
-    }
     async runExec(params) {
         const { log_writer, log_level } = this.params;
         const query_id = params.query_id;
@@ -480,425 +424,7 @@ class NodeBaseConnection {
         }
     }
     async request(params, op) {
-        // allows the event loop to process the idle socket timers, if the CPU load is high
-        // otherwise, we can occasionally get an expired socket, see https://github.com/ClickHouse/clickhouse-js/issues/294
-        await (0, client_common_1.sleep)(0);
-        const { log_writer, query_id, log_level } = params;
-        const currentStackTrace = this.params.capture_enhanced_stack_trace
-            ? (0, client_common_1.getCurrentStackTrace)()
-            : undefined;
-        const requestTimeout = this.params.request_timeout;
-        return new Promise((resolve, reject) => {
-            const start = Date.now();
-            const request = this.createClientRequest(params);
-            const request_id = this.getNewRequestId();
-            function onError(e) {
-                removeRequestListeners();
-                const err = (0, client_common_1.enhanceStackTrace)(e, currentStackTrace);
-                reject(err);
-            }
-            let responseStream;
-            const onResponse = async (_response) => {
-                if (this.params.log_level <= client_common_1.ClickHouseLogLevel.DEBUG) {
-                    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-                    const { authorization, host, ...headers } = request.getHeaders();
-                    const duration = Date.now() - start;
-                    // Redact query parameter from URL search params unless explicitly allowed
-                    let searchParams = params.url.searchParams;
-                    if (!this.params.unsafeLogUnredactedQueries) {
-                        // Clone to avoid mutating the original search params
-                        searchParams = new URLSearchParams(searchParams);
-                        searchParams.delete('query');
-                    }
-                    this.params.log_writer.debug({
-                        module: 'HTTP Adapter',
-                        message: `${op}: got a response from ClickHouse`,
-                        args: {
-                            operation: op,
-                            connection_id: this.connectionId,
-                            query_id,
-                            request_id,
-                            request_method: params.method,
-                            request_path: params.url.pathname,
-                            request_params: searchParams.toString(),
-                            request_headers: headers,
-                            response_status: _response.statusCode,
-                            response_headers: _response.headers,
-                            response_time_ms: duration,
-                        },
-                    });
-                }
-                const tryDecompressResponseStream = params.try_decompress_response_stream ?? true;
-                const ignoreErrorResponse = params.ignore_error_response ?? false;
-                // even if the stream decompression is disabled, we have to decompress it in case of an error
-                const isFailedResponse = !(0, client_common_1.isSuccessfulResponse)(_response.statusCode);
-                if (tryDecompressResponseStream ||
-                    (isFailedResponse && !ignoreErrorResponse)) {
-                    const decompressionResult = (0, compression_1.decompressResponse)(_response, log_writer, log_level);
-                    if ((0, compression_1.isDecompressionError)(decompressionResult)) {
-                        const err = (0, client_common_1.enhanceStackTrace)(decompressionResult.error, currentStackTrace);
-                        return reject(err);
-                    }
-                    responseStream = decompressionResult.response;
-                }
-                else {
-                    responseStream = _response;
-                }
-                if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
-                    log_writer.trace({
-                        message: `${op}: response stream created`,
-                        args: {
-                            operation: op,
-                            connection_id: this.connectionId,
-                            query_id,
-                            request_id,
-                            stream_state: {
-                                readable: responseStream.readable,
-                                readableEnded: responseStream.readableEnded,
-                                readableLength: responseStream.readableLength,
-                            },
-                            is_failed_response: isFailedResponse,
-                            will_decompress: tryDecompressResponseStream,
-                        },
-                    });
-                }
-                if (isFailedResponse && !ignoreErrorResponse) {
-                    try {
-                        const errorMessage = await (0, utils_1.getAsText)(responseStream);
-                        const err = (0, client_common_1.enhanceStackTrace)((0, client_common_1.parseError)(errorMessage), currentStackTrace);
-                        reject(err);
-                    }
-                    catch (e) {
-                        // If the ClickHouse response is malformed
-                        const err = (0, client_common_1.enhanceStackTrace)(e, currentStackTrace);
-                        reject(err);
-                    }
-                }
-                else {
-                    return resolve({
-                        stream: responseStream,
-                        summary: params.parse_summary
-                            ? this.parseSummary(op, _response)
-                            : undefined,
-                        response_headers: { ..._response.headers },
-                        http_status_code: _response.statusCode ?? undefined,
-                    });
-                }
-            };
-            function onAbort() {
-                // Prefer 'abort' event since it always triggered unlike 'error' and 'close'
-                // see the full sequence of events https://nodejs.org/api/http.html#httprequesturl-options-callback
-                removeRequestListeners();
-                request.once('error', function () {
-                    /**
-                     * catch "Error: ECONNRESET" error which shouldn't be reported to users.
-                     * see the full sequence of events https://nodejs.org/api/http.html#httprequesturl-options-callback
-                     * */
-                });
-                const err = (0, client_common_1.enhanceStackTrace)(new Error('The user aborted a request.'), currentStackTrace);
-                reject(err);
-            }
-            function onClose() {
-                // Adapter uses 'close' event to clean up listeners after the successful response.
-                // It's necessary in order to handle 'abort' and 'timeout' events while response is streamed.
-                // It's always the last event, according to https://nodejs.org/docs/latest-v14.x/api/http.html#http_http_request_url_options_callback
-                removeRequestListeners();
-            }
-            function pipeStream() {
-                // if request.end() was called due to no data to send
-                if (request.writableEnded) {
-                    return;
-                }
-                const bodyStream = (0, utils_1.isStream)(params.body)
-                    ? params.body
-                    : stream_1.default.Readable.from([params.body]);
-                const callback = (e) => {
-                    if (e) {
-                        removeRequestListeners();
-                        const err = (0, client_common_1.enhanceStackTrace)(e, currentStackTrace);
-                        reject(err);
-                    }
-                };
-                if (params.enable_request_compression) {
-                    stream_1.default.pipeline(bodyStream, zlib_1.default.createGzip(), request, callback);
-                }
-                else {
-                    stream_1.default.pipeline(bodyStream, request, callback);
-                }
-            }
-            const onSocket = (socket) => {
-                try {
-                    if (this.params.keep_alive.enabled &&
-                        this.params.keep_alive.idle_socket_ttl > 0) {
-                        const socketInfo = this.knownSockets.get(socket);
-                        // It is the first time we've encountered this socket,
-                        // so it doesn't have the idle timeout handler attached to it
-                        if (socketInfo === undefined) {
-                            const socket_id = this.getNewSocketId();
-                            if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
-                                log_writer.trace({
-                                    message: `${op}: using a fresh socket, setting up a new 'free' listener`,
-                                    args: {
-                                        operation: op,
-                                        connection_id: this.connectionId,
-                                        query_id,
-                                        request_id,
-                                        socket_id,
-                                    },
-                                });
-                            }
-                            const newSocketInfo = {
-                                id: socket_id,
-                                idle_timeout_handle: undefined,
-                                usage_count: 1,
-                            };
-                            this.knownSockets.set(socket, newSocketInfo);
-                            // When the request is complete and the socket is released,
-                            // make sure that the socket is removed after `idleSocketTTL`.
-                            socket.on('free', () => {
-                                if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
-                                    log_writer.trace({
-                                        message: `${op}: socket was released`,
-                                        args: {
-                                            operation: op,
-                                            connection_id: this.connectionId,
-                                            query_id,
-                                            request_id,
-                                            socket_id,
-                                        },
-                                    });
-                                }
-                                // Avoiding the built-in socket.timeout() method usage here,
-                                // as we don't want to clash with the actual request timeout.
-                                const idleTimeoutHandle = setTimeout(() => {
-                                    if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
-                                        log_writer.trace({
-                                            message: `${op}: removing idle socket`,
-                                            args: {
-                                                operation: op,
-                                                connection_id: this.connectionId,
-                                                query_id,
-                                                request_id,
-                                                socket_id,
-                                                idle_socket_ttl_ms: this.idleSocketTTL,
-                                            },
-                                        });
-                                    }
-                                    this.knownSockets.delete(socket);
-                                    socket.destroy();
-                                }, this.idleSocketTTL).unref();
-                                newSocketInfo.idle_timeout_handle = idleTimeoutHandle;
-                            });
-                            const cleanup = (eventName) => () => {
-                                const maybeSocketInfo = this.knownSockets.get(socket);
-                                // clean up a possibly dangling idle timeout handle (preventing leaks)
-                                if (maybeSocketInfo?.idle_timeout_handle) {
-                                    clearTimeout(maybeSocketInfo.idle_timeout_handle);
-                                }
-                                if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
-                                    log_writer.trace({
-                                        message: `${op}: received '${eventName}' event, 'free' listener removed`,
-                                        args: {
-                                            operation: op,
-                                            connection_id: this.connectionId,
-                                            query_id,
-                                            request_id,
-                                            socket_id,
-                                            event: eventName,
-                                        },
-                                    });
-                                }
-                                if (log_level <= client_common_1.ClickHouseLogLevel.WARN) {
-                                    if (responseStream && !responseStream.readableEnded) {
-                                        log_writer.warn({
-                                            message: `${op}: socket was closed or ended before the response was fully read. ` +
-                                                'This can potentially result in an uncaught ECONNRESET error! ' +
-                                                'Consider fully consuming, draining, or destroying the response stream.',
-                                            args: {
-                                                operation: op,
-                                                connection_id: this.connectionId,
-                                                query_id,
-                                                request_id,
-                                                socket_id,
-                                                event: eventName,
-                                                query: this.params.unsafeLogUnredactedQueries
-                                                    ? params.query
-                                                    : undefined,
-                                            },
-                                        });
-                                    }
-                                }
-                            };
-                            socket.once('end', cleanup('end'));
-                            socket.once('close', cleanup('close'));
-                        }
-                        else {
-                            clearTimeout(socketInfo.idle_timeout_handle);
-                            socketInfo.idle_timeout_handle = undefined;
-                            if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
-                                log_writer.trace({
-                                    message: `${op}: reusing socket`,
-                                    args: {
-                                        operation: op,
-                                        connection_id: this.connectionId,
-                                        query_id,
-                                        request_id,
-                                        socket_id: socketInfo.id,
-                                        usage_count: socketInfo.usage_count,
-                                    },
-                                });
-                            }
-                            socketInfo.usage_count++;
-                        }
-                    }
-                }
-                catch (e) {
-                    if (log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
-                        log_writer.error({
-                            message: `${op}: an error occurred while housekeeping the idle sockets`,
-                            err: e,
-                            args: {
-                                operation: op,
-                                connection_id: this.connectionId,
-                                query_id,
-                                request_id,
-                            },
-                        });
-                    }
-                }
-                // Socket is "prepared" with idle handlers, continue with our request
-                pipeStream();
-                // This is for request timeout only. Surprisingly, it is not always enough to set in the HTTP request.
-                // The socket won't be destroyed, and it will be returned to the pool.
-                if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
-                    const socketInfo = this.knownSockets.get(socket);
-                    if (socketInfo) {
-                        log_writer.trace({
-                            message: `${op}: setting up request timeout`,
-                            args: {
-                                operation: op,
-                                connection_id: this.connectionId,
-                                query_id,
-                                request_id,
-                                socket_id: socketInfo.id,
-                                timeout_ms: requestTimeout,
-                            },
-                        });
-                    }
-                    else {
-                        log_writer.trace({
-                            message: `${op}: setting up request timeout on a socket`,
-                            args: {
-                                operation: op,
-                                connection_id: this.connectionId,
-                                query_id,
-                                request_id,
-                                timeout_ms: requestTimeout,
-                            },
-                        });
-                    }
-                }
-                socket.setTimeout(this.params.request_timeout, onTimeout);
-            };
-            const onTimeout = () => {
-                removeRequestListeners();
-                if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
-                    const socket = request.socket;
-                    const maybeSocketInfo = socket
-                        ? this.knownSockets.get(socket)
-                        : undefined;
-                    const socketState = request.socket
-                        ? {
-                            connecting: request.socket.connecting,
-                            pending: request.socket.pending,
-                            destroyed: request.socket.destroyed,
-                            readyState: request.socket.readyState,
-                        }
-                        : undefined;
-                    const responseStreamState = responseStream
-                        ? {
-                            readable: responseStream.readable,
-                            readableEnded: responseStream.readableEnded,
-                            readableLength: responseStream.readableLength,
-                        }
-                        : undefined;
-                    log_writer.trace({
-                        message: `${op}: timeout occurred`,
-                        args: {
-                            operation: op,
-                            connection_id: this.connectionId,
-                            query_id,
-                            request_id,
-                            socket_id: maybeSocketInfo?.id,
-                            timeout_ms: requestTimeout,
-                            socket_state: socketState,
-                            response_stream_state: responseStreamState,
-                            has_response_stream: responseStream !== undefined,
-                        },
-                    });
-                }
-                const err = (0, client_common_1.enhanceStackTrace)(new Error('Timeout error.'), currentStackTrace);
-                try {
-                    request.destroy();
-                }
-                catch (e) {
-                    if (log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
-                        log_writer.error({
-                            message: `${op}: An error occurred while destroying the request`,
-                            err: e,
-                            args: {
-                                operation: op,
-                                connection_id: this.connectionId,
-                                query_id,
-                                request_id,
-                            },
-                        });
-                    }
-                }
-                reject(err);
-            };
-            function removeRequestListeners() {
-                if (request.socket !== null) {
-                    request.socket.setTimeout(0); // reset previously set timeout
-                    request.socket.removeListener('timeout', onTimeout);
-                }
-                request.removeListener('socket', onSocket);
-                request.removeListener('response', onResponse);
-                request.removeListener('error', onError);
-                request.removeListener('close', onClose);
-                if (params.abort_signal !== undefined) {
-                    request.removeListener('abort', onAbort);
-                }
-            }
-            request.on('socket', onSocket);
-            request.on('response', onResponse);
-            request.on('error', onError);
-            request.on('close', onClose);
-            if (params.abort_signal !== undefined) {
-                params.abort_signal.addEventListener('abort', onAbort, {
-                    once: true,
-                });
-            }
-            if (!params.body) {
-                try {
-                    return request.end();
-                }
-                catch (e) {
-                    if (log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
-                        log_writer.error({
-                            message: `${op}: an error occurred while ending the request without body`,
-                            err: e,
-                            args: {
-                                operation: op,
-                                connection_id: this.connectionId,
-                                query_id,
-                                request_id,
-                            },
-                        });
-                    }
-                }
-            }
-        });
+        return this.socketPool.request(params, op);
     }
 }
 exports.NodeBaseConnection = NodeBaseConnection;