@clickhouse/client 1.17.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connection/compression.d.ts +2 -1
- package/dist/connection/compression.js +8 -5
- package/dist/connection/compression.js.map +1 -1
- package/dist/connection/node_base_connection.d.ts +11 -3
- package/dist/connection/node_base_connection.js +403 -153
- package/dist/connection/node_base_connection.js.map +1 -1
- package/dist/connection/node_custom_agent_connection.js +1 -6
- package/dist/connection/node_custom_agent_connection.js.map +1 -1
- package/dist/connection/stream.d.ts +9 -1
- package/dist/connection/stream.js +71 -3
- package/dist/connection/stream.js.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/result_set.js +10 -42
- package/dist/result_set.js.map +1 -1
- package/dist/utils/encoder.js +1 -6
- package/dist/utils/encoder.js.map +1 -1
- package/dist/utils/runtime.js +3 -18
- package/dist/utils/runtime.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +5 -2
|
@@ -12,55 +12,31 @@ const utils_1 = require("../utils");
|
|
|
12
12
|
const compression_1 = require("./compression");
|
|
13
13
|
const stream_2 = require("./stream");
|
|
14
14
|
class NodeBaseConnection {
|
|
15
|
+
params;
|
|
16
|
+
agent;
|
|
17
|
+
defaultAuthHeader;
|
|
18
|
+
defaultHeaders;
|
|
19
|
+
jsonHandling;
|
|
20
|
+
knownSockets = new WeakMap();
|
|
21
|
+
idleSocketTTL;
|
|
22
|
+
connectionId = crypto_1.default.randomUUID();
|
|
23
|
+
socketCounter = 0;
|
|
24
|
+
// For overflow concerns:
|
|
25
|
+
// node -e 'console.log(Number.MAX_SAFE_INTEGER / (1_000_000 * 60 * 60 * 24 * 366))'
|
|
26
|
+
// gives 284 years of continuous operation at 1M requests per second
|
|
27
|
+
// before overflowing the 53-bit integer
|
|
28
|
+
requestCounter = 0;
|
|
29
|
+
getNewSocketId() {
|
|
30
|
+
this.socketCounter += 1;
|
|
31
|
+
return `${this.connectionId}:${this.socketCounter}`;
|
|
32
|
+
}
|
|
33
|
+
getNewRequestId() {
|
|
34
|
+
this.requestCounter += 1;
|
|
35
|
+
return `${this.connectionId}:${this.requestCounter}`;
|
|
36
|
+
}
|
|
15
37
|
constructor(params, agent) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
configurable: true,
|
|
19
|
-
writable: true,
|
|
20
|
-
value: params
|
|
21
|
-
});
|
|
22
|
-
Object.defineProperty(this, "agent", {
|
|
23
|
-
enumerable: true,
|
|
24
|
-
configurable: true,
|
|
25
|
-
writable: true,
|
|
26
|
-
value: agent
|
|
27
|
-
});
|
|
28
|
-
Object.defineProperty(this, "defaultAuthHeader", {
|
|
29
|
-
enumerable: true,
|
|
30
|
-
configurable: true,
|
|
31
|
-
writable: true,
|
|
32
|
-
value: void 0
|
|
33
|
-
});
|
|
34
|
-
Object.defineProperty(this, "defaultHeaders", {
|
|
35
|
-
enumerable: true,
|
|
36
|
-
configurable: true,
|
|
37
|
-
writable: true,
|
|
38
|
-
value: void 0
|
|
39
|
-
});
|
|
40
|
-
Object.defineProperty(this, "jsonHandling", {
|
|
41
|
-
enumerable: true,
|
|
42
|
-
configurable: true,
|
|
43
|
-
writable: true,
|
|
44
|
-
value: void 0
|
|
45
|
-
});
|
|
46
|
-
Object.defineProperty(this, "logger", {
|
|
47
|
-
enumerable: true,
|
|
48
|
-
configurable: true,
|
|
49
|
-
writable: true,
|
|
50
|
-
value: void 0
|
|
51
|
-
});
|
|
52
|
-
Object.defineProperty(this, "knownSockets", {
|
|
53
|
-
enumerable: true,
|
|
54
|
-
configurable: true,
|
|
55
|
-
writable: true,
|
|
56
|
-
value: new WeakMap()
|
|
57
|
-
});
|
|
58
|
-
Object.defineProperty(this, "idleSocketTTL", {
|
|
59
|
-
enumerable: true,
|
|
60
|
-
configurable: true,
|
|
61
|
-
writable: true,
|
|
62
|
-
value: void 0
|
|
63
|
-
});
|
|
38
|
+
this.params = params;
|
|
39
|
+
this.agent = agent;
|
|
64
40
|
if (params.auth.type === 'Credentials') {
|
|
65
41
|
this.defaultAuthHeader = `Basic ${Buffer.from(`${params.auth.username}:${params.auth.password}`).toString('base64')}`;
|
|
66
42
|
}
|
|
@@ -75,7 +51,6 @@ class NodeBaseConnection {
|
|
|
75
51
|
Connection: this.params.keep_alive.enabled ? 'keep-alive' : 'close',
|
|
76
52
|
'User-Agent': (0, utils_1.getUserAgent)(this.params.application_id),
|
|
77
53
|
};
|
|
78
|
-
this.logger = params.log_writer;
|
|
79
54
|
this.idleSocketTTL = params.keep_alive.idle_socket_ttl;
|
|
80
55
|
this.jsonHandling = params.json ?? {
|
|
81
56
|
parse: JSON.parse,
|
|
@@ -83,6 +58,7 @@ class NodeBaseConnection {
|
|
|
83
58
|
};
|
|
84
59
|
}
|
|
85
60
|
async ping(params) {
|
|
61
|
+
const { log_writer, log_level } = this.params;
|
|
86
62
|
const query_id = this.getQueryId(params.query_id);
|
|
87
63
|
const { controller, controllerCleanup } = this.getAbortController(params);
|
|
88
64
|
let result;
|
|
@@ -99,6 +75,9 @@ class NodeBaseConnection {
|
|
|
99
75
|
query: PingQuery,
|
|
100
76
|
abort_signal: controller.signal,
|
|
101
77
|
headers: this.buildRequestHeaders(),
|
|
78
|
+
query_id,
|
|
79
|
+
log_writer,
|
|
80
|
+
log_level,
|
|
102
81
|
}, 'Ping');
|
|
103
82
|
}
|
|
104
83
|
else {
|
|
@@ -108,9 +87,17 @@ class NodeBaseConnection {
|
|
|
108
87
|
abort_signal: controller.signal,
|
|
109
88
|
headers: this.buildRequestHeaders(),
|
|
110
89
|
query: 'ping',
|
|
90
|
+
query_id,
|
|
91
|
+
log_writer,
|
|
92
|
+
log_level,
|
|
111
93
|
}, 'Ping');
|
|
112
94
|
}
|
|
113
|
-
await (0, stream_2.drainStream)(
|
|
95
|
+
await (0, stream_2.drainStream)({
|
|
96
|
+
op: 'Ping',
|
|
97
|
+
log_writer,
|
|
98
|
+
query_id,
|
|
99
|
+
log_level,
|
|
100
|
+
}, result.stream);
|
|
114
101
|
return { success: true };
|
|
115
102
|
}
|
|
116
103
|
catch (error) {
|
|
@@ -118,13 +105,16 @@ class NodeBaseConnection {
|
|
|
118
105
|
// and we don't get unhandled error propagation later
|
|
119
106
|
controller.abort('Ping failed');
|
|
120
107
|
// not an error, as this might be semi-expected
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
108
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.WARN) {
|
|
109
|
+
log_writer.warn({
|
|
110
|
+
message: this.httpRequestErrorMessage('Ping'),
|
|
111
|
+
err: error,
|
|
112
|
+
args: {
|
|
113
|
+
connection_id: this.connectionId,
|
|
114
|
+
query_id,
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
}
|
|
128
118
|
return {
|
|
129
119
|
success: false,
|
|
130
120
|
error: error, // should NOT be propagated to the user
|
|
@@ -135,6 +125,7 @@ class NodeBaseConnection {
|
|
|
135
125
|
}
|
|
136
126
|
}
|
|
137
127
|
async query(params) {
|
|
128
|
+
const { log_writer, log_level } = this.params;
|
|
138
129
|
const query_id = this.getQueryId(params.query_id);
|
|
139
130
|
const clickhouse_settings = (0, client_common_1.withHttpSettings)(params.clickhouse_settings, this.params.compression.decompress_response);
|
|
140
131
|
const searchParams = (0, client_common_1.toSearchParams)({
|
|
@@ -157,6 +148,9 @@ class NodeBaseConnection {
|
|
|
157
148
|
enable_response_compression: enableResponseCompression,
|
|
158
149
|
headers: this.buildRequestHeaders(params),
|
|
159
150
|
query: params.query,
|
|
151
|
+
query_id,
|
|
152
|
+
log_writer,
|
|
153
|
+
log_level,
|
|
160
154
|
}, 'Query');
|
|
161
155
|
return {
|
|
162
156
|
stream,
|
|
@@ -185,6 +179,7 @@ class NodeBaseConnection {
|
|
|
185
179
|
}
|
|
186
180
|
}
|
|
187
181
|
async insert(params) {
|
|
182
|
+
const { log_writer, log_level } = this.params;
|
|
188
183
|
const query_id = this.getQueryId(params.query_id);
|
|
189
184
|
const searchParams = (0, client_common_1.toSearchParams)({
|
|
190
185
|
database: this.params.database,
|
|
@@ -206,8 +201,16 @@ class NodeBaseConnection {
|
|
|
206
201
|
parse_summary: true,
|
|
207
202
|
headers: this.buildRequestHeaders(params),
|
|
208
203
|
query: params.query,
|
|
204
|
+
query_id,
|
|
205
|
+
log_writer,
|
|
206
|
+
log_level,
|
|
209
207
|
}, 'Insert');
|
|
210
|
-
await (0, stream_2.drainStream)(
|
|
208
|
+
await (0, stream_2.drainStream)({
|
|
209
|
+
op: 'Insert',
|
|
210
|
+
log_writer,
|
|
211
|
+
query_id,
|
|
212
|
+
log_level,
|
|
213
|
+
}, stream);
|
|
211
214
|
return { query_id, summary, response_headers, http_status_code };
|
|
212
215
|
}
|
|
213
216
|
catch (err) {
|
|
@@ -229,18 +232,74 @@ class NodeBaseConnection {
|
|
|
229
232
|
}
|
|
230
233
|
}
|
|
231
234
|
async exec(params) {
|
|
235
|
+
const query_id = this.getQueryId(params.query_id);
|
|
232
236
|
return this.runExec({
|
|
233
237
|
...params,
|
|
238
|
+
query_id,
|
|
234
239
|
op: 'Exec',
|
|
235
240
|
});
|
|
236
241
|
}
|
|
237
242
|
async command(params) {
|
|
238
|
-
const {
|
|
243
|
+
const { log_writer, log_level } = this.params;
|
|
244
|
+
const query_id = this.getQueryId(params.query_id);
|
|
245
|
+
const commandStartTime = Date.now();
|
|
246
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
247
|
+
log_writer.trace({
|
|
248
|
+
message: 'Command: operation started',
|
|
249
|
+
args: {
|
|
250
|
+
operation: 'Command',
|
|
251
|
+
connection_id: this.connectionId,
|
|
252
|
+
query_id,
|
|
253
|
+
query: this.params.unsafeLogUnredactedQueries
|
|
254
|
+
? params.query
|
|
255
|
+
: undefined,
|
|
256
|
+
},
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
const { stream, summary, response_headers } = await this.runExec({
|
|
239
260
|
...params,
|
|
261
|
+
query_id,
|
|
240
262
|
op: 'Command',
|
|
241
263
|
});
|
|
264
|
+
const runExecDuration = Date.now() - commandStartTime;
|
|
265
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
266
|
+
log_writer.trace({
|
|
267
|
+
message: 'Command: runExec completed, starting stream drain',
|
|
268
|
+
args: {
|
|
269
|
+
operation: 'Command',
|
|
270
|
+
connection_id: this.connectionId,
|
|
271
|
+
query_id,
|
|
272
|
+
runExec_duration_ms: runExecDuration,
|
|
273
|
+
stream_state: {
|
|
274
|
+
readable: stream.readable,
|
|
275
|
+
readableEnded: stream.readableEnded,
|
|
276
|
+
readableLength: stream.readableLength,
|
|
277
|
+
},
|
|
278
|
+
},
|
|
279
|
+
});
|
|
280
|
+
}
|
|
242
281
|
// ignore the response stream and release the socket immediately
|
|
243
|
-
|
|
282
|
+
const drainStartTime = Date.now();
|
|
283
|
+
await (0, stream_2.drainStream)({
|
|
284
|
+
op: 'Command',
|
|
285
|
+
log_writer,
|
|
286
|
+
query_id,
|
|
287
|
+
log_level,
|
|
288
|
+
}, stream);
|
|
289
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
290
|
+
const drainDuration = Date.now() - drainStartTime;
|
|
291
|
+
const totalDuration = Date.now() - commandStartTime;
|
|
292
|
+
log_writer.trace({
|
|
293
|
+
message: 'Command: operation completed',
|
|
294
|
+
args: {
|
|
295
|
+
operation: 'Command',
|
|
296
|
+
connection_id: this.connectionId,
|
|
297
|
+
query_id,
|
|
298
|
+
drain_duration_ms: drainDuration,
|
|
299
|
+
total_duration_ms: totalDuration,
|
|
300
|
+
},
|
|
301
|
+
});
|
|
302
|
+
}
|
|
244
303
|
return { query_id, summary, response_headers };
|
|
245
304
|
}
|
|
246
305
|
async close() {
|
|
@@ -303,37 +362,31 @@ class NodeBaseConnection {
|
|
|
303
362
|
},
|
|
304
363
|
};
|
|
305
364
|
}
|
|
306
|
-
logResponse(op, request, params, response, startTimestamp) {
|
|
307
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
308
|
-
const { authorization, host, ...headers } = request.getHeaders();
|
|
309
|
-
const duration = Date.now() - startTimestamp;
|
|
310
|
-
this.params.log_writer.debug({
|
|
311
|
-
module: 'HTTP Adapter',
|
|
312
|
-
message: `${op}: got a response from ClickHouse`,
|
|
313
|
-
args: {
|
|
314
|
-
request_method: params.method,
|
|
315
|
-
request_path: params.url.pathname,
|
|
316
|
-
request_params: params.url.search,
|
|
317
|
-
request_headers: headers,
|
|
318
|
-
response_status: response.statusCode,
|
|
319
|
-
response_headers: response.headers,
|
|
320
|
-
response_time_ms: duration,
|
|
321
|
-
},
|
|
322
|
-
});
|
|
323
|
-
}
|
|
324
365
|
logRequestError({ op, err, query_id, query_params, search_params, extra_args, }) {
|
|
325
|
-
this.
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
search_params
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
366
|
+
if (this.params.log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
367
|
+
// Redact query parameter from search params unless explicitly allowed
|
|
368
|
+
if (!this.params.unsafeLogUnredactedQueries && search_params) {
|
|
369
|
+
// Clone to avoid mutating the original search params
|
|
370
|
+
search_params = new URLSearchParams(search_params);
|
|
371
|
+
search_params.delete('query');
|
|
372
|
+
}
|
|
373
|
+
this.params.log_writer.error({
|
|
374
|
+
message: this.httpRequestErrorMessage(op),
|
|
375
|
+
err: err,
|
|
376
|
+
args: {
|
|
377
|
+
operation: op,
|
|
378
|
+
connection_id: this.connectionId,
|
|
379
|
+
query_id,
|
|
380
|
+
query: this.params.unsafeLogUnredactedQueries
|
|
381
|
+
? query_params.query
|
|
382
|
+
: undefined,
|
|
383
|
+
search_params: search_params?.toString(),
|
|
384
|
+
with_abort_signal: query_params.abort_signal !== undefined,
|
|
385
|
+
session_id: query_params.session_id,
|
|
386
|
+
...extra_args,
|
|
387
|
+
},
|
|
388
|
+
});
|
|
389
|
+
}
|
|
337
390
|
}
|
|
338
391
|
httpRequestErrorMessage(op) {
|
|
339
392
|
return `${op}: HTTP request error.`;
|
|
@@ -345,18 +398,23 @@ class NodeBaseConnection {
|
|
|
345
398
|
return this.jsonHandling.parse(summaryHeader);
|
|
346
399
|
}
|
|
347
400
|
catch (err) {
|
|
348
|
-
this.
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
401
|
+
if (this.params.log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
402
|
+
this.params.log_writer.error({
|
|
403
|
+
message: `${op}: failed to parse X-ClickHouse-Summary header.`,
|
|
404
|
+
args: {
|
|
405
|
+
operation: op,
|
|
406
|
+
connection_id: this.connectionId,
|
|
407
|
+
'X-ClickHouse-Summary': summaryHeader,
|
|
408
|
+
},
|
|
409
|
+
err: err,
|
|
410
|
+
});
|
|
411
|
+
}
|
|
355
412
|
}
|
|
356
413
|
}
|
|
357
414
|
}
|
|
358
415
|
async runExec(params) {
|
|
359
|
-
const
|
|
416
|
+
const { log_writer, log_level } = this.params;
|
|
417
|
+
const query_id = params.query_id;
|
|
360
418
|
const sendQueryInParams = params.values !== undefined;
|
|
361
419
|
const clickhouse_settings = (0, client_common_1.withHttpSettings)(params.clickhouse_settings, this.params.compression.decompress_response);
|
|
362
420
|
const toSearchParamsOptions = {
|
|
@@ -391,6 +449,9 @@ class NodeBaseConnection {
|
|
|
391
449
|
ignore_error_response: ignoreErrorResponse,
|
|
392
450
|
headers: this.buildRequestHeaders(params),
|
|
393
451
|
query: params.query,
|
|
452
|
+
query_id,
|
|
453
|
+
log_writer,
|
|
454
|
+
log_level,
|
|
394
455
|
}, params.op);
|
|
395
456
|
return {
|
|
396
457
|
stream,
|
|
@@ -422,13 +483,15 @@ class NodeBaseConnection {
|
|
|
422
483
|
// allows the event loop to process the idle socket timers, if the CPU load is high
|
|
423
484
|
// otherwise, we can occasionally get an expired socket, see https://github.com/ClickHouse/clickhouse-js/issues/294
|
|
424
485
|
await (0, client_common_1.sleep)(0);
|
|
486
|
+
const { log_writer, query_id, log_level } = params;
|
|
425
487
|
const currentStackTrace = this.params.capture_enhanced_stack_trace
|
|
426
488
|
? (0, client_common_1.getCurrentStackTrace)()
|
|
427
489
|
: undefined;
|
|
428
|
-
const
|
|
490
|
+
const requestTimeout = this.params.request_timeout;
|
|
429
491
|
return new Promise((resolve, reject) => {
|
|
430
492
|
const start = Date.now();
|
|
431
493
|
const request = this.createClientRequest(params);
|
|
494
|
+
const request_id = this.getNewRequestId();
|
|
432
495
|
function onError(e) {
|
|
433
496
|
removeRequestListeners();
|
|
434
497
|
const err = (0, client_common_1.enhanceStackTrace)(e, currentStackTrace);
|
|
@@ -436,14 +499,42 @@ class NodeBaseConnection {
|
|
|
436
499
|
}
|
|
437
500
|
let responseStream;
|
|
438
501
|
const onResponse = async (_response) => {
|
|
439
|
-
this.
|
|
502
|
+
if (this.params.log_level <= client_common_1.ClickHouseLogLevel.DEBUG) {
|
|
503
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
504
|
+
const { authorization, host, ...headers } = request.getHeaders();
|
|
505
|
+
const duration = Date.now() - start;
|
|
506
|
+
// Redact query parameter from URL search params unless explicitly allowed
|
|
507
|
+
let searchParams = params.url.searchParams;
|
|
508
|
+
if (!this.params.unsafeLogUnredactedQueries) {
|
|
509
|
+
// Clone to avoid mutating the original search params
|
|
510
|
+
searchParams = new URLSearchParams(searchParams);
|
|
511
|
+
searchParams.delete('query');
|
|
512
|
+
}
|
|
513
|
+
this.params.log_writer.debug({
|
|
514
|
+
module: 'HTTP Adapter',
|
|
515
|
+
message: `${op}: got a response from ClickHouse`,
|
|
516
|
+
args: {
|
|
517
|
+
operation: op,
|
|
518
|
+
connection_id: this.connectionId,
|
|
519
|
+
query_id,
|
|
520
|
+
request_id,
|
|
521
|
+
request_method: params.method,
|
|
522
|
+
request_path: params.url.pathname,
|
|
523
|
+
request_params: searchParams.toString(),
|
|
524
|
+
request_headers: headers,
|
|
525
|
+
response_status: _response.statusCode,
|
|
526
|
+
response_headers: _response.headers,
|
|
527
|
+
response_time_ms: duration,
|
|
528
|
+
},
|
|
529
|
+
});
|
|
530
|
+
}
|
|
440
531
|
const tryDecompressResponseStream = params.try_decompress_response_stream ?? true;
|
|
441
532
|
const ignoreErrorResponse = params.ignore_error_response ?? false;
|
|
442
533
|
// even if the stream decompression is disabled, we have to decompress it in case of an error
|
|
443
534
|
const isFailedResponse = !(0, client_common_1.isSuccessfulResponse)(_response.statusCode);
|
|
444
535
|
if (tryDecompressResponseStream ||
|
|
445
536
|
(isFailedResponse && !ignoreErrorResponse)) {
|
|
446
|
-
const decompressionResult = (0, compression_1.decompressResponse)(_response,
|
|
537
|
+
const decompressionResult = (0, compression_1.decompressResponse)(_response, log_writer, log_level);
|
|
447
538
|
if ((0, compression_1.isDecompressionError)(decompressionResult)) {
|
|
448
539
|
const err = (0, client_common_1.enhanceStackTrace)(decompressionResult.error, currentStackTrace);
|
|
449
540
|
return reject(err);
|
|
@@ -453,6 +544,24 @@ class NodeBaseConnection {
|
|
|
453
544
|
else {
|
|
454
545
|
responseStream = _response;
|
|
455
546
|
}
|
|
547
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
548
|
+
log_writer.trace({
|
|
549
|
+
message: `${op}: response stream created`,
|
|
550
|
+
args: {
|
|
551
|
+
operation: op,
|
|
552
|
+
connection_id: this.connectionId,
|
|
553
|
+
query_id,
|
|
554
|
+
request_id,
|
|
555
|
+
stream_state: {
|
|
556
|
+
readable: responseStream.readable,
|
|
557
|
+
readableEnded: responseStream.readableEnded,
|
|
558
|
+
readableLength: responseStream.readableLength,
|
|
559
|
+
},
|
|
560
|
+
is_failed_response: isFailedResponse,
|
|
561
|
+
will_decompress: tryDecompressResponseStream,
|
|
562
|
+
},
|
|
563
|
+
});
|
|
564
|
+
}
|
|
456
565
|
if (isFailedResponse && !ignoreErrorResponse) {
|
|
457
566
|
try {
|
|
458
567
|
const errorMessage = await (0, utils_1.getAsText)(responseStream);
|
|
@@ -525,96 +634,229 @@ class NodeBaseConnection {
|
|
|
525
634
|
// It is the first time we've encountered this socket,
|
|
526
635
|
// so it doesn't have the idle timeout handler attached to it
|
|
527
636
|
if (socketInfo === undefined) {
|
|
528
|
-
const
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
637
|
+
const socket_id = this.getNewSocketId();
|
|
638
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
639
|
+
log_writer.trace({
|
|
640
|
+
message: `${op}: using a fresh socket, setting up a new 'free' listener`,
|
|
641
|
+
args: {
|
|
642
|
+
operation: op,
|
|
643
|
+
connection_id: this.connectionId,
|
|
644
|
+
query_id,
|
|
645
|
+
request_id,
|
|
646
|
+
socket_id,
|
|
647
|
+
},
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
const newSocketInfo = {
|
|
651
|
+
id: socket_id,
|
|
534
652
|
idle_timeout_handle: undefined,
|
|
535
|
-
|
|
653
|
+
usage_count: 1,
|
|
654
|
+
};
|
|
655
|
+
this.knownSockets.set(socket, newSocketInfo);
|
|
536
656
|
// When the request is complete and the socket is released,
|
|
537
657
|
// make sure that the socket is removed after `idleSocketTTL`.
|
|
538
658
|
socket.on('free', () => {
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
659
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
660
|
+
log_writer.trace({
|
|
661
|
+
message: `${op}: socket was released`,
|
|
662
|
+
args: {
|
|
663
|
+
operation: op,
|
|
664
|
+
connection_id: this.connectionId,
|
|
665
|
+
query_id,
|
|
666
|
+
request_id,
|
|
667
|
+
socket_id,
|
|
668
|
+
},
|
|
669
|
+
});
|
|
670
|
+
}
|
|
542
671
|
// Avoiding the built-in socket.timeout() method usage here,
|
|
543
672
|
// as we don't want to clash with the actual request timeout.
|
|
544
673
|
const idleTimeoutHandle = setTimeout(() => {
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
674
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
675
|
+
log_writer.trace({
|
|
676
|
+
message: `${op}: removing idle socket`,
|
|
677
|
+
args: {
|
|
678
|
+
operation: op,
|
|
679
|
+
connection_id: this.connectionId,
|
|
680
|
+
query_id,
|
|
681
|
+
request_id,
|
|
682
|
+
socket_id,
|
|
683
|
+
idle_socket_ttl_ms: this.idleSocketTTL,
|
|
684
|
+
},
|
|
685
|
+
});
|
|
686
|
+
}
|
|
548
687
|
this.knownSockets.delete(socket);
|
|
549
688
|
socket.destroy();
|
|
550
689
|
}, this.idleSocketTTL).unref();
|
|
551
|
-
|
|
552
|
-
id: socketId,
|
|
553
|
-
idle_timeout_handle: idleTimeoutHandle,
|
|
554
|
-
});
|
|
690
|
+
newSocketInfo.idle_timeout_handle = idleTimeoutHandle;
|
|
555
691
|
});
|
|
556
|
-
const cleanup = () => {
|
|
692
|
+
const cleanup = (eventName) => () => {
|
|
557
693
|
const maybeSocketInfo = this.knownSockets.get(socket);
|
|
558
694
|
// clean up a possibly dangling idle timeout handle (preventing leaks)
|
|
559
695
|
if (maybeSocketInfo?.idle_timeout_handle) {
|
|
560
696
|
clearTimeout(maybeSocketInfo.idle_timeout_handle);
|
|
561
697
|
}
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
if (responseStream && !responseStream.readableEnded) {
|
|
566
|
-
this.logger.warn({
|
|
567
|
-
message: `${op}: socket was closed or ended before the response was fully read. ` +
|
|
568
|
-
'This can potentially result in an uncaught ECONNRESET error! ' +
|
|
569
|
-
'Consider fully consuming, draining, or destroying the response stream.',
|
|
698
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
699
|
+
log_writer.trace({
|
|
700
|
+
message: `${op}: received '${eventName}' event, 'free' listener removed`,
|
|
570
701
|
args: {
|
|
571
|
-
|
|
572
|
-
|
|
702
|
+
operation: op,
|
|
703
|
+
connection_id: this.connectionId,
|
|
704
|
+
query_id,
|
|
705
|
+
request_id,
|
|
706
|
+
socket_id,
|
|
707
|
+
event: eventName,
|
|
573
708
|
},
|
|
574
709
|
});
|
|
575
710
|
}
|
|
711
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.WARN) {
|
|
712
|
+
if (responseStream && !responseStream.readableEnded) {
|
|
713
|
+
log_writer.warn({
|
|
714
|
+
message: `${op}: socket was closed or ended before the response was fully read. ` +
|
|
715
|
+
'This can potentially result in an uncaught ECONNRESET error! ' +
|
|
716
|
+
'Consider fully consuming, draining, or destroying the response stream.',
|
|
717
|
+
args: {
|
|
718
|
+
operation: op,
|
|
719
|
+
connection_id: this.connectionId,
|
|
720
|
+
query_id,
|
|
721
|
+
request_id,
|
|
722
|
+
socket_id,
|
|
723
|
+
event: eventName,
|
|
724
|
+
query: this.params.unsafeLogUnredactedQueries
|
|
725
|
+
? params.query
|
|
726
|
+
: undefined,
|
|
727
|
+
},
|
|
728
|
+
});
|
|
729
|
+
}
|
|
730
|
+
}
|
|
576
731
|
};
|
|
577
|
-
socket.once('end', cleanup);
|
|
578
|
-
socket.once('close', cleanup);
|
|
732
|
+
socket.once('end', cleanup('end'));
|
|
733
|
+
socket.once('close', cleanup('close'));
|
|
579
734
|
}
|
|
580
735
|
else {
|
|
581
736
|
clearTimeout(socketInfo.idle_timeout_handle);
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
737
|
+
socketInfo.idle_timeout_handle = undefined;
|
|
738
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
739
|
+
log_writer.trace({
|
|
740
|
+
message: `${op}: reusing socket`,
|
|
741
|
+
args: {
|
|
742
|
+
operation: op,
|
|
743
|
+
connection_id: this.connectionId,
|
|
744
|
+
query_id,
|
|
745
|
+
request_id,
|
|
746
|
+
socket_id: socketInfo.id,
|
|
747
|
+
usage_count: socketInfo.usage_count,
|
|
748
|
+
},
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
socketInfo.usage_count++;
|
|
589
752
|
}
|
|
590
753
|
}
|
|
591
754
|
}
|
|
592
755
|
catch (e) {
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
756
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
757
|
+
log_writer.error({
|
|
758
|
+
message: `${op}: an error occurred while housekeeping the idle sockets`,
|
|
759
|
+
err: e,
|
|
760
|
+
args: {
|
|
761
|
+
operation: op,
|
|
762
|
+
connection_id: this.connectionId,
|
|
763
|
+
query_id,
|
|
764
|
+
request_id,
|
|
765
|
+
},
|
|
766
|
+
});
|
|
767
|
+
}
|
|
597
768
|
}
|
|
598
769
|
// Socket is "prepared" with idle handlers, continue with our request
|
|
599
770
|
pipeStream();
|
|
600
771
|
// This is for request timeout only. Surprisingly, it is not always enough to set in the HTTP request.
|
|
601
772
|
// The socket won't be destroyed, and it will be returned to the pool.
|
|
773
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
774
|
+
const socketInfo = this.knownSockets.get(socket);
|
|
775
|
+
if (socketInfo) {
|
|
776
|
+
log_writer.trace({
|
|
777
|
+
message: `${op}: setting up request timeout`,
|
|
778
|
+
args: {
|
|
779
|
+
operation: op,
|
|
780
|
+
connection_id: this.connectionId,
|
|
781
|
+
query_id,
|
|
782
|
+
request_id,
|
|
783
|
+
socket_id: socketInfo.id,
|
|
784
|
+
timeout_ms: requestTimeout,
|
|
785
|
+
},
|
|
786
|
+
});
|
|
787
|
+
}
|
|
788
|
+
else {
|
|
789
|
+
log_writer.trace({
|
|
790
|
+
message: `${op}: setting up request timeout on a socket`,
|
|
791
|
+
args: {
|
|
792
|
+
operation: op,
|
|
793
|
+
connection_id: this.connectionId,
|
|
794
|
+
query_id,
|
|
795
|
+
request_id,
|
|
796
|
+
timeout_ms: requestTimeout,
|
|
797
|
+
},
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
}
|
|
602
801
|
socket.setTimeout(this.params.request_timeout, onTimeout);
|
|
603
802
|
};
|
|
604
|
-
|
|
605
|
-
const err = (0, client_common_1.enhanceStackTrace)(new Error('Timeout error.'), currentStackTrace);
|
|
803
|
+
const onTimeout = () => {
|
|
606
804
|
removeRequestListeners();
|
|
805
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
806
|
+
const socket = request.socket;
|
|
807
|
+
const maybeSocketInfo = socket
|
|
808
|
+
? this.knownSockets.get(socket)
|
|
809
|
+
: undefined;
|
|
810
|
+
const socketState = request.socket
|
|
811
|
+
? {
|
|
812
|
+
connecting: request.socket.connecting,
|
|
813
|
+
pending: request.socket.pending,
|
|
814
|
+
destroyed: request.socket.destroyed,
|
|
815
|
+
readyState: request.socket.readyState,
|
|
816
|
+
}
|
|
817
|
+
: undefined;
|
|
818
|
+
const responseStreamState = responseStream
|
|
819
|
+
? {
|
|
820
|
+
readable: responseStream.readable,
|
|
821
|
+
readableEnded: responseStream.readableEnded,
|
|
822
|
+
readableLength: responseStream.readableLength,
|
|
823
|
+
}
|
|
824
|
+
: undefined;
|
|
825
|
+
log_writer.trace({
|
|
826
|
+
message: `${op}: timeout occurred`,
|
|
827
|
+
args: {
|
|
828
|
+
operation: op,
|
|
829
|
+
connection_id: this.connectionId,
|
|
830
|
+
query_id,
|
|
831
|
+
request_id,
|
|
832
|
+
socket_id: maybeSocketInfo?.id,
|
|
833
|
+
timeout_ms: requestTimeout,
|
|
834
|
+
socket_state: socketState,
|
|
835
|
+
response_stream_state: responseStreamState,
|
|
836
|
+
has_response_stream: responseStream !== undefined,
|
|
837
|
+
},
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
const err = (0, client_common_1.enhanceStackTrace)(new Error('Timeout error.'), currentStackTrace);
|
|
607
841
|
try {
|
|
608
842
|
request.destroy();
|
|
609
843
|
}
|
|
610
844
|
catch (e) {
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
845
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
846
|
+
log_writer.error({
|
|
847
|
+
message: `${op}: An error occurred while destroying the request`,
|
|
848
|
+
err: e,
|
|
849
|
+
args: {
|
|
850
|
+
operation: op,
|
|
851
|
+
connection_id: this.connectionId,
|
|
852
|
+
query_id,
|
|
853
|
+
request_id,
|
|
854
|
+
},
|
|
855
|
+
});
|
|
856
|
+
}
|
|
615
857
|
}
|
|
616
858
|
reject(err);
|
|
617
|
-
}
|
|
859
|
+
};
|
|
618
860
|
function removeRequestListeners() {
|
|
619
861
|
if (request.socket !== null) {
|
|
620
862
|
request.socket.setTimeout(0); // reset previously set timeout
|
|
@@ -642,10 +884,18 @@ class NodeBaseConnection {
|
|
|
642
884
|
return request.end();
|
|
643
885
|
}
|
|
644
886
|
catch (e) {
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
887
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
888
|
+
log_writer.error({
|
|
889
|
+
message: `${op}: an error occurred while ending the request without body`,
|
|
890
|
+
err: e,
|
|
891
|
+
args: {
|
|
892
|
+
operation: op,
|
|
893
|
+
connection_id: this.connectionId,
|
|
894
|
+
query_id,
|
|
895
|
+
request_id,
|
|
896
|
+
},
|
|
897
|
+
});
|
|
898
|
+
}
|
|
649
899
|
}
|
|
650
900
|
}
|
|
651
901
|
});
|