@clickhouse/client 1.16.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connection/compression.d.ts +2 -1
- package/dist/connection/compression.js +8 -5
- package/dist/connection/compression.js.map +1 -1
- package/dist/connection/node_base_connection.d.ts +11 -3
- package/dist/connection/node_base_connection.js +410 -157
- package/dist/connection/node_base_connection.js.map +1 -1
- package/dist/connection/node_custom_agent_connection.js +1 -6
- package/dist/connection/node_custom_agent_connection.js.map +1 -1
- package/dist/connection/stream.d.ts +9 -1
- package/dist/connection/stream.js +71 -3
- package/dist/connection/stream.js.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/result_set.js +34 -59
- package/dist/result_set.js.map +1 -1
- package/dist/utils/encoder.js +1 -6
- package/dist/utils/encoder.js.map +1 -1
- package/dist/utils/runtime.js +3 -18
- package/dist/utils/runtime.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +5 -2
|
@@ -12,55 +12,31 @@ const utils_1 = require("../utils");
|
|
|
12
12
|
const compression_1 = require("./compression");
|
|
13
13
|
const stream_2 = require("./stream");
|
|
14
14
|
class NodeBaseConnection {
|
|
15
|
+
params;
|
|
16
|
+
agent;
|
|
17
|
+
defaultAuthHeader;
|
|
18
|
+
defaultHeaders;
|
|
19
|
+
jsonHandling;
|
|
20
|
+
knownSockets = new WeakMap();
|
|
21
|
+
idleSocketTTL;
|
|
22
|
+
connectionId = crypto_1.default.randomUUID();
|
|
23
|
+
socketCounter = 0;
|
|
24
|
+
// For overflow concerns:
|
|
25
|
+
// node -e 'console.log(Number.MAX_SAFE_INTEGER / (1_000_000 * 60 * 60 * 24 * 366))'
|
|
26
|
+
// gives 284 years of continuous operation at 1M requests per second
|
|
27
|
+
// before overflowing the 53-bit integer
|
|
28
|
+
requestCounter = 0;
|
|
29
|
+
getNewSocketId() {
|
|
30
|
+
this.socketCounter += 1;
|
|
31
|
+
return `${this.connectionId}:${this.socketCounter}`;
|
|
32
|
+
}
|
|
33
|
+
getNewRequestId() {
|
|
34
|
+
this.requestCounter += 1;
|
|
35
|
+
return `${this.connectionId}:${this.requestCounter}`;
|
|
36
|
+
}
|
|
15
37
|
constructor(params, agent) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
configurable: true,
|
|
19
|
-
writable: true,
|
|
20
|
-
value: params
|
|
21
|
-
});
|
|
22
|
-
Object.defineProperty(this, "agent", {
|
|
23
|
-
enumerable: true,
|
|
24
|
-
configurable: true,
|
|
25
|
-
writable: true,
|
|
26
|
-
value: agent
|
|
27
|
-
});
|
|
28
|
-
Object.defineProperty(this, "defaultAuthHeader", {
|
|
29
|
-
enumerable: true,
|
|
30
|
-
configurable: true,
|
|
31
|
-
writable: true,
|
|
32
|
-
value: void 0
|
|
33
|
-
});
|
|
34
|
-
Object.defineProperty(this, "defaultHeaders", {
|
|
35
|
-
enumerable: true,
|
|
36
|
-
configurable: true,
|
|
37
|
-
writable: true,
|
|
38
|
-
value: void 0
|
|
39
|
-
});
|
|
40
|
-
Object.defineProperty(this, "jsonHandling", {
|
|
41
|
-
enumerable: true,
|
|
42
|
-
configurable: true,
|
|
43
|
-
writable: true,
|
|
44
|
-
value: void 0
|
|
45
|
-
});
|
|
46
|
-
Object.defineProperty(this, "logger", {
|
|
47
|
-
enumerable: true,
|
|
48
|
-
configurable: true,
|
|
49
|
-
writable: true,
|
|
50
|
-
value: void 0
|
|
51
|
-
});
|
|
52
|
-
Object.defineProperty(this, "knownSockets", {
|
|
53
|
-
enumerable: true,
|
|
54
|
-
configurable: true,
|
|
55
|
-
writable: true,
|
|
56
|
-
value: new WeakMap()
|
|
57
|
-
});
|
|
58
|
-
Object.defineProperty(this, "idleSocketTTL", {
|
|
59
|
-
enumerable: true,
|
|
60
|
-
configurable: true,
|
|
61
|
-
writable: true,
|
|
62
|
-
value: void 0
|
|
63
|
-
});
|
|
38
|
+
this.params = params;
|
|
39
|
+
this.agent = agent;
|
|
64
40
|
if (params.auth.type === 'Credentials') {
|
|
65
41
|
this.defaultAuthHeader = `Basic ${Buffer.from(`${params.auth.username}:${params.auth.password}`).toString('base64')}`;
|
|
66
42
|
}
|
|
@@ -75,7 +51,6 @@ class NodeBaseConnection {
|
|
|
75
51
|
Connection: this.params.keep_alive.enabled ? 'keep-alive' : 'close',
|
|
76
52
|
'User-Agent': (0, utils_1.getUserAgent)(this.params.application_id),
|
|
77
53
|
};
|
|
78
|
-
this.logger = params.log_writer;
|
|
79
54
|
this.idleSocketTTL = params.keep_alive.idle_socket_ttl;
|
|
80
55
|
this.jsonHandling = params.json ?? {
|
|
81
56
|
parse: JSON.parse,
|
|
@@ -83,6 +58,7 @@ class NodeBaseConnection {
|
|
|
83
58
|
};
|
|
84
59
|
}
|
|
85
60
|
async ping(params) {
|
|
61
|
+
const { log_writer, log_level } = this.params;
|
|
86
62
|
const query_id = this.getQueryId(params.query_id);
|
|
87
63
|
const { controller, controllerCleanup } = this.getAbortController(params);
|
|
88
64
|
let result;
|
|
@@ -99,6 +75,9 @@ class NodeBaseConnection {
|
|
|
99
75
|
query: PingQuery,
|
|
100
76
|
abort_signal: controller.signal,
|
|
101
77
|
headers: this.buildRequestHeaders(),
|
|
78
|
+
query_id,
|
|
79
|
+
log_writer,
|
|
80
|
+
log_level,
|
|
102
81
|
}, 'Ping');
|
|
103
82
|
}
|
|
104
83
|
else {
|
|
@@ -108,9 +87,17 @@ class NodeBaseConnection {
|
|
|
108
87
|
abort_signal: controller.signal,
|
|
109
88
|
headers: this.buildRequestHeaders(),
|
|
110
89
|
query: 'ping',
|
|
90
|
+
query_id,
|
|
91
|
+
log_writer,
|
|
92
|
+
log_level,
|
|
111
93
|
}, 'Ping');
|
|
112
94
|
}
|
|
113
|
-
await (0, stream_2.drainStream)(
|
|
95
|
+
await (0, stream_2.drainStream)({
|
|
96
|
+
op: 'Ping',
|
|
97
|
+
log_writer,
|
|
98
|
+
query_id,
|
|
99
|
+
log_level,
|
|
100
|
+
}, result.stream);
|
|
114
101
|
return { success: true };
|
|
115
102
|
}
|
|
116
103
|
catch (error) {
|
|
@@ -118,13 +105,16 @@ class NodeBaseConnection {
|
|
|
118
105
|
// and we don't get unhandled error propagation later
|
|
119
106
|
controller.abort('Ping failed');
|
|
120
107
|
// not an error, as this might be semi-expected
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
108
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.WARN) {
|
|
109
|
+
log_writer.warn({
|
|
110
|
+
message: this.httpRequestErrorMessage('Ping'),
|
|
111
|
+
err: error,
|
|
112
|
+
args: {
|
|
113
|
+
connection_id: this.connectionId,
|
|
114
|
+
query_id,
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
}
|
|
128
118
|
return {
|
|
129
119
|
success: false,
|
|
130
120
|
error: error, // should NOT be propagated to the user
|
|
@@ -135,6 +125,7 @@ class NodeBaseConnection {
|
|
|
135
125
|
}
|
|
136
126
|
}
|
|
137
127
|
async query(params) {
|
|
128
|
+
const { log_writer, log_level } = this.params;
|
|
138
129
|
const query_id = this.getQueryId(params.query_id);
|
|
139
130
|
const clickhouse_settings = (0, client_common_1.withHttpSettings)(params.clickhouse_settings, this.params.compression.decompress_response);
|
|
140
131
|
const searchParams = (0, client_common_1.toSearchParams)({
|
|
@@ -149,7 +140,7 @@ class NodeBaseConnection {
|
|
|
149
140
|
// allows enforcing the compression via the settings even if the client instance has it disabled
|
|
150
141
|
const enableResponseCompression = clickhouse_settings.enable_http_compression === 1;
|
|
151
142
|
try {
|
|
152
|
-
const { response_headers, stream } = await this.request({
|
|
143
|
+
const { response_headers, stream, http_status_code } = await this.request({
|
|
153
144
|
method: 'POST',
|
|
154
145
|
url: (0, client_common_1.transformUrl)({ url: this.params.url, searchParams }),
|
|
155
146
|
body: params.query,
|
|
@@ -157,11 +148,15 @@ class NodeBaseConnection {
|
|
|
157
148
|
enable_response_compression: enableResponseCompression,
|
|
158
149
|
headers: this.buildRequestHeaders(params),
|
|
159
150
|
query: params.query,
|
|
151
|
+
query_id,
|
|
152
|
+
log_writer,
|
|
153
|
+
log_level,
|
|
160
154
|
}, 'Query');
|
|
161
155
|
return {
|
|
162
156
|
stream,
|
|
163
157
|
response_headers,
|
|
164
158
|
query_id,
|
|
159
|
+
http_status_code,
|
|
165
160
|
};
|
|
166
161
|
}
|
|
167
162
|
catch (err) {
|
|
@@ -184,6 +179,7 @@ class NodeBaseConnection {
|
|
|
184
179
|
}
|
|
185
180
|
}
|
|
186
181
|
async insert(params) {
|
|
182
|
+
const { log_writer, log_level } = this.params;
|
|
187
183
|
const query_id = this.getQueryId(params.query_id);
|
|
188
184
|
const searchParams = (0, client_common_1.toSearchParams)({
|
|
189
185
|
database: this.params.database,
|
|
@@ -196,7 +192,7 @@ class NodeBaseConnection {
|
|
|
196
192
|
});
|
|
197
193
|
const { controller, controllerCleanup } = this.getAbortController(params);
|
|
198
194
|
try {
|
|
199
|
-
const { stream, summary, response_headers } = await this.request({
|
|
195
|
+
const { stream, summary, response_headers, http_status_code } = await this.request({
|
|
200
196
|
method: 'POST',
|
|
201
197
|
url: (0, client_common_1.transformUrl)({ url: this.params.url, searchParams }),
|
|
202
198
|
body: params.values,
|
|
@@ -205,9 +201,17 @@ class NodeBaseConnection {
|
|
|
205
201
|
parse_summary: true,
|
|
206
202
|
headers: this.buildRequestHeaders(params),
|
|
207
203
|
query: params.query,
|
|
204
|
+
query_id,
|
|
205
|
+
log_writer,
|
|
206
|
+
log_level,
|
|
208
207
|
}, 'Insert');
|
|
209
|
-
await (0, stream_2.drainStream)(
|
|
210
|
-
|
|
208
|
+
await (0, stream_2.drainStream)({
|
|
209
|
+
op: 'Insert',
|
|
210
|
+
log_writer,
|
|
211
|
+
query_id,
|
|
212
|
+
log_level,
|
|
213
|
+
}, stream);
|
|
214
|
+
return { query_id, summary, response_headers, http_status_code };
|
|
211
215
|
}
|
|
212
216
|
catch (err) {
|
|
213
217
|
controller.abort('Insert HTTP request failed');
|
|
@@ -228,18 +232,74 @@ class NodeBaseConnection {
|
|
|
228
232
|
}
|
|
229
233
|
}
|
|
230
234
|
async exec(params) {
|
|
235
|
+
const query_id = this.getQueryId(params.query_id);
|
|
231
236
|
return this.runExec({
|
|
232
237
|
...params,
|
|
238
|
+
query_id,
|
|
233
239
|
op: 'Exec',
|
|
234
240
|
});
|
|
235
241
|
}
|
|
236
242
|
async command(params) {
|
|
237
|
-
const {
|
|
243
|
+
const { log_writer, log_level } = this.params;
|
|
244
|
+
const query_id = this.getQueryId(params.query_id);
|
|
245
|
+
const commandStartTime = Date.now();
|
|
246
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
247
|
+
log_writer.trace({
|
|
248
|
+
message: 'Command: operation started',
|
|
249
|
+
args: {
|
|
250
|
+
operation: 'Command',
|
|
251
|
+
connection_id: this.connectionId,
|
|
252
|
+
query_id,
|
|
253
|
+
query: this.params.unsafeLogUnredactedQueries
|
|
254
|
+
? params.query
|
|
255
|
+
: undefined,
|
|
256
|
+
},
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
const { stream, summary, response_headers } = await this.runExec({
|
|
238
260
|
...params,
|
|
261
|
+
query_id,
|
|
239
262
|
op: 'Command',
|
|
240
263
|
});
|
|
264
|
+
const runExecDuration = Date.now() - commandStartTime;
|
|
265
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
266
|
+
log_writer.trace({
|
|
267
|
+
message: 'Command: runExec completed, starting stream drain',
|
|
268
|
+
args: {
|
|
269
|
+
operation: 'Command',
|
|
270
|
+
connection_id: this.connectionId,
|
|
271
|
+
query_id,
|
|
272
|
+
runExec_duration_ms: runExecDuration,
|
|
273
|
+
stream_state: {
|
|
274
|
+
readable: stream.readable,
|
|
275
|
+
readableEnded: stream.readableEnded,
|
|
276
|
+
readableLength: stream.readableLength,
|
|
277
|
+
},
|
|
278
|
+
},
|
|
279
|
+
});
|
|
280
|
+
}
|
|
241
281
|
// ignore the response stream and release the socket immediately
|
|
242
|
-
|
|
282
|
+
const drainStartTime = Date.now();
|
|
283
|
+
await (0, stream_2.drainStream)({
|
|
284
|
+
op: 'Command',
|
|
285
|
+
log_writer,
|
|
286
|
+
query_id,
|
|
287
|
+
log_level,
|
|
288
|
+
}, stream);
|
|
289
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
290
|
+
const drainDuration = Date.now() - drainStartTime;
|
|
291
|
+
const totalDuration = Date.now() - commandStartTime;
|
|
292
|
+
log_writer.trace({
|
|
293
|
+
message: 'Command: operation completed',
|
|
294
|
+
args: {
|
|
295
|
+
operation: 'Command',
|
|
296
|
+
connection_id: this.connectionId,
|
|
297
|
+
query_id,
|
|
298
|
+
drain_duration_ms: drainDuration,
|
|
299
|
+
total_duration_ms: totalDuration,
|
|
300
|
+
},
|
|
301
|
+
});
|
|
302
|
+
}
|
|
243
303
|
return { query_id, summary, response_headers };
|
|
244
304
|
}
|
|
245
305
|
async close() {
|
|
@@ -302,37 +362,31 @@ class NodeBaseConnection {
|
|
|
302
362
|
},
|
|
303
363
|
};
|
|
304
364
|
}
|
|
305
|
-
logResponse(op, request, params, response, startTimestamp) {
|
|
306
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
307
|
-
const { authorization, host, ...headers } = request.getHeaders();
|
|
308
|
-
const duration = Date.now() - startTimestamp;
|
|
309
|
-
this.params.log_writer.debug({
|
|
310
|
-
module: 'HTTP Adapter',
|
|
311
|
-
message: `${op}: got a response from ClickHouse`,
|
|
312
|
-
args: {
|
|
313
|
-
request_method: params.method,
|
|
314
|
-
request_path: params.url.pathname,
|
|
315
|
-
request_params: params.url.search,
|
|
316
|
-
request_headers: headers,
|
|
317
|
-
response_status: response.statusCode,
|
|
318
|
-
response_headers: response.headers,
|
|
319
|
-
response_time_ms: duration,
|
|
320
|
-
},
|
|
321
|
-
});
|
|
322
|
-
}
|
|
323
365
|
logRequestError({ op, err, query_id, query_params, search_params, extra_args, }) {
|
|
324
|
-
this.
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
search_params
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
366
|
+
if (this.params.log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
367
|
+
// Redact query parameter from search params unless explicitly allowed
|
|
368
|
+
if (!this.params.unsafeLogUnredactedQueries && search_params) {
|
|
369
|
+
// Clone to avoid mutating the original search params
|
|
370
|
+
search_params = new URLSearchParams(search_params);
|
|
371
|
+
search_params.delete('query');
|
|
372
|
+
}
|
|
373
|
+
this.params.log_writer.error({
|
|
374
|
+
message: this.httpRequestErrorMessage(op),
|
|
375
|
+
err: err,
|
|
376
|
+
args: {
|
|
377
|
+
operation: op,
|
|
378
|
+
connection_id: this.connectionId,
|
|
379
|
+
query_id,
|
|
380
|
+
query: this.params.unsafeLogUnredactedQueries
|
|
381
|
+
? query_params.query
|
|
382
|
+
: undefined,
|
|
383
|
+
search_params: search_params?.toString(),
|
|
384
|
+
with_abort_signal: query_params.abort_signal !== undefined,
|
|
385
|
+
session_id: query_params.session_id,
|
|
386
|
+
...extra_args,
|
|
387
|
+
},
|
|
388
|
+
});
|
|
389
|
+
}
|
|
336
390
|
}
|
|
337
391
|
httpRequestErrorMessage(op) {
|
|
338
392
|
return `${op}: HTTP request error.`;
|
|
@@ -344,18 +398,23 @@ class NodeBaseConnection {
|
|
|
344
398
|
return this.jsonHandling.parse(summaryHeader);
|
|
345
399
|
}
|
|
346
400
|
catch (err) {
|
|
347
|
-
this.
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
401
|
+
if (this.params.log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
402
|
+
this.params.log_writer.error({
|
|
403
|
+
message: `${op}: failed to parse X-ClickHouse-Summary header.`,
|
|
404
|
+
args: {
|
|
405
|
+
operation: op,
|
|
406
|
+
connection_id: this.connectionId,
|
|
407
|
+
'X-ClickHouse-Summary': summaryHeader,
|
|
408
|
+
},
|
|
409
|
+
err: err,
|
|
410
|
+
});
|
|
411
|
+
}
|
|
354
412
|
}
|
|
355
413
|
}
|
|
356
414
|
}
|
|
357
415
|
async runExec(params) {
|
|
358
|
-
const
|
|
416
|
+
const { log_writer, log_level } = this.params;
|
|
417
|
+
const query_id = params.query_id;
|
|
359
418
|
const sendQueryInParams = params.values !== undefined;
|
|
360
419
|
const clickhouse_settings = (0, client_common_1.withHttpSettings)(params.clickhouse_settings, this.params.compression.decompress_response);
|
|
361
420
|
const toSearchParamsOptions = {
|
|
@@ -378,7 +437,7 @@ class NodeBaseConnection {
|
|
|
378
437
|
false;
|
|
379
438
|
const ignoreErrorResponse = params.ignore_error_response ?? false;
|
|
380
439
|
try {
|
|
381
|
-
const { stream, summary, response_headers } = await this.request({
|
|
440
|
+
const { stream, summary, response_headers, http_status_code } = await this.request({
|
|
382
441
|
method: 'POST',
|
|
383
442
|
url: (0, client_common_1.transformUrl)({ url: this.params.url, searchParams }),
|
|
384
443
|
body: sendQueryInParams ? params.values : params.query,
|
|
@@ -390,12 +449,16 @@ class NodeBaseConnection {
|
|
|
390
449
|
ignore_error_response: ignoreErrorResponse,
|
|
391
450
|
headers: this.buildRequestHeaders(params),
|
|
392
451
|
query: params.query,
|
|
452
|
+
query_id,
|
|
453
|
+
log_writer,
|
|
454
|
+
log_level,
|
|
393
455
|
}, params.op);
|
|
394
456
|
return {
|
|
395
457
|
stream,
|
|
396
458
|
query_id,
|
|
397
459
|
summary,
|
|
398
460
|
response_headers,
|
|
461
|
+
http_status_code,
|
|
399
462
|
};
|
|
400
463
|
}
|
|
401
464
|
catch (err) {
|
|
@@ -420,13 +483,15 @@ class NodeBaseConnection {
|
|
|
420
483
|
// allows the event loop to process the idle socket timers, if the CPU load is high
|
|
421
484
|
// otherwise, we can occasionally get an expired socket, see https://github.com/ClickHouse/clickhouse-js/issues/294
|
|
422
485
|
await (0, client_common_1.sleep)(0);
|
|
486
|
+
const { log_writer, query_id, log_level } = params;
|
|
423
487
|
const currentStackTrace = this.params.capture_enhanced_stack_trace
|
|
424
488
|
? (0, client_common_1.getCurrentStackTrace)()
|
|
425
489
|
: undefined;
|
|
426
|
-
const
|
|
490
|
+
const requestTimeout = this.params.request_timeout;
|
|
427
491
|
return new Promise((resolve, reject) => {
|
|
428
492
|
const start = Date.now();
|
|
429
493
|
const request = this.createClientRequest(params);
|
|
494
|
+
const request_id = this.getNewRequestId();
|
|
430
495
|
function onError(e) {
|
|
431
496
|
removeRequestListeners();
|
|
432
497
|
const err = (0, client_common_1.enhanceStackTrace)(e, currentStackTrace);
|
|
@@ -434,14 +499,42 @@ class NodeBaseConnection {
|
|
|
434
499
|
}
|
|
435
500
|
let responseStream;
|
|
436
501
|
const onResponse = async (_response) => {
|
|
437
|
-
this.
|
|
502
|
+
if (this.params.log_level <= client_common_1.ClickHouseLogLevel.DEBUG) {
|
|
503
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
504
|
+
const { authorization, host, ...headers } = request.getHeaders();
|
|
505
|
+
const duration = Date.now() - start;
|
|
506
|
+
// Redact query parameter from URL search params unless explicitly allowed
|
|
507
|
+
let searchParams = params.url.searchParams;
|
|
508
|
+
if (!this.params.unsafeLogUnredactedQueries) {
|
|
509
|
+
// Clone to avoid mutating the original search params
|
|
510
|
+
searchParams = new URLSearchParams(searchParams);
|
|
511
|
+
searchParams.delete('query');
|
|
512
|
+
}
|
|
513
|
+
this.params.log_writer.debug({
|
|
514
|
+
module: 'HTTP Adapter',
|
|
515
|
+
message: `${op}: got a response from ClickHouse`,
|
|
516
|
+
args: {
|
|
517
|
+
operation: op,
|
|
518
|
+
connection_id: this.connectionId,
|
|
519
|
+
query_id,
|
|
520
|
+
request_id,
|
|
521
|
+
request_method: params.method,
|
|
522
|
+
request_path: params.url.pathname,
|
|
523
|
+
request_params: searchParams.toString(),
|
|
524
|
+
request_headers: headers,
|
|
525
|
+
response_status: _response.statusCode,
|
|
526
|
+
response_headers: _response.headers,
|
|
527
|
+
response_time_ms: duration,
|
|
528
|
+
},
|
|
529
|
+
});
|
|
530
|
+
}
|
|
438
531
|
const tryDecompressResponseStream = params.try_decompress_response_stream ?? true;
|
|
439
532
|
const ignoreErrorResponse = params.ignore_error_response ?? false;
|
|
440
533
|
// even if the stream decompression is disabled, we have to decompress it in case of an error
|
|
441
534
|
const isFailedResponse = !(0, client_common_1.isSuccessfulResponse)(_response.statusCode);
|
|
442
535
|
if (tryDecompressResponseStream ||
|
|
443
536
|
(isFailedResponse && !ignoreErrorResponse)) {
|
|
444
|
-
const decompressionResult = (0, compression_1.decompressResponse)(_response,
|
|
537
|
+
const decompressionResult = (0, compression_1.decompressResponse)(_response, log_writer, log_level);
|
|
445
538
|
if ((0, compression_1.isDecompressionError)(decompressionResult)) {
|
|
446
539
|
const err = (0, client_common_1.enhanceStackTrace)(decompressionResult.error, currentStackTrace);
|
|
447
540
|
return reject(err);
|
|
@@ -451,6 +544,24 @@ class NodeBaseConnection {
|
|
|
451
544
|
else {
|
|
452
545
|
responseStream = _response;
|
|
453
546
|
}
|
|
547
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
548
|
+
log_writer.trace({
|
|
549
|
+
message: `${op}: response stream created`,
|
|
550
|
+
args: {
|
|
551
|
+
operation: op,
|
|
552
|
+
connection_id: this.connectionId,
|
|
553
|
+
query_id,
|
|
554
|
+
request_id,
|
|
555
|
+
stream_state: {
|
|
556
|
+
readable: responseStream.readable,
|
|
557
|
+
readableEnded: responseStream.readableEnded,
|
|
558
|
+
readableLength: responseStream.readableLength,
|
|
559
|
+
},
|
|
560
|
+
is_failed_response: isFailedResponse,
|
|
561
|
+
will_decompress: tryDecompressResponseStream,
|
|
562
|
+
},
|
|
563
|
+
});
|
|
564
|
+
}
|
|
454
565
|
if (isFailedResponse && !ignoreErrorResponse) {
|
|
455
566
|
try {
|
|
456
567
|
const errorMessage = await (0, utils_1.getAsText)(responseStream);
|
|
@@ -470,6 +581,7 @@ class NodeBaseConnection {
|
|
|
470
581
|
? this.parseSummary(op, _response)
|
|
471
582
|
: undefined,
|
|
472
583
|
response_headers: { ..._response.headers },
|
|
584
|
+
http_status_code: _response.statusCode ?? undefined,
|
|
473
585
|
});
|
|
474
586
|
}
|
|
475
587
|
};
|
|
@@ -522,96 +634,229 @@ class NodeBaseConnection {
|
|
|
522
634
|
// It is the first time we've encountered this socket,
|
|
523
635
|
// so it doesn't have the idle timeout handler attached to it
|
|
524
636
|
if (socketInfo === undefined) {
|
|
525
|
-
const
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
637
|
+
const socket_id = this.getNewSocketId();
|
|
638
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
639
|
+
log_writer.trace({
|
|
640
|
+
message: `${op}: using a fresh socket, setting up a new 'free' listener`,
|
|
641
|
+
args: {
|
|
642
|
+
operation: op,
|
|
643
|
+
connection_id: this.connectionId,
|
|
644
|
+
query_id,
|
|
645
|
+
request_id,
|
|
646
|
+
socket_id,
|
|
647
|
+
},
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
const newSocketInfo = {
|
|
651
|
+
id: socket_id,
|
|
531
652
|
idle_timeout_handle: undefined,
|
|
532
|
-
|
|
653
|
+
usage_count: 1,
|
|
654
|
+
};
|
|
655
|
+
this.knownSockets.set(socket, newSocketInfo);
|
|
533
656
|
// When the request is complete and the socket is released,
|
|
534
657
|
// make sure that the socket is removed after `idleSocketTTL`.
|
|
535
658
|
socket.on('free', () => {
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
659
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
660
|
+
log_writer.trace({
|
|
661
|
+
message: `${op}: socket was released`,
|
|
662
|
+
args: {
|
|
663
|
+
operation: op,
|
|
664
|
+
connection_id: this.connectionId,
|
|
665
|
+
query_id,
|
|
666
|
+
request_id,
|
|
667
|
+
socket_id,
|
|
668
|
+
},
|
|
669
|
+
});
|
|
670
|
+
}
|
|
539
671
|
// Avoiding the built-in socket.timeout() method usage here,
|
|
540
672
|
// as we don't want to clash with the actual request timeout.
|
|
541
673
|
const idleTimeoutHandle = setTimeout(() => {
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
674
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
675
|
+
log_writer.trace({
|
|
676
|
+
message: `${op}: removing idle socket`,
|
|
677
|
+
args: {
|
|
678
|
+
operation: op,
|
|
679
|
+
connection_id: this.connectionId,
|
|
680
|
+
query_id,
|
|
681
|
+
request_id,
|
|
682
|
+
socket_id,
|
|
683
|
+
idle_socket_ttl_ms: this.idleSocketTTL,
|
|
684
|
+
},
|
|
685
|
+
});
|
|
686
|
+
}
|
|
545
687
|
this.knownSockets.delete(socket);
|
|
546
688
|
socket.destroy();
|
|
547
689
|
}, this.idleSocketTTL).unref();
|
|
548
|
-
|
|
549
|
-
id: socketId,
|
|
550
|
-
idle_timeout_handle: idleTimeoutHandle,
|
|
551
|
-
});
|
|
690
|
+
newSocketInfo.idle_timeout_handle = idleTimeoutHandle;
|
|
552
691
|
});
|
|
553
|
-
const cleanup = () => {
|
|
692
|
+
const cleanup = (eventName) => () => {
|
|
554
693
|
const maybeSocketInfo = this.knownSockets.get(socket);
|
|
555
694
|
// clean up a possibly dangling idle timeout handle (preventing leaks)
|
|
556
695
|
if (maybeSocketInfo?.idle_timeout_handle) {
|
|
557
696
|
clearTimeout(maybeSocketInfo.idle_timeout_handle);
|
|
558
697
|
}
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
if (responseStream && !responseStream.readableEnded) {
|
|
563
|
-
this.logger.warn({
|
|
564
|
-
message: `${op}: socket was closed or ended before the response was fully read. ` +
|
|
565
|
-
'This can potentially result in an uncaught ECONNRESET error! ' +
|
|
566
|
-
'Consider fully consuming, draining, or destroying the response stream.',
|
|
698
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
699
|
+
log_writer.trace({
|
|
700
|
+
message: `${op}: received '${eventName}' event, 'free' listener removed`,
|
|
567
701
|
args: {
|
|
568
|
-
|
|
569
|
-
|
|
702
|
+
operation: op,
|
|
703
|
+
connection_id: this.connectionId,
|
|
704
|
+
query_id,
|
|
705
|
+
request_id,
|
|
706
|
+
socket_id,
|
|
707
|
+
event: eventName,
|
|
570
708
|
},
|
|
571
709
|
});
|
|
572
710
|
}
|
|
711
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.WARN) {
|
|
712
|
+
if (responseStream && !responseStream.readableEnded) {
|
|
713
|
+
log_writer.warn({
|
|
714
|
+
message: `${op}: socket was closed or ended before the response was fully read. ` +
|
|
715
|
+
'This can potentially result in an uncaught ECONNRESET error! ' +
|
|
716
|
+
'Consider fully consuming, draining, or destroying the response stream.',
|
|
717
|
+
args: {
|
|
718
|
+
operation: op,
|
|
719
|
+
connection_id: this.connectionId,
|
|
720
|
+
query_id,
|
|
721
|
+
request_id,
|
|
722
|
+
socket_id,
|
|
723
|
+
event: eventName,
|
|
724
|
+
query: this.params.unsafeLogUnredactedQueries
|
|
725
|
+
? params.query
|
|
726
|
+
: undefined,
|
|
727
|
+
},
|
|
728
|
+
});
|
|
729
|
+
}
|
|
730
|
+
}
|
|
573
731
|
};
|
|
574
|
-
socket.once('end', cleanup);
|
|
575
|
-
socket.once('close', cleanup);
|
|
732
|
+
socket.once('end', cleanup('end'));
|
|
733
|
+
socket.once('close', cleanup('close'));
|
|
576
734
|
}
|
|
577
735
|
else {
|
|
578
736
|
clearTimeout(socketInfo.idle_timeout_handle);
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
737
|
+
socketInfo.idle_timeout_handle = undefined;
|
|
738
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
739
|
+
log_writer.trace({
|
|
740
|
+
message: `${op}: reusing socket`,
|
|
741
|
+
args: {
|
|
742
|
+
operation: op,
|
|
743
|
+
connection_id: this.connectionId,
|
|
744
|
+
query_id,
|
|
745
|
+
request_id,
|
|
746
|
+
socket_id: socketInfo.id,
|
|
747
|
+
usage_count: socketInfo.usage_count,
|
|
748
|
+
},
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
socketInfo.usage_count++;
|
|
586
752
|
}
|
|
587
753
|
}
|
|
588
754
|
}
|
|
589
755
|
catch (e) {
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
756
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
757
|
+
log_writer.error({
|
|
758
|
+
message: `${op}: an error occurred while housekeeping the idle sockets`,
|
|
759
|
+
err: e,
|
|
760
|
+
args: {
|
|
761
|
+
operation: op,
|
|
762
|
+
connection_id: this.connectionId,
|
|
763
|
+
query_id,
|
|
764
|
+
request_id,
|
|
765
|
+
},
|
|
766
|
+
});
|
|
767
|
+
}
|
|
594
768
|
}
|
|
595
769
|
// Socket is "prepared" with idle handlers, continue with our request
|
|
596
770
|
pipeStream();
|
|
597
771
|
// This is for request timeout only. Surprisingly, it is not always enough to set in the HTTP request.
|
|
598
772
|
// The socket won't be destroyed, and it will be returned to the pool.
|
|
773
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
774
|
+
const socketInfo = this.knownSockets.get(socket);
|
|
775
|
+
if (socketInfo) {
|
|
776
|
+
log_writer.trace({
|
|
777
|
+
message: `${op}: setting up request timeout`,
|
|
778
|
+
args: {
|
|
779
|
+
operation: op,
|
|
780
|
+
connection_id: this.connectionId,
|
|
781
|
+
query_id,
|
|
782
|
+
request_id,
|
|
783
|
+
socket_id: socketInfo.id,
|
|
784
|
+
timeout_ms: requestTimeout,
|
|
785
|
+
},
|
|
786
|
+
});
|
|
787
|
+
}
|
|
788
|
+
else {
|
|
789
|
+
log_writer.trace({
|
|
790
|
+
message: `${op}: setting up request timeout on a socket`,
|
|
791
|
+
args: {
|
|
792
|
+
operation: op,
|
|
793
|
+
connection_id: this.connectionId,
|
|
794
|
+
query_id,
|
|
795
|
+
request_id,
|
|
796
|
+
timeout_ms: requestTimeout,
|
|
797
|
+
},
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
}
|
|
599
801
|
socket.setTimeout(this.params.request_timeout, onTimeout);
|
|
600
802
|
};
|
|
601
|
-
|
|
602
|
-
const err = (0, client_common_1.enhanceStackTrace)(new Error('Timeout error.'), currentStackTrace);
|
|
803
|
+
const onTimeout = () => {
|
|
603
804
|
removeRequestListeners();
|
|
805
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.TRACE) {
|
|
806
|
+
const socket = request.socket;
|
|
807
|
+
const maybeSocketInfo = socket
|
|
808
|
+
? this.knownSockets.get(socket)
|
|
809
|
+
: undefined;
|
|
810
|
+
const socketState = request.socket
|
|
811
|
+
? {
|
|
812
|
+
connecting: request.socket.connecting,
|
|
813
|
+
pending: request.socket.pending,
|
|
814
|
+
destroyed: request.socket.destroyed,
|
|
815
|
+
readyState: request.socket.readyState,
|
|
816
|
+
}
|
|
817
|
+
: undefined;
|
|
818
|
+
const responseStreamState = responseStream
|
|
819
|
+
? {
|
|
820
|
+
readable: responseStream.readable,
|
|
821
|
+
readableEnded: responseStream.readableEnded,
|
|
822
|
+
readableLength: responseStream.readableLength,
|
|
823
|
+
}
|
|
824
|
+
: undefined;
|
|
825
|
+
log_writer.trace({
|
|
826
|
+
message: `${op}: timeout occurred`,
|
|
827
|
+
args: {
|
|
828
|
+
operation: op,
|
|
829
|
+
connection_id: this.connectionId,
|
|
830
|
+
query_id,
|
|
831
|
+
request_id,
|
|
832
|
+
socket_id: maybeSocketInfo?.id,
|
|
833
|
+
timeout_ms: requestTimeout,
|
|
834
|
+
socket_state: socketState,
|
|
835
|
+
response_stream_state: responseStreamState,
|
|
836
|
+
has_response_stream: responseStream !== undefined,
|
|
837
|
+
},
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
const err = (0, client_common_1.enhanceStackTrace)(new Error('Timeout error.'), currentStackTrace);
|
|
604
841
|
try {
|
|
605
842
|
request.destroy();
|
|
606
843
|
}
|
|
607
844
|
catch (e) {
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
845
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
846
|
+
log_writer.error({
|
|
847
|
+
message: `${op}: An error occurred while destroying the request`,
|
|
848
|
+
err: e,
|
|
849
|
+
args: {
|
|
850
|
+
operation: op,
|
|
851
|
+
connection_id: this.connectionId,
|
|
852
|
+
query_id,
|
|
853
|
+
request_id,
|
|
854
|
+
},
|
|
855
|
+
});
|
|
856
|
+
}
|
|
612
857
|
}
|
|
613
858
|
reject(err);
|
|
614
|
-
}
|
|
859
|
+
};
|
|
615
860
|
function removeRequestListeners() {
|
|
616
861
|
if (request.socket !== null) {
|
|
617
862
|
request.socket.setTimeout(0); // reset previously set timeout
|
|
@@ -639,10 +884,18 @@ class NodeBaseConnection {
|
|
|
639
884
|
return request.end();
|
|
640
885
|
}
|
|
641
886
|
catch (e) {
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
887
|
+
if (log_level <= client_common_1.ClickHouseLogLevel.ERROR) {
|
|
888
|
+
log_writer.error({
|
|
889
|
+
message: `${op}: an error occurred while ending the request without body`,
|
|
890
|
+
err: e,
|
|
891
|
+
args: {
|
|
892
|
+
operation: op,
|
|
893
|
+
connection_id: this.connectionId,
|
|
894
|
+
query_id,
|
|
895
|
+
request_id,
|
|
896
|
+
},
|
|
897
|
+
});
|
|
898
|
+
}
|
|
646
899
|
}
|
|
647
900
|
}
|
|
648
901
|
});
|