@lightdash/warehouses 3.136.2 → 3.137.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAI7B,MAAM,EAIN,mBAAmB,EAEnB,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,EACvB,mBAAmB,
|
|
1
|
+
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAGH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAI7B,MAAM,EAIN,mBAAmB,EAEnB,gBAAgB,EAChB,wBAAwB,EACxB,uBAAuB,EACvB,mBAAmB,EAEnB,gBAAgB,EAChB,eAAe,EACf,cAAc,EAEd,KAAK,eAAe,EAEpB,KAAK,mBAAmB,EAE3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAc,WAAW,EAAS,MAAM,IAAI,CAAC;AAKpD,OAAO,mBAAmB,CAAC;AAE3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE,eAAO,MAAM,gCAAgC,aAAc,MAAM,KAAG,OACZ,CAAC;AAEzD,eAAO,MAAM,iCAAiC,aAAc,MAAM,WAC2B,CAAC;AAE9F,OAAO,QAAQ,IAAI,CAAC;IAChB,UAAU,UAAU;QAEhB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;KAC5B;CACJ;AAKD,oBAAY,aAAa;IACrB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,WAAW,gBAAgB;IAC3B,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,gBAAgB,qBAAqB;IACrC,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,WAAW;IAClB,sBAAsB,2BAA2B;IACjD,SAAS,cAAc;IACvB,YAAY,gBAAgB;IAC5B,2BAA2B,gCAAgC;IAC3D,wBAAwB,6BAA6B;CACxD;AA4CD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,KACb,eAAe,GAAG,SAWpB,CAAC;AAiFF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,IAAI,iBAA2B;IAE/B,cAAc,IAAI,mBAAmB,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAahD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAED,MAAM,MAAM,0BAA0B,GAAG;IACrC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,KAAK,IAAI,CAAC;CAC5E,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,aAAa,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;IACnC,YAAY,EAAE,MAAM,OAAO,CAAC;IAC5B,cAAc,EAAE,MAAM,OAAO,CAAC;IAC9B,mFAAmF;IACnF,KAAK,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,OAAO,CAAC;CAClD,CAAC;AAEF,KAAK,mBAAmB,GAAG;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,qBAAa,cAAc,CACvB,CAAC,SAAS,6BAA6B,CACzC,SAAQ,mBAAmB,CAAC,CAAC,CAAC;IACtB,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAYjD;IAED,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAEtB,YACI,WAAW,EAAE,CAAC,EACd,MAAM,EAAE,EAAE,CAAC,UAAU,EACrB,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAA;KAAE,EAavC;IAED,SAAS,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAE9D;IAED,SAAS,CAAC,wBAAwB,IAAI,MAAM,CAE3C;IAED,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAOrB;IAED,OAAO,CAAC,kBAAkB;IAQ1B,MAAM,CAAC,wBAAwB,CAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GACvC,gBAAgB,CAAC,QAAQ,CAAC,CAgB5B;IAED,OAAO,CAAC,MAAM,CAAC,cAAc;IAavB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE,0BAA0B,GACpC,OAAO,CAAC,IAAI,CAAC,CAEf;IAED,UAAgB,kBAAkB,CAC9B,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE,0BAA0B,EACnC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,oBAAoB,KAAK,IAAI,CAAC,GAAG,IAAI,GACjE,OAAO,CAAC,IAAI,CAAC,CA6Rf;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,gCA4GN;IAED,OAAO,CAAC,aAAa,CAA8B;IAEnD,SAAS,CAAC,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,CAK5C;IAGD,UAAgB,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,CAKnD;IAED,UAAgB,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC,CAE7C;IAGK,YAAY;;;;;SA4BjB;IAGD,SAAS,CAAC,iBAAiB,CACvB,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM;;;;MAQpB;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAyC3B;IAED,SAAS,CAAC,oBAAoB,CAC1B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAChC,gBAAgB,CAMlB;IAED,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,EAAE,KAAK,GAAE,MAAW,uBAsDrD;CACJ;AA0CD,qBAAa,uBAAwB,SAAQ,cAAc,CAAC,yBAAyB,CAAC;IAClF,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAarB;IAED,YACI,WAAW,EAAE,yBAAyB,EACtC,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,OAAO,CAAA;KAAE,EAevC;IAED,OAAO,CAAC,gBAAgB;YASV,qBAAqB;IAoB7B,aAAa,IAAI,OAAO,CAAC,wBAAwB,CAAC,CAkCvD;IAEK,oBAAoB,CACtB,cAAc,EAAE,uBAAuB,GACxC,OAAO,CAAC,eAAe,CAAC,CAS1B;CACJ"}
|
|
@@ -168,6 +168,12 @@ const DEFAULT_STATEMENT_TIMEOUT_MS = 1000 * 60 * 9; // 9 minutes
|
|
|
168
168
|
// surfaces a clear message; the client backstop only triggers if the server
|
|
169
169
|
// never reports back (e.g. a dead SSH tunnel socket).
|
|
170
170
|
const CLIENT_STATEMENT_TIMEOUT_BUFFER_MS = 1000 * 30; // 30 seconds
|
|
171
|
+
const formatTimeoutDetails = ({ step, stepElapsedMs, completedStepTimingsMs, rowsFetched, backendPid, }) => {
|
|
172
|
+
const completed = Object.entries(completedStepTimingsMs)
|
|
173
|
+
.map(([phase, ms]) => `${phase}=${ms}ms`)
|
|
174
|
+
.join(' ');
|
|
175
|
+
return `step=${step} step_elapsed=${stepElapsedMs}ms completed=[${completed}] rows=${rowsFetched} backend_pid=${backendPid ?? 'unknown'}`;
|
|
176
|
+
};
|
|
171
177
|
const convertDataTypeIdToDimensionType = (dataTypeId) => {
|
|
172
178
|
switch (dataTypeId) {
|
|
173
179
|
case builtins.NUMERIC:
|
|
@@ -325,7 +331,19 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
325
331
|
let backendPid = null;
|
|
326
332
|
let sentQuery = false;
|
|
327
333
|
let receivedRow = false;
|
|
328
|
-
|
|
334
|
+
// Tracked so a backstop timeout can say which step never finished.
|
|
335
|
+
const completedStepTimingsMs = {};
|
|
336
|
+
let currentStep = 'connect';
|
|
337
|
+
let currentStepStartedAt = performance.now();
|
|
338
|
+
let rowsFetched = 0;
|
|
339
|
+
const startStep = (step) => {
|
|
340
|
+
currentStep = step;
|
|
341
|
+
currentStepStartedAt = performance.now();
|
|
342
|
+
};
|
|
343
|
+
const reportPhase = (phase, durationMs) => {
|
|
344
|
+
completedStepTimingsMs[phase] = Math.round(durationMs);
|
|
345
|
+
options.onPhaseTiming?.(phase, durationMs);
|
|
346
|
+
};
|
|
329
347
|
// Enforce the ceiling with a server-side `statement_timeout` (set
|
|
330
348
|
// below) plus this client-side wall-clock backstop, which fires shortly
|
|
331
349
|
// after in case the server never reports back (e.g. a stalled SSH
|
|
@@ -337,7 +355,16 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
337
355
|
const clientTimeoutMs = statementTimeoutMs + CLIENT_STATEMENT_TIMEOUT_BUFFER_MS;
|
|
338
356
|
return new Promise((resolve, reject) => {
|
|
339
357
|
clientTimeout = setTimeout(() => {
|
|
340
|
-
const
|
|
358
|
+
const message = `Query timed out after ${Math.round(clientTimeoutMs / 1000)}s`;
|
|
359
|
+
const details = {
|
|
360
|
+
step: currentStep,
|
|
361
|
+
stepElapsedMs: Math.round(performance.now() - currentStepStartedAt),
|
|
362
|
+
completedStepTimingsMs: { ...completedStepTimingsMs },
|
|
363
|
+
rowsFetched,
|
|
364
|
+
backendPid,
|
|
365
|
+
};
|
|
366
|
+
console.warn(`${message} with no response from ${this.credentials.type}: ${formatTimeoutDetails(details)} project_uuid=${options.tags?.project_uuid ?? 'unknown'} query_context=${options.tags?.query_context ?? 'unknown'}`);
|
|
367
|
+
const timeoutError = new common_1.WarehouseQueryTimeoutError(message, details);
|
|
341
368
|
activeStream?.destroy(timeoutError);
|
|
342
369
|
reject(timeoutError);
|
|
343
370
|
}, clientTimeoutMs);
|
|
@@ -380,8 +407,9 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
380
407
|
reject(new Error('client undefined'));
|
|
381
408
|
return;
|
|
382
409
|
}
|
|
383
|
-
reportPhase
|
|
410
|
+
reportPhase('connect', performance.now() - connectStart);
|
|
384
411
|
backendPid = client.processID;
|
|
412
|
+
startStep('session');
|
|
385
413
|
client.on('error', (e) => {
|
|
386
414
|
console.error(`Postgres client error ${(0, common_1.getErrorMessage)(e)}`);
|
|
387
415
|
reject(e);
|
|
@@ -413,9 +441,11 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
413
441
|
if (cachedFields === null) {
|
|
414
442
|
cachedFields =
|
|
415
443
|
PostgresClient.convertQueryResultFields(chunk.fields);
|
|
416
|
-
reportPhase
|
|
444
|
+
reportPhase('query', performance.now() - queryStart);
|
|
417
445
|
fetchStart = performance.now();
|
|
446
|
+
startStep('fetch');
|
|
418
447
|
}
|
|
448
|
+
rowsFetched += 1;
|
|
419
449
|
await streamCallback({
|
|
420
450
|
fields: cachedFields,
|
|
421
451
|
rows: [chunk.row],
|
|
@@ -436,11 +466,11 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
436
466
|
// (not 'end' on readable - async write callbacks may still be in flight)
|
|
437
467
|
writable.on('finish', () => {
|
|
438
468
|
if (fetchStart === undefined) {
|
|
439
|
-
reportPhase
|
|
440
|
-
reportPhase
|
|
469
|
+
reportPhase('query', performance.now() - queryStart);
|
|
470
|
+
reportPhase('fetch', 0);
|
|
441
471
|
}
|
|
442
472
|
else {
|
|
443
|
-
reportPhase
|
|
473
|
+
reportPhase('fetch', performance.now() - fetchStart);
|
|
444
474
|
}
|
|
445
475
|
resolve();
|
|
446
476
|
});
|
|
@@ -473,7 +503,8 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
473
503
|
return undefined;
|
|
474
504
|
})
|
|
475
505
|
.then(() => {
|
|
476
|
-
reportPhase
|
|
506
|
+
reportPhase('session', performance.now() - sessionStart);
|
|
507
|
+
startStep('query');
|
|
477
508
|
runQuery();
|
|
478
509
|
})
|
|
479
510
|
.catch((sessionError) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.137.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"@databricks/sql": "1.13.0",
|
|
22
22
|
"@duckdb/node-api": "1.5.2-r.2",
|
|
23
23
|
"@google-cloud/bigquery": "8.3.1",
|
|
24
|
-
"@lightdash/common": "3.
|
|
24
|
+
"@lightdash/common": "3.137.1",
|
|
25
25
|
"node-fetch": "2.7.0",
|
|
26
26
|
"pg": "8.13.1",
|
|
27
27
|
"pg-cursor": "2.10.0",
|