@lightdash/warehouses 1.242.6 → 1.243.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.js +9 -5
- package/dist/warehouseClients/DatabricksWarehouseClient.test.js +33 -10
- package/dist/warehouseClients/PostgresWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/PostgresWarehouseClient.js +18 -27
- package/dist/warehouseClients/PostgresWarehouseClient.test.js +8 -0
- package/package.json +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DatabricksWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DatabricksWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAqB,EACjB,iBAAiB,EACpB,MAAM,6CAA6C,CAAC;
|
|
1
|
+
{"version":3,"file":"DatabricksWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DatabricksWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAqB,EACjB,iBAAiB,EACpB,MAAM,6CAA6C,CAAC;AAKrD,OAAO,EACH,OAAO,EACP,2BAA2B,EAI3B,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAIhB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE;;;;;;GAMG;AACH,eAAO,MAAM,kCAAkC,mBAAmB,CAAC;AAQnE,eAAO,MAAM,4BAA4B,aAC3B,MAAM,GAAG,SAAS,KAC7B,OAAsE,CAAC;AAoJ1E,eAAO,MAAM,4BAA4B,SAC/B,MAAM,KACb,eAAe,GAAG,SASpB,CAAC;AAEF,qBAAa,oBAAqB,SAAQ,uBAAuB;IAC7D,QAAQ,CAAC,IAAI,6BAA6B;IAE1C,cAAc,IAAI,mBAAmB,CAEpC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAShD;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAO1D;CACJ;AAaD,qBAAa,yBAA0B,SAAQ,mBAAmB,CAAC,2BAA2B,CAAC;IAC3F,MAAM,EAAE,MAAM,CAAC;IAEf,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,iBAAiB,EAAE,iBAAiB,CAAC;IAErC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IAEzC,YAAY,WAAW,EAAE,2BAA2B,EA+CnD;YAEa,UAAU;IAqClB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC,CAwEf;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,6BA0EN;IAEK,YAAY;;;;SAajB;IAEK,SAAS,CACX,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAoB3B;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAyB3B;CACJ;AAED;;GAEG;AACH,eAAO,MAAM,kCAAkC,SACrC,MAAM,YACF,MAAM,gBACF,MAAM,KACrB,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,CA+BxD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,2BAA2B,SAC9B,MAAM,YACF,MAAM,gBACF,MAAM,iBACL,MAAM,KACtB,OAAO,CAAC;IACP,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACrB,CAqCA,CAAC"}
|
|
@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.refreshDatabricksOAuthToken = exports.exchangeDatabricksOAuthCredentials = exports.DatabricksWarehouseClient = exports.DatabricksSqlBuilder = exports.getDatabricksTimestampDomain = exports.isDatabricksCliOAuthClientId = exports.DATABRICKS_DEFAULT_OAUTH_CLIENT_ID = void 0;
|
|
7
7
|
const sql_1 = require("@databricks/sql");
|
|
8
|
+
const StatusError_1 = __importDefault(require("@databricks/sql/dist/errors/StatusError"));
|
|
8
9
|
const TCLIService_types_1 = require("@databricks/sql/thrift/TCLIService_types");
|
|
9
10
|
const common_1 = require("@lightdash/common");
|
|
10
11
|
const node_fetch_1 = __importDefault(require("node-fetch"));
|
|
@@ -216,6 +217,9 @@ const DATABRICKS_QUERY_TIMEOUT_SECONDS = 300;
|
|
|
216
217
|
// @databricks/sql defaults fetchChunk to 100k rows, which materializes entire
|
|
217
218
|
// wide results in one array and can OOM the worker on large queries
|
|
218
219
|
const DATABRICKS_FETCH_CHUNK_MAX_ROWS = 5000;
|
|
220
|
+
const getDatabricksErrorMessage = (error) => error instanceof StatusError_1.default && error.message
|
|
221
|
+
? error.message
|
|
222
|
+
: (0, common_1.getErrorMessage)(error);
|
|
219
223
|
class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
220
224
|
constructor(credentials) {
|
|
221
225
|
super(credentials, new DatabricksSqlBuilder(credentials.startOfWeek));
|
|
@@ -267,7 +271,7 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
267
271
|
connection = await client.connect(this.connectionOptions);
|
|
268
272
|
}
|
|
269
273
|
catch (e) {
|
|
270
|
-
throw new common_1.WarehouseConnectionError((
|
|
274
|
+
throw new common_1.WarehouseConnectionError(getDatabricksErrorMessage(e));
|
|
271
275
|
}
|
|
272
276
|
try {
|
|
273
277
|
session = await connection.openSession({
|
|
@@ -282,7 +286,7 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
282
286
|
catch (closeError) {
|
|
283
287
|
console.error('Error closing connection after session failure', closeError);
|
|
284
288
|
}
|
|
285
|
-
throw new common_1.WarehouseConnectionError((
|
|
289
|
+
throw new common_1.WarehouseConnectionError(getDatabricksErrorMessage(e));
|
|
286
290
|
}
|
|
287
291
|
return {
|
|
288
292
|
session,
|
|
@@ -335,7 +339,7 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
335
339
|
} while (await query.hasMoreRows());
|
|
336
340
|
}
|
|
337
341
|
catch (e) {
|
|
338
|
-
throw new common_1.WarehouseQueryError((
|
|
342
|
+
throw new common_1.WarehouseQueryError(getDatabricksErrorMessage(e));
|
|
339
343
|
}
|
|
340
344
|
finally {
|
|
341
345
|
try {
|
|
@@ -364,7 +368,7 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
364
368
|
return (await query.fetchAll());
|
|
365
369
|
}
|
|
366
370
|
catch (e) {
|
|
367
|
-
throw new common_1.WarehouseQueryError((
|
|
371
|
+
throw new common_1.WarehouseQueryError(getDatabricksErrorMessage(e));
|
|
368
372
|
}
|
|
369
373
|
finally {
|
|
370
374
|
try {
|
|
@@ -379,7 +383,7 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
379
383
|
});
|
|
380
384
|
}
|
|
381
385
|
catch (e) {
|
|
382
|
-
throw new common_1.WarehouseQueryError((
|
|
386
|
+
throw new common_1.WarehouseQueryError(getDatabricksErrorMessage(e));
|
|
383
387
|
}
|
|
384
388
|
finally {
|
|
385
389
|
try {
|
|
@@ -1,32 +1,55 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
6
|
/* eslint-disable prefer-arrow-callback, func-names */
|
|
7
|
+
const StatusError_1 = __importDefault(require("@databricks/sql/dist/errors/StatusError"));
|
|
8
|
+
const TCLIService_types_1 = require("@databricks/sql/thrift/TCLIService_types");
|
|
4
9
|
const DatabricksWarehouseClient_1 = require("./DatabricksWarehouseClient");
|
|
5
10
|
const DatabricksWarehouseClient_mock_1 = require("./DatabricksWarehouseClient.mock");
|
|
6
11
|
const WarehouseClient_mock_1 = require("./WarehouseClient.mock");
|
|
7
12
|
const mocks = vi.hoisted(() => ({
|
|
8
13
|
fetchChunk: vi.fn(),
|
|
14
|
+
openSession: vi.fn(),
|
|
15
|
+
closeConnection: vi.fn(),
|
|
9
16
|
}));
|
|
10
17
|
vi.mock('@databricks/sql', async () => ({
|
|
11
18
|
...(await vi.importActual('@databricks/sql')),
|
|
12
19
|
DBSQLClient: vi.fn(function () {
|
|
13
20
|
return {
|
|
14
21
|
connect: vi.fn(() => ({
|
|
15
|
-
openSession:
|
|
16
|
-
|
|
17
|
-
getSchema: vi.fn(async () => DatabricksWarehouseClient_mock_1.schema),
|
|
18
|
-
fetchChunk: mocks.fetchChunk.mockImplementation(async () => DatabricksWarehouseClient_mock_1.rows),
|
|
19
|
-
hasMoreRows: vi.fn(async () => false),
|
|
20
|
-
close: vi.fn(async () => undefined),
|
|
21
|
-
})),
|
|
22
|
-
close: vi.fn(async () => undefined),
|
|
23
|
-
})),
|
|
24
|
-
close: vi.fn(async () => undefined),
|
|
22
|
+
openSession: mocks.openSession,
|
|
23
|
+
close: mocks.closeConnection,
|
|
25
24
|
})),
|
|
26
25
|
};
|
|
27
26
|
}),
|
|
28
27
|
}));
|
|
29
28
|
describe('DatabricksWarehouseClient', () => {
|
|
29
|
+
beforeEach(() => {
|
|
30
|
+
mocks.fetchChunk.mockReset().mockResolvedValue(DatabricksWarehouseClient_mock_1.rows);
|
|
31
|
+
mocks.closeConnection.mockReset().mockResolvedValue(undefined);
|
|
32
|
+
mocks.openSession.mockReset().mockResolvedValue({
|
|
33
|
+
executeStatement: vi.fn(() => ({
|
|
34
|
+
getSchema: vi.fn(async () => DatabricksWarehouseClient_mock_1.schema),
|
|
35
|
+
fetchChunk: mocks.fetchChunk,
|
|
36
|
+
hasMoreRows: vi.fn(async () => false),
|
|
37
|
+
close: vi.fn(async () => undefined),
|
|
38
|
+
})),
|
|
39
|
+
close: vi.fn(async () => undefined),
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
it('surfaces Databricks status messages when opening a session fails', async () => {
|
|
43
|
+
const message = 'SQL warehouse xyz is not ready to accept connections (current state: STARTING)';
|
|
44
|
+
mocks.openSession.mockRejectedValueOnce(new StatusError_1.default({
|
|
45
|
+
statusCode: TCLIService_types_1.TStatusCode.ERROR_STATUS,
|
|
46
|
+
errorMessage: message,
|
|
47
|
+
}));
|
|
48
|
+
const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
|
|
49
|
+
await expect(warehouse.runQuery('fake sql')).rejects.toMatchObject({
|
|
50
|
+
message,
|
|
51
|
+
});
|
|
52
|
+
});
|
|
30
53
|
it('expect query fields and rows', async () => {
|
|
31
54
|
const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
|
|
32
55
|
const results = await warehouse.runQuery('fake sql');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAG7B,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAc,WAAW,EAAS,MAAM,IAAI,CAAC;AAKpD,OAAO,mBAAmB,CAAC;AAE3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,aAAa;IACrB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,WAAW,gBAAgB;IAC3B,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,gBAAgB,qBAAqB;IACrC,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,WAAW;IAClB,sBAAsB,2BAA2B;IACjD,SAAS,cAAc;IACvB,YAAY,gBAAgB;IAC5B,2BAA2B,gCAAgC;IAC3D,wBAAwB,6BAA6B;CACxD;AA4CD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,KACb,eAAe,GAAG,SAWpB,CAAC;AA6CF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,IAAI,iBAA2B;IAE/B,cAAc,IAAI,mBAAmB,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAahD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAED,KAAK,mBAAmB,GAAG;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,qBAAa,cAAc,CACvB,CAAC,SAAS,6BAA6B,CACzC,SAAQ,mBAAmB,CAAC,CAAC,CAAC;IACtB,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAYjD;IAED,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAEtB,YAAY,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,EAGhD;IAED,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAOrB;IAED,OAAO,CAAC,kBAAkB;IAQ1B,MAAM,CAAC,wBAAwB,CAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GACvC,gBAAgB,CAAC,QAAQ,CAAC,CAO5B;IAED,OAAO,CAAC,MAAM,CAAC,cAAc;IAavB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"PostgresWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/PostgresWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,yBAAyB,EACzB,6BAA6B,EAG7B,MAAM,EAIN,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACpB,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAG3B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAc,WAAW,EAAS,MAAM,IAAI,CAAC;AAKpD,OAAO,mBAAmB,CAAC;AAE3B,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,aAAa;IACrB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,IAAI,SAAS;IACb,IAAI,SAAS;IACb,KAAK,UAAU;IACf,WAAW,gBAAgB;IAC3B,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,gBAAgB,qBAAqB;IACrC,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,OAAO,WAAW;IAClB,sBAAsB,2BAA2B;IACjD,SAAS,cAAc;IACvB,YAAY,gBAAgB;IAC5B,2BAA2B,gCAAgC;IAC3D,wBAAwB,6BAA6B;CACxD;AA4CD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,KACb,eAAe,GAAG,SAWpB,CAAC;AA6CF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,IAAI,iBAA2B;IAE/B,cAAc,IAAI,mBAAmB,CAEpC;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAahD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;CACJ;AAED,KAAK,mBAAmB,GAAG;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF,qBAAa,cAAc,CACvB,CAAC,SAAS,6BAA6B,CACzC,SAAQ,mBAAmB,CAAC,CAAC,CAAC;IACtB,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAYjD;IAED,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAEtB,YAAY,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,UAAU,EAGhD;IAED,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAOrB;IAED,OAAO,CAAC,kBAAkB;IAQ1B,MAAM,CAAC,wBAAwB,CAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,GACvC,gBAAgB,CAAC,QAAQ,CAAC,CAO5B;IAED,OAAO,CAAC,MAAM,CAAC,cAAc;IAavB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAiOf;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,gCAmHN;IAEK,YAAY;;;;SAuBjB;IAEK,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CA0B3B;IAED,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,EAAE,KAAK,GAAE,MAAW,uBAsDrD;CACJ;AA0CD,qBAAa,uBAAwB,SAAQ,cAAc,CAAC,yBAAyB,CAAC;IAClF,SAAS,CAAC,sBAAsB,CAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,EACpB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GACpB,mBAAmB,CAarB;IAED,YAAY,WAAW,EAAE,yBAAyB,EAUjD;CACJ"}
|
|
@@ -154,10 +154,10 @@ const { builtins } = pg.types;
|
|
|
154
154
|
const POSTGRES_NAME_TOO_LONG_SQLSTATE = '42622';
|
|
155
155
|
// Server-side ceiling for a single streamed query, bounded just under the
|
|
156
156
|
// 10-min scheduler job timeout so a stalled cursor fails clearly instead of
|
|
157
|
-
// hanging the whole job.
|
|
158
|
-
//
|
|
159
|
-
//
|
|
160
|
-
// `timeoutSeconds`.
|
|
157
|
+
// hanging the whole job. Enforced via `statement_timeout` plus a client-side
|
|
158
|
+
// wall-clock backstop. Do not also configure the pg pool's `query_timeout`: it
|
|
159
|
+
// applies to cursor queries and would race these deliberately ordered limits.
|
|
160
|
+
// Overridable per-connection via `timeoutSeconds`.
|
|
161
161
|
const DEFAULT_STATEMENT_TIMEOUT_MS = 1000 * 60 * 9; // 9 minutes
|
|
162
162
|
// The client-side backstop fires this long after the server-side
|
|
163
163
|
// statement_timeout, so the server's cancellation error wins the race and
|
|
@@ -278,19 +278,19 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
278
278
|
let pool;
|
|
279
279
|
let closeClient;
|
|
280
280
|
let activeStream;
|
|
281
|
-
let
|
|
281
|
+
let clientTimeout;
|
|
282
282
|
const reportPhase = options.onPhaseTiming;
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
//
|
|
286
|
-
//
|
|
287
|
-
//
|
|
283
|
+
// Enforce the ceiling with a server-side `statement_timeout` (set
|
|
284
|
+
// below) plus this client-side wall-clock backstop, which fires shortly
|
|
285
|
+
// after in case the server never reports back (e.g. a stalled SSH
|
|
286
|
+
// tunnel socket). A pg `query_timeout` is deliberately omitted because
|
|
287
|
+
// it also applies to cursor queries and would race the server timeout.
|
|
288
288
|
const statementTimeoutMs = this.credentials.timeoutSeconds
|
|
289
289
|
? this.credentials.timeoutSeconds * 1000
|
|
290
290
|
: DEFAULT_STATEMENT_TIMEOUT_MS;
|
|
291
291
|
const clientTimeoutMs = statementTimeoutMs + CLIENT_STATEMENT_TIMEOUT_BUFFER_MS;
|
|
292
292
|
return new Promise((resolve, reject) => {
|
|
293
|
-
|
|
293
|
+
clientTimeout = setTimeout(() => {
|
|
294
294
|
const timeoutError = new common_1.WarehouseQueryError(`Query timed out after ${Math.round(clientTimeoutMs / 1000)}s`);
|
|
295
295
|
activeStream?.destroy(timeoutError);
|
|
296
296
|
reject(timeoutError);
|
|
@@ -298,9 +298,6 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
298
298
|
pool = new pg.Pool({
|
|
299
299
|
...this.config,
|
|
300
300
|
connectionTimeoutMillis: 30000,
|
|
301
|
-
query_timeout: this.credentials.timeoutSeconds
|
|
302
|
-
? this.credentials.timeoutSeconds * 1000
|
|
303
|
-
: 1000 * 60 * 5, // sets the default query timeout to 5 minutes
|
|
304
301
|
});
|
|
305
302
|
pool.on('error', (err) => {
|
|
306
303
|
console.error(`Postgres pool error ${(0, common_1.getErrorMessage)(err)}`);
|
|
@@ -343,13 +340,7 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
343
340
|
let fetchStart;
|
|
344
341
|
// CodeQL: This will raise a security warning because user defined raw SQL is being passed into the database module.
|
|
345
342
|
// In this case this is exactly what we want to do. We're hitting the user's warehouse not the application's database.
|
|
346
|
-
activeStream = client.query(
|
|
347
|
-
// callback is not defined in types when using QueryStream
|
|
348
|
-
// @ts-ignore
|
|
349
|
-
new PgQueryStream_1.default(this.getSQLWithMetadata(sql, options?.tags), options?.values),
|
|
350
|
-
// there is a bug in PG lib where callback is required when passing `query_timeout` to the Pool
|
|
351
|
-
// see the code: https://github.com/brianc/node-postgres/blob/master/packages/pg/lib/client.js#L541-L542
|
|
352
|
-
() => { });
|
|
343
|
+
activeStream = client.query(new PgQueryStream_1.default(this.getSQLWithMetadata(sql, options?.tags), options?.values));
|
|
353
344
|
// Cache field conversion — result.fields is the same
|
|
354
345
|
// array reference for every row in a query, so we only
|
|
355
346
|
// need to convert it once.
|
|
@@ -402,10 +393,10 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
402
393
|
reject(err2);
|
|
403
394
|
});
|
|
404
395
|
};
|
|
405
|
-
// Always enforce
|
|
406
|
-
//
|
|
407
|
-
//
|
|
408
|
-
//
|
|
396
|
+
// Always enforce the primary query-execution ceiling on the
|
|
397
|
+
// server. Issued as its own single statement (followed by the
|
|
398
|
+
// optional timezone) to stay portable across Postgres and
|
|
399
|
+
// Redshift.
|
|
409
400
|
const sessionStart = performance.now();
|
|
410
401
|
client
|
|
411
402
|
.query(`SET statement_timeout = ${statementTimeoutMs}`)
|
|
@@ -433,8 +424,8 @@ class PostgresClient extends WarehouseBaseClient_1.default {
|
|
|
433
424
|
throw this.parseError(error, sql);
|
|
434
425
|
})
|
|
435
426
|
.finally(async () => {
|
|
436
|
-
if (
|
|
437
|
-
clearTimeout(
|
|
427
|
+
if (clientTimeout) {
|
|
428
|
+
clearTimeout(clientTimeout);
|
|
438
429
|
}
|
|
439
430
|
// Release the client first, then end the pool
|
|
440
431
|
if (closeClient) {
|
|
@@ -211,6 +211,14 @@ describe('PostgresWarehouseClient statement timeout', () => {
|
|
|
211
211
|
.find((arg) => typeof arg === 'string');
|
|
212
212
|
expect(sessionStatement).toContain('SET statement_timeout = 540000');
|
|
213
213
|
});
|
|
214
|
+
it('omits the competing pg query_timeout from the pool', async () => {
|
|
215
|
+
const queryMock = respondingQueryMock();
|
|
216
|
+
mockPoolWithQuery(queryMock);
|
|
217
|
+
const warehouse = new PostgresWarehouseClient_1.PostgresWarehouseClient(PostgresWarehouseClient_mock_1.credentials);
|
|
218
|
+
await warehouse.runQuery('select 1');
|
|
219
|
+
const poolConfig = pg.Pool.mock.lastCall?.[0];
|
|
220
|
+
expect(poolConfig).not.toHaveProperty('query_timeout');
|
|
221
|
+
});
|
|
214
222
|
it('honors a configured timeoutSeconds for the statement_timeout', async () => {
|
|
215
223
|
const queryMock = respondingQueryMock();
|
|
216
224
|
mockPoolWithQuery(queryMock);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.243.0",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"snowflake-sdk": "2.3.4",
|
|
29
29
|
"ssh2": "1.14.0",
|
|
30
30
|
"trino-client": "0.2.9",
|
|
31
|
-
"@lightdash/common": "1.
|
|
31
|
+
"@lightdash/common": "1.243.0"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"oxlint": "1.75.0",
|