@lightdash/warehouses 0.3404.0 → 0.3405.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/DatabricksWarehouseClient.js +6 -2
- package/dist/warehouseClients/DatabricksWarehouseClient.test.js +9 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.d.ts.map +1 -1
- package/dist/warehouseClients/SnowflakeWarehouseClient.js +7 -0
- package/dist/warehouseClients/SnowflakeWarehouseClient.test.js +8 -0
- package/package.json +2 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DatabricksWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DatabricksWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAqB,EACjB,iBAAiB,EACpB,MAAM,6CAA6C,CAAC;AAIrD,OAAO,EACH,OAAO,EACP,2BAA2B,EAI3B,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAIhB,gBAAgB,EAChB,cAAc,EACjB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE;;;;;;GAMG;AACH,eAAO,MAAM,kCAAkC,mBAAmB,CAAC;AAQnE,eAAO,MAAM,4BAA4B,GACrC,UAAU,MAAM,GAAG,SAAS,KAC7B,OAAsE,CAAC;AAgJ1E,qBAAa,oBAAqB,SAAQ,uBAAuB;IAC7D,QAAQ,CAAC,IAAI,6BAA6B;IAE1C,cAAc,IAAI,mBAAmB;IAIrC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,iBAAiB,IAAI,MAAM;IAI3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmBnC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM;IAM7D,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM;IAKT,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAKtC,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAKtC,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM;CAQ9D;
|
|
1
|
+
{"version":3,"file":"DatabricksWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DatabricksWarehouseClient.ts"],"names":[],"mappings":"AACA,OAAqB,EACjB,iBAAiB,EACpB,MAAM,6CAA6C,CAAC;AAIrD,OAAO,EACH,OAAO,EACP,2BAA2B,EAI3B,MAAM,EAGN,mBAAmB,EACnB,gBAAgB,EAIhB,gBAAgB,EAChB,cAAc,EACjB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAEhE;;;;;;GAMG;AACH,eAAO,MAAM,kCAAkC,mBAAmB,CAAC;AAQnE,eAAO,MAAM,4BAA4B,GACrC,UAAU,MAAM,GAAG,SAAS,KAC7B,OAAsE,CAAC;AAgJ1E,qBAAa,oBAAqB,SAAQ,uBAAuB;IAC7D,QAAQ,CAAC,IAAI,6BAA6B;IAE1C,cAAc,IAAI,mBAAmB;IAIrC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAWjD,iBAAiB,IAAI,MAAM;IAI3B,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmBnC,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM;IAM7D,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM;IAKT,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAKtC,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAKtC,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM;CAQ9D;AAQD,qBAAa,yBAA0B,SAAQ,mBAAmB,CAAC,2BAA2B,CAAC;IAC3F,MAAM,EAAE,MAAM,CAAC;IAEf,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,iBAAiB,EAAE,iBAAiB,CAAC;IAErC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;gBAE7B,WAAW,EAAE,2BAA2B;YAiDtC,UAAU;IAqClB,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;IA0EV,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE;IAmED,YAAY;;;;;IAeZ,SAAS,CACX,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC;IAkBtB,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC;CAsB/B;AAED;;GAEG;AACH,eAAO,MAAM,kCAAkC,GAC3C,MAAM,MAAM,EACZ,UAAU,MAAM,EAChB,cAAc,MAAM,KACrB,OAAO,CAAC;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAE,CA+BxD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,2BAA2B,GACpC,MAAM,MAAM,EACZ,UAAU,MAAM,EAChB,cAAc,MAAM,EACpB,eAAe,MAAM,KACtB,OAAO,CAAC;IACP,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;CACrB,CAqCA,CAAC"}
|
|
@@ -198,6 +198,9 @@ class DatabricksSqlBuilder extends WarehouseBaseSqlBuilder_1.default {
|
|
|
198
198
|
exports.DatabricksSqlBuilder = DatabricksSqlBuilder;
|
|
199
199
|
const DATABRICKS_SOCKET_TIMEOUT_MS = 60000;
|
|
200
200
|
const DATABRICKS_QUERY_TIMEOUT_SECONDS = 300;
|
|
201
|
+
// @databricks/sql defaults fetchChunk to 100k rows, which materializes entire
|
|
202
|
+
// wide results in one array and can OOM the worker on large queries
|
|
203
|
+
const DATABRICKS_FETCH_CHUNK_MAX_ROWS = 5000;
|
|
201
204
|
class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
202
205
|
constructor(credentials) {
|
|
203
206
|
super(credentials, new DatabricksSqlBuilder(credentials.startOfWeek));
|
|
@@ -308,8 +311,9 @@ class DatabricksWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
308
311
|
}), {});
|
|
309
312
|
do {
|
|
310
313
|
// eslint-disable-next-line no-await-in-loop
|
|
311
|
-
const chunk = await query.fetchChunk(
|
|
312
|
-
|
|
314
|
+
const chunk = await query.fetchChunk({
|
|
315
|
+
maxRows: DATABRICKS_FETCH_CHUNK_MAX_ROWS,
|
|
316
|
+
});
|
|
313
317
|
// eslint-disable-next-line no-await-in-loop
|
|
314
318
|
await streamCallback({ fields, rows: chunk });
|
|
315
319
|
// eslint-disable-next-line no-await-in-loop
|
|
@@ -4,6 +4,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
4
4
|
const DatabricksWarehouseClient_1 = require("./DatabricksWarehouseClient");
|
|
5
5
|
const DatabricksWarehouseClient_mock_1 = require("./DatabricksWarehouseClient.mock");
|
|
6
6
|
const WarehouseClient_mock_1 = require("./WarehouseClient.mock");
|
|
7
|
+
const mocks = vi.hoisted(() => ({
|
|
8
|
+
fetchChunk: vi.fn(),
|
|
9
|
+
}));
|
|
7
10
|
vi.mock('@databricks/sql', async () => ({
|
|
8
11
|
...(await vi.importActual('@databricks/sql')),
|
|
9
12
|
DBSQLClient: vi.fn(function () {
|
|
@@ -12,7 +15,7 @@ vi.mock('@databricks/sql', async () => ({
|
|
|
12
15
|
openSession: vi.fn(() => ({
|
|
13
16
|
executeStatement: vi.fn(() => ({
|
|
14
17
|
getSchema: vi.fn(async () => DatabricksWarehouseClient_mock_1.schema),
|
|
15
|
-
fetchChunk:
|
|
18
|
+
fetchChunk: mocks.fetchChunk.mockImplementation(async () => DatabricksWarehouseClient_mock_1.rows),
|
|
16
19
|
hasMoreRows: vi.fn(async () => false),
|
|
17
20
|
close: vi.fn(async () => undefined),
|
|
18
21
|
})),
|
|
@@ -30,6 +33,11 @@ describe('DatabricksWarehouseClient', () => {
|
|
|
30
33
|
expect(results.fields).toEqual(WarehouseClient_mock_1.expectedFields);
|
|
31
34
|
expect(results.rows[0]).toEqual(DatabricksWarehouseClient_mock_1.rows[0]);
|
|
32
35
|
});
|
|
36
|
+
it('caps fetchChunk size to avoid materializing whole results', async () => {
|
|
37
|
+
const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
|
|
38
|
+
await warehouse.runQuery('fake sql');
|
|
39
|
+
expect(mocks.fetchChunk).toHaveBeenCalledWith({ maxRows: 5000 });
|
|
40
|
+
});
|
|
33
41
|
});
|
|
34
42
|
describe('DatabricksSqlBuilder escaping', () => {
|
|
35
43
|
const databricksSqlBuilder = new DatabricksWarehouseClient_1.DatabricksSqlBuilder();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SnowflakeWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/SnowflakeWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,0BAA0B,EAC1B,aAAa,EAGb,MAAM,EAIN,mBAAmB,EAGnB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,0BAA0B,EAC/B,KAAK,8BAA8B,EACtC,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAGH,UAAU,EACV,iBAAiB,EAEjB,cAAc,EAGjB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAchE,oBAAY,cAAc;IACtB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,gBAAgB,qBAAqB;IACrC,IAAI,SAAS;IACb,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,aAAa,kBAAkB;IAC/B,aAAa,kBAAkB;IAC/B,YAAY,iBAAiB;IAC7B,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,KAAK,UAAU;IACf,SAAS,cAAc;CAC1B;
|
|
1
|
+
{"version":3,"file":"SnowflakeWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/SnowflakeWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,0BAA0B,EAC1B,aAAa,EAGb,MAAM,EAIN,mBAAmB,EAGnB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,0BAA0B,EAC/B,KAAK,8BAA8B,EACtC,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAGH,UAAU,EACV,iBAAiB,EAEjB,cAAc,EAGjB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAM5C,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAchE,oBAAY,cAAc;IACtB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,GAAG,QAAQ;IACX,MAAM,WAAW;IACjB,QAAQ,aAAa;IACrB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,gBAAgB,qBAAqB;IACrC,IAAI,SAAS;IACb,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,IAAI,SAAS;IACb,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,aAAa,kBAAkB;IAC/B,aAAa,kBAAkB;IAC/B,YAAY,iBAAiB;IAC7B,OAAO,YAAY;IACnB,MAAM,WAAW;IACjB,KAAK,UAAU;IACf,SAAS,cAAc;CAC1B;AAqBD,MAAM,MAAM,oBAAoB,GAAG;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;CAC1B,CAAC;AAqFF,MAAM,MAAM,wBAAwB,GAAG;IACnC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,SAAS,EAAE;QACP,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;KACvB,EAAE,CAAC;IACJ,UAAU,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;KACrC,EAAE,CAAC;IACJ,KAAK,EAAE;QACH,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,OAAO,CAAC;KACtB,EAAE,CAAC;IACJ,OAAO,EAAE;QACL,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;KAChB,EAAE,CAAC;CACP,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,wBAAwB,CAAC;IACnC,SAAS,EAAE,yBAAyB,CAAC;CACxC,CAAC;AAEF,MAAM,MAAM,gCAAgC,GACtC,oBAAoB,GACpB,gBAAgB,GAChB,aAAa,GACb,iBAAiB,GACjB,kBAAkB,GAClB,gBAAgB,GAChB,SAAS,CAAC;AAEhB,MAAM,MAAM,+BAA+B,GAAG;IAC1C,QAAQ,EAAE,gCAAgC,CAAC;IAC3C,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAiGF,eAAO,MAAM,mBAAmB,GAAI,OAAO,MAAM,KAAG,MAGV,CAAC;AAE3C,eAAO,MAAM,2BAA2B,GACpC,OAAO,OAAO,KACf,+BA+DF,CAAC;AAEF,qBAAa,wBAAyB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,OAAO,EAAE,+BAA+B,CAAC;IAElD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;gBAExB,KAAK,EAAE,OAAO;IAa1B,WAAW,IAAI,OAAO;CAGzB;AAED,MAAM,MAAM,0BAA0B,GAAG;IACrC,UAAU,EAAE,UAAU,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG,gBAAgB,GAAG,kBAAkB,CAAC;AAE3E,MAAM,MAAM,uBAAuB,GAAG;IAClC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,CAAC;AAEF,eAAO,MAAM,YAAY,GAAI,MAAM,MAAM,KAAG,aA+B3C,CAAC;AAgBF,qBAAa,mBAAoB,SAAQ,uBAAuB;IAC5D,QAAQ,CAAC,IAAI,4BAA4B;IAEzC,cAAc,IAAI,mBAAmB;IAIrC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAajD,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAmBnC,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM;IAKT,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAKtC,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM;CAO9D;AAED,qBAAa,wBAAyB,SAAQ,mBAAmB,CAAC,0BAA0B,CAAC;IACzF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAEpD,iBAAiB,EAAE,iBAAiB,CAAC;IAErC,2BAA2B,CAAC,EAAE,OAAO,CAAC;IAEtC,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM;IAO3D,OAAO,CAAC,4BAA4B,CAAC,CAAsB;IAE3D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAEhD,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAqB;IAE1D,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAkC;gBAE9D,WAAW,EAAE,0BAA0B;YAkHrC,aAAa;IAU3B,OAAO,CAAC,0BAA0B;YASpB,oBAAoB;YAwBpB,2BAA2B;YA+C3B,gBAAgB;IAoBxB,wBAAwB,IAAI,OAAO,CAAC,0BAA0B,CAAC;IAY/D,6BAA6B,CAC/B,OAAO,EAAE,0BAA0B,GACpC,OAAO,CAAC,IAAI,CAAC;IAQV,yBAAyB,CAC3B,OAAO,EAAE,0BAA0B,GACpC,OAAO,CAAC,IAAI,CAAC;IAOV,mBAAmB,CACrB,IAAI,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,yBAAyB,CAAC;IA6E/B,qBAAqB,CACvB,IAAI,EAAE,MAAM,GACb,OAAO,CAAC,uBAAuB,CAAC;IA6B7B,gBAAgB,CAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,sBAAsB,EAC5B,SAAS,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC;IAaV,kBAAkB,CACpB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,sBAAsB,GAC7B,OAAO,CAAC,IAAI,CAAC;IAQhB,cAAc,IAAI,oBAAoB,GAAG,IAAI;IAIvC,6BAA6B,CAC/B,SAAS,GAAE,MAAsC,EACjD,YAAY,GAAE,MAAU,EACxB,yBAAyB,GAAE,MAAa,EACxC,eAAe,GAAE,MAAM,GAAG,IAAW,GACtC,OAAO,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;YAoGxC,gBAAgB;IAmG9B,OAAO,CAAC,sBAAsB;YAkBhB,iBAAiB;IAuBzB,iBAAiB,CACnB,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,8BAA8B,EAC/D,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAC1B,OAAO,CAAC,0BAA0B,CAAC;YA2CxB,qBAAqB;IA2I7B,WAAW,CACb,GAAG,EAAE,MAAM,EACX,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC;YAuBF,sBAAsB;YA6DtB,iBAAiB;YAmCjB,oBAAoB;IA0B5B,UAAU,CACZ,MAAM,EAAE;QACJ,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE;IAuCD,YAAY;;;;;IA8BZ,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC;IAsC5B,OAAO,CAAC,wBAAwB;IA0ChC,OAAO,CAAC,2BAA2B;IAcnC,UAAU,CAAC,KAAK,EAAE,cAAc,EAAE,KAAK,GAAE,MAAW;CA8FvD"}
|
|
@@ -96,6 +96,8 @@ const EXTERNAL_BROWSER_AUTHENTICATOR = 'EXTERNALBROWSER';
|
|
|
96
96
|
const OAUTH_AUTHORIZATION_CODE_AUTHENTICATOR = 'OAUTH_AUTHORIZATION_CODE';
|
|
97
97
|
const SESSION_DISCOVERY_LIMIT = 100;
|
|
98
98
|
const SESSION_SCHEMA_DISCOVERY_LIMIT = 1000;
|
|
99
|
+
// Minimum allowed by Snowflake (range 16-160, default 160)
|
|
100
|
+
const SNOWFLAKE_RESULT_CHUNK_SIZE_MB = 16;
|
|
99
101
|
const getAccessTokenExpiresAt = (token) => {
|
|
100
102
|
try {
|
|
101
103
|
const payload = token.split('.')[1];
|
|
@@ -798,6 +800,11 @@ class SnowflakeWarehouseClient extends WarehouseBaseClient_1.default {
|
|
|
798
800
|
const timeoutSeconds = this.credentials.timeoutSeconds ?? 300;
|
|
799
801
|
console.debug(`Setting Snowflake session STATEMENT_TIMEOUT_IN_SECONDS = ${timeoutSeconds}`);
|
|
800
802
|
sessionParams.push(`STATEMENT_TIMEOUT_IN_SECONDS = ${timeoutSeconds}`);
|
|
803
|
+
// Snowflake grows result chunks up to 160MB and the driver holds ~4
|
|
804
|
+
// chunks in memory regardless of consumer backpressure, so uncapped
|
|
805
|
+
// chunks make heap usage scale with the query row limit and can OOM
|
|
806
|
+
// the worker on large results
|
|
807
|
+
sessionParams.push(`CLIENT_RESULT_CHUNK_SIZE = ${SNOWFLAKE_RESULT_CHUNK_SIZE_MB}`);
|
|
801
808
|
await this.executeStatements(connection, `ALTER SESSION SET ${sessionParams.join(', ')};`);
|
|
802
809
|
}
|
|
803
810
|
getFieldsFromStatement(stmt) {
|
|
@@ -63,6 +63,14 @@ describe('SnowflakeWarehouseClient', () => {
|
|
|
63
63
|
beforeEach(() => {
|
|
64
64
|
vi.clearAllMocks();
|
|
65
65
|
});
|
|
66
|
+
it('caps result chunk size via CLIENT_RESULT_CHUNK_SIZE session parameter', async () => {
|
|
67
|
+
const warehouse = new SnowflakeWarehouseClient_1.SnowflakeWarehouseClient(SnowflakeWarehouseClient_mock_1.credentials);
|
|
68
|
+
await warehouse.streamQuery('SELECT 1', () => { }, {});
|
|
69
|
+
const alterSessionSql = executeMock.mock.calls
|
|
70
|
+
.map((call) => call[0].sqlText)
|
|
71
|
+
.find((sql) => sql.startsWith('ALTER SESSION SET'));
|
|
72
|
+
expect(alterSessionSql).toContain('CLIENT_RESULT_CHUNK_SIZE = 16');
|
|
73
|
+
});
|
|
66
74
|
it('configures the local application authorization-code flow with SDK defaults', () => {
|
|
67
75
|
const warehouse = new SnowflakeWarehouseClient_1.SnowflakeWarehouseClient({
|
|
68
76
|
...SnowflakeWarehouseClient_mock_1.credentials,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/warehouses",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3405.1",
|
|
4
4
|
"description": "Warehouse connectors for Lightdash",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
"snowflake-sdk": "2.3.4",
|
|
31
31
|
"ssh2": "1.14.0",
|
|
32
32
|
"trino-client": "0.2.9",
|
|
33
|
-
"@lightdash/common": "0.
|
|
33
|
+
"@lightdash/common": "0.3405.1"
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
36
|
"@types/big.js": "6.2.2",
|