@lightdash/warehouses 2.228.1 → 2.229.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/dist/.tsbuildinfo.build +1 -0
  2. package/package.json +4 -4
  3. package/dist/.tsbuildinfo +0 -1
  4. package/dist/ssh/sshTunnelFailure.test.d.ts +0 -2
  5. package/dist/ssh/sshTunnelFailure.test.d.ts.map +0 -1
  6. package/dist/ssh/sshTunnelFailure.test.js +0 -90
  7. package/dist/validateWarehouseColumnReferences.test.d.ts +0 -2
  8. package/dist/validateWarehouseColumnReferences.test.d.ts.map +0 -1
  9. package/dist/validateWarehouseColumnReferences.test.js +0 -168
  10. package/dist/warehouseClients/AthenaWarehouseClient.test.d.ts +0 -2
  11. package/dist/warehouseClients/AthenaWarehouseClient.test.d.ts.map +0 -1
  12. package/dist/warehouseClients/AthenaWarehouseClient.test.js +0 -261
  13. package/dist/warehouseClients/BigqueryWarehouseClient.mock.d.ts +0 -83
  14. package/dist/warehouseClients/BigqueryWarehouseClient.mock.d.ts.map +0 -1
  15. package/dist/warehouseClients/BigqueryWarehouseClient.mock.js +0 -130
  16. package/dist/warehouseClients/BigqueryWarehouseClient.test.d.ts +0 -2
  17. package/dist/warehouseClients/BigqueryWarehouseClient.test.d.ts.map +0 -1
  18. package/dist/warehouseClients/BigqueryWarehouseClient.test.js +0 -493
  19. package/dist/warehouseClients/ClickhouseWarehouseClient.test.d.ts +0 -2
  20. package/dist/warehouseClients/ClickhouseWarehouseClient.test.d.ts.map +0 -1
  21. package/dist/warehouseClients/ClickhouseWarehouseClient.test.js +0 -115
  22. package/dist/warehouseClients/DatabricksWarehouseClient.mock.d.ts +0 -17
  23. package/dist/warehouseClients/DatabricksWarehouseClient.mock.d.ts.map +0 -1
  24. package/dist/warehouseClients/DatabricksWarehouseClient.mock.js +0 -111
  25. package/dist/warehouseClients/DatabricksWarehouseClient.test.d.ts +0 -2
  26. package/dist/warehouseClients/DatabricksWarehouseClient.test.d.ts.map +0 -1
  27. package/dist/warehouseClients/DatabricksWarehouseClient.test.js +0 -592
  28. package/dist/warehouseClients/DatabricksWarehouseStartupRetry.test.d.ts +0 -2
  29. package/dist/warehouseClients/DatabricksWarehouseStartupRetry.test.d.ts.map +0 -1
  30. package/dist/warehouseClients/DatabricksWarehouseStartupRetry.test.js +0 -129
  31. package/dist/warehouseClients/DuckdbWarehouseClient.embedded.test.d.ts +0 -2
  32. package/dist/warehouseClients/DuckdbWarehouseClient.embedded.test.d.ts.map +0 -1
  33. package/dist/warehouseClients/DuckdbWarehouseClient.embedded.test.js +0 -92
  34. package/dist/warehouseClients/DuckdbWarehouseClient.test.d.ts +0 -2
  35. package/dist/warehouseClients/DuckdbWarehouseClient.test.d.ts.map +0 -1
  36. package/dist/warehouseClients/DuckdbWarehouseClient.test.js +0 -1749
  37. package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts +0 -2
  38. package/dist/warehouseClients/MotherduckErrorClassifier.test.d.ts.map +0 -1
  39. package/dist/warehouseClients/MotherduckErrorClassifier.test.js +0 -31
  40. package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts +0 -2
  41. package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.d.ts.map +0 -1
  42. package/dist/warehouseClients/MotherduckInstanceCache.catalog.test.js +0 -52
  43. package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts +0 -2
  44. package/dist/warehouseClients/MotherduckInstanceCache.integration.test.d.ts.map +0 -1
  45. package/dist/warehouseClients/MotherduckInstanceCache.integration.test.js +0 -172
  46. package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts +0 -2
  47. package/dist/warehouseClients/MotherduckInstanceCache.security.test.d.ts.map +0 -1
  48. package/dist/warehouseClients/MotherduckInstanceCache.security.test.js +0 -192
  49. package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts +0 -2
  50. package/dist/warehouseClients/MotherduckInstanceCache.test.d.ts.map +0 -1
  51. package/dist/warehouseClients/MotherduckInstanceCache.test.js +0 -302
  52. package/dist/warehouseClients/PostgresWarehouseClient.mock.d.ts +0 -8
  53. package/dist/warehouseClients/PostgresWarehouseClient.mock.d.ts.map +0 -1
  54. package/dist/warehouseClients/PostgresWarehouseClient.mock.js +0 -66
  55. package/dist/warehouseClients/PostgresWarehouseClient.test.d.ts +0 -2
  56. package/dist/warehouseClients/PostgresWarehouseClient.test.d.ts.map +0 -1
  57. package/dist/warehouseClients/PostgresWarehouseClient.test.js +0 -371
  58. package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts +0 -2
  59. package/dist/warehouseClients/RedshiftWarehouseClient.test.d.ts.map +0 -1
  60. package/dist/warehouseClients/RedshiftWarehouseClient.test.js +0 -126
  61. package/dist/warehouseClients/SnowflakeWarehouseClient.mock.d.ts +0 -41
  62. package/dist/warehouseClients/SnowflakeWarehouseClient.mock.d.ts.map +0 -1
  63. package/dist/warehouseClients/SnowflakeWarehouseClient.mock.js +0 -184
  64. package/dist/warehouseClients/SnowflakeWarehouseClient.test.d.ts +0 -2
  65. package/dist/warehouseClients/SnowflakeWarehouseClient.test.d.ts.map +0 -1
  66. package/dist/warehouseClients/SnowflakeWarehouseClient.test.js +0 -777
  67. package/dist/warehouseClients/TrinoWarehouseClient.mock.d.ts +0 -23
  68. package/dist/warehouseClients/TrinoWarehouseClient.mock.d.ts.map +0 -1
  69. package/dist/warehouseClients/TrinoWarehouseClient.mock.js +0 -102
  70. package/dist/warehouseClients/TrinoWarehouseClient.test.d.ts +0 -2
  71. package/dist/warehouseClients/TrinoWarehouseClient.test.d.ts.map +0 -1
  72. package/dist/warehouseClients/TrinoWarehouseClient.test.js +0 -230
  73. package/dist/warehouseClients/WarehouseClient.mock.d.ts +0 -13
  74. package/dist/warehouseClients/WarehouseClient.mock.d.ts.map +0 -1
  75. package/dist/warehouseClients/WarehouseClient.mock.js +0 -51
  76. package/dist/warehouseClients/numericKinds.test.d.ts +0 -2
  77. package/dist/warehouseClients/numericKinds.test.d.ts.map +0 -1
  78. package/dist/warehouseClients/numericKinds.test.js +0 -168
  79. package/dist/warehouseClients/pgProtocolGuard.integration.test.d.ts +0 -2
  80. package/dist/warehouseClients/pgProtocolGuard.integration.test.d.ts.map +0 -1
  81. package/dist/warehouseClients/pgProtocolGuard.integration.test.js +0 -93
  82. package/dist/warehouseClients/pgProtocolGuard.test.d.ts +0 -2
  83. package/dist/warehouseClients/pgProtocolGuard.test.d.ts.map +0 -1
  84. package/dist/warehouseClients/pgProtocolGuard.test.js +0 -48
  85. package/dist/warehouseClients/redshiftIamCredentials.test.d.ts +0 -2
  86. package/dist/warehouseClients/redshiftIamCredentials.test.d.ts.map +0 -1
  87. package/dist/warehouseClients/redshiftIamCredentials.test.js +0 -210
  88. package/dist/warehouseClients/timestampDomain.test.d.ts +0 -2
  89. package/dist/warehouseClients/timestampDomain.test.d.ts.map +0 -1
  90. package/dist/warehouseClients/timestampDomain.test.js +0 -101
@@ -1,592 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- /* eslint-disable prefer-arrow-callback, func-names */
7
- const StatusError_1 = __importDefault(require("@databricks/sql/dist/errors/StatusError"));
8
- const TCLIService_types_1 = require("@databricks/sql/thrift/TCLIService_types");
9
- const common_1 = require("@lightdash/common");
10
- const DatabricksWarehouseClient_1 = require("./DatabricksWarehouseClient");
11
- const DatabricksWarehouseClient_mock_1 = require("./DatabricksWarehouseClient.mock");
12
- const WarehouseClient_mock_1 = require("./WarehouseClient.mock");
13
- const mocks = vi.hoisted(() => ({
14
- fetchChunk: vi.fn(),
15
- openSession: vi.fn(),
16
- closeConnection: vi.fn(),
17
- }));
18
- vi.mock('@databricks/sql', async () => ({
19
- ...(await vi.importActual('@databricks/sql')),
20
- DBSQLClient: vi.fn(function () {
21
- return {
22
- connect: vi.fn(() => ({
23
- openSession: mocks.openSession,
24
- close: mocks.closeConnection,
25
- })),
26
- };
27
- }),
28
- }));
29
- const statusError = (errorMessage) => new StatusError_1.default({ statusCode: TCLIService_types_1.TStatusCode.ERROR_STATUS, errorMessage });
30
- const sessionLostError = () => statusError('requirement failed: Session handle: SessionHandle [01f19fd1-ac81-1e09-bb7c-357018ed26f1] has not been initialized or had already closed.');
31
- const sessionLostMessage = sessionLostError().message;
32
- const createOperation = (overrides = {}) => ({
33
- getSchema: vi.fn(async () => DatabricksWarehouseClient_mock_1.schema),
34
- fetchChunk: mocks.fetchChunk,
35
- fetchAll: vi.fn(async () => []),
36
- hasMoreRows: vi.fn(async () => false),
37
- close: vi.fn(async () => undefined),
38
- ...overrides,
39
- });
40
- const createSession = (overrides = {}) => ({
41
- executeStatement: vi.fn(async () => createOperation()),
42
- getColumns: vi.fn(async () => createOperation()),
43
- close: vi.fn(async () => undefined),
44
- ...overrides,
45
- });
46
- // Retries sleep with real backoff; drive the timers while the promise settles.
47
- const withTimers = async (run) => {
48
- const settled = run().then((value) => ({ ok: true, value }), (error) => ({ ok: false, error }));
49
- await vi.runAllTimersAsync();
50
- const result = await settled;
51
- if (result.ok)
52
- return result.value;
53
- throw result.error;
54
- };
55
- const tableRequest = (table) => ({
56
- database: 'database',
57
- schema: 'schema',
58
- table,
59
- });
60
- describe('DatabricksWarehouseClient', () => {
61
- beforeEach(() => {
62
- vi.useFakeTimers();
63
- vi.spyOn(console, 'warn').mockImplementation(() => { });
64
- mocks.fetchChunk.mockReset().mockResolvedValue(DatabricksWarehouseClient_mock_1.rows);
65
- mocks.closeConnection.mockReset().mockResolvedValue(undefined);
66
- mocks.openSession.mockReset().mockResolvedValue(createSession());
67
- });
68
- afterEach(() => {
69
- vi.useRealTimers();
70
- vi.restoreAllMocks();
71
- });
72
- it('surfaces Databricks status messages when opening a session fails', async () => {
73
- const message = 'PERMISSION_DENIED: User does not have USE CATALOG on Catalog';
74
- mocks.openSession.mockRejectedValueOnce(statusError(message));
75
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
76
- await expect(warehouse.runQuery('fake sql')).rejects.toMatchObject({
77
- message,
78
- });
79
- expect(mocks.openSession).toHaveBeenCalledOnce();
80
- });
81
- it('retries opening a session while the warehouse is starting, then surfaces the error', async () => {
82
- const message = 'SQL warehouse xyz is not ready to accept connections (current state: STARTING)';
83
- mocks.openSession.mockRejectedValue(statusError(message));
84
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
85
- await expect(withTimers(() => warehouse.runQuery('fake sql'))).rejects.toMatchObject({ message });
86
- // 2+4+8+16 then 30s waits fit 23 retries in the 10 minute deadline
87
- expect(mocks.openSession).toHaveBeenCalledTimes(24);
88
- });
89
- it('expect query fields and rows', async () => {
90
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
91
- const results = await warehouse.runQuery('fake sql');
92
- expect(results.fields).toEqual({
93
- ...WarehouseClient_mock_1.expectedFields,
94
- myNumberColumn: {
95
- ...WarehouseClient_mock_1.expectedFields.myNumberColumn,
96
- numericKind: { kind: 'integer' },
97
- },
98
- });
99
- expect(results.rows[0]).toEqual(DatabricksWarehouseClient_mock_1.rows[0]);
100
- });
101
- it('caps fetchChunk size to avoid materializing whole results', async () => {
102
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
103
- await warehouse.runQuery('fake sql');
104
- expect(mocks.fetchChunk).toHaveBeenCalledWith({ maxRows: 5000 });
105
- });
106
- describe('session lost before rows are streamed', () => {
107
- it('reopens the session and re-runs the query', async () => {
108
- const firstSession = createSession({
109
- executeStatement: vi.fn(() => Promise.reject(sessionLostError())),
110
- });
111
- mocks.openSession
112
- .mockResolvedValueOnce(firstSession)
113
- .mockResolvedValueOnce(createSession());
114
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
115
- const results = await withTimers(() => warehouse.runQuery('fake sql'));
116
- expect(results.rows).toEqual(DatabricksWarehouseClient_mock_1.rows);
117
- expect(mocks.openSession).toHaveBeenCalledTimes(2);
118
- expect(firstSession.close).toHaveBeenCalledOnce();
119
- });
120
- it('re-runs the query when only an empty chunk was emitted', async () => {
121
- mocks.openSession
122
- .mockResolvedValueOnce(createSession({
123
- executeStatement: vi.fn(async () => createOperation({
124
- fetchChunk: vi.fn(async () => []),
125
- hasMoreRows: vi.fn(() => Promise.reject(sessionLostError())),
126
- })),
127
- }))
128
- .mockResolvedValueOnce(createSession());
129
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
130
- const results = await withTimers(() => warehouse.runQuery('fake sql'));
131
- expect(results.rows).toEqual(DatabricksWarehouseClient_mock_1.rows);
132
- expect(mocks.openSession).toHaveBeenCalledTimes(2);
133
- });
134
- it('gives up once the retry budget is spent', async () => {
135
- mocks.openSession.mockResolvedValue(createSession({
136
- executeStatement: vi.fn(() => Promise.reject(sessionLostError())),
137
- }));
138
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
139
- await expect(withTimers(() => warehouse.runQuery('fake sql'))).rejects.toMatchObject({ message: sessionLostMessage });
140
- expect(mocks.openSession).toHaveBeenCalledTimes(24);
141
- });
142
- });
143
- it('does not re-run the query once rows have been streamed', async () => {
144
- const streamCallback = vi.fn();
145
- mocks.openSession.mockResolvedValueOnce(createSession({
146
- executeStatement: vi.fn(async () => createOperation({
147
- hasMoreRows: vi.fn(() => Promise.reject(sessionLostError())),
148
- })),
149
- }));
150
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
151
- await expect(withTimers(() => warehouse.streamQuery('fake sql', streamCallback, {}))).rejects.toMatchObject({ message: sessionLostMessage });
152
- expect(streamCallback).toHaveBeenCalledOnce();
153
- expect(mocks.openSession).toHaveBeenCalledOnce();
154
- });
155
- it('does not retry SQL errors', async () => {
156
- mocks.openSession.mockResolvedValueOnce(createSession({
157
- executeStatement: vi.fn(() => Promise.reject(statusError('Syntax error near FROM'))),
158
- }));
159
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
160
- await expect(withTimers(() => warehouse.runQuery('fake sql'))).rejects.toMatchObject({ message: 'Syntax error near FROM' });
161
- expect(mocks.openSession).toHaveBeenCalledOnce();
162
- });
163
- describe('getCatalog', () => {
164
- const columns = (name, type) => [
165
- { COLUMN_NAME: name, TYPE_NAME: type },
166
- ];
167
- const describedTable = (sql) => /`([^`]+)` AS JSON$/.exec(sql)?.[1] ?? '';
168
- const jsonDescription = (cols) => [
169
- {
170
- json_metadata: JSON.stringify({
171
- columns: cols.map((col) => ({
172
- name: col.COLUMN_NAME,
173
- type: { name: col.TYPE_NAME.toLowerCase() },
174
- })),
175
- }),
176
- },
177
- ];
178
- const columnsSession = (resolve, lostTables) => createSession({
179
- executeStatement: vi.fn(async (sql) => {
180
- const table = describedTable(sql);
181
- return lostTables.has(table)
182
- ? Promise.reject(sessionLostError())
183
- : createOperation({
184
- fetchAll: vi.fn(async () => jsonDescription(resolve(table))),
185
- });
186
- }),
187
- });
188
- // Captured from DESCRIBE TABLE EXTENDED ... AS JSON on a serverless warehouse.
189
- const describedTransactions = {
190
- columns: [
191
- {
192
- name: 'order_id',
193
- type: { name: 'string', collation: 'UTF8_BINARY' },
194
- nullable: true,
195
- },
196
- { name: 'amount', type: { name: 'double' }, nullable: true },
197
- {
198
- name: 'created_at',
199
- type: { name: 'timestamp_ltz' },
200
- nullable: true,
201
- },
202
- {
203
- name: 'updated_at',
204
- type: { name: 'timestamp_ntz' },
205
- nullable: true,
206
- },
207
- {
208
- name: 'customer',
209
- type: {
210
- name: 'struct',
211
- fields: [
212
- {
213
- name: 'id',
214
- type: { name: 'string' },
215
- nullable: true,
216
- },
217
- {
218
- name: 'location',
219
- type: { name: 'string' },
220
- nullable: true,
221
- },
222
- ],
223
- },
224
- nullable: true,
225
- },
226
- {
227
- name: 'product',
228
- type: {
229
- name: 'array',
230
- element_type: {
231
- name: 'struct',
232
- fields: [
233
- {
234
- name: 'sku',
235
- type: { name: 'string' },
236
- nullable: true,
237
- },
238
- {
239
- name: 'price',
240
- type: { name: 'double' },
241
- nullable: true,
242
- },
243
- {
244
- name: 'attributes',
245
- type: {
246
- name: 'struct',
247
- fields: [
248
- {
249
- name: 'colour',
250
- type: { name: 'string' },
251
- nullable: true,
252
- },
253
- ],
254
- },
255
- nullable: true,
256
- },
257
- {
258
- name: 'variants',
259
- type: {
260
- name: 'array',
261
- element_type: {
262
- name: 'struct',
263
- fields: [
264
- {
265
- name: 'size',
266
- type: { name: 'string' },
267
- nullable: true,
268
- },
269
- {
270
- name: 'stock',
271
- type: { name: 'int' },
272
- nullable: true,
273
- },
274
- ],
275
- },
276
- element_nullable: true,
277
- },
278
- nullable: true,
279
- },
280
- ],
281
- },
282
- element_nullable: true,
283
- },
284
- nullable: true,
285
- },
286
- {
287
- name: 'tags',
288
- type: {
289
- name: 'array',
290
- element_type: { name: 'string' },
291
- element_nullable: true,
292
- },
293
- nullable: true,
294
- },
295
- {
296
- name: 'scores',
297
- type: { name: 'array', element_type: { name: 'int' } },
298
- nullable: true,
299
- },
300
- {
301
- name: 'matrix',
302
- type: {
303
- name: 'array',
304
- element_type: {
305
- name: 'array',
306
- element_type: { name: 'int' },
307
- },
308
- },
309
- nullable: true,
310
- },
311
- {
312
- name: 'labels',
313
- type: {
314
- name: 'map',
315
- key_type: { name: 'string' },
316
- value_type: { name: 'string' },
317
- value_nullable: true,
318
- },
319
- nullable: true,
320
- },
321
- ],
322
- };
323
- const describeSession = (describe, flatColumns = []) => createSession({
324
- executeStatement: vi.fn(async (sql) => createOperation({ fetchAll: vi.fn(() => describe(sql)) })),
325
- getColumns: vi.fn(async () => createOperation({
326
- fetchAll: vi.fn(async () => flatColumns),
327
- })),
328
- });
329
- it('reads dotted paths and nested shapes from the JSON table description', async () => {
330
- const session = describeSession(async () => [
331
- { json_metadata: JSON.stringify(describedTransactions) },
332
- ]);
333
- mocks.openSession.mockResolvedValue(session);
334
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
335
- const catalog = await warehouse.getCatalog([
336
- tableRequest('transactions'),
337
- ]);
338
- const table = catalog[DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT'].schema.transactions;
339
- expect(session.executeStatement).toHaveBeenCalledWith('DESCRIBE TABLE EXTENDED `database`.`schema`.`transactions` AS JSON');
340
- expect(session.getColumns).not.toHaveBeenCalled();
341
- expect(table).toEqual({
342
- order_id: common_1.DimensionType.STRING,
343
- amount: common_1.DimensionType.NUMBER,
344
- created_at: common_1.DimensionType.TIMESTAMP,
345
- updated_at: common_1.DimensionType.TIMESTAMP,
346
- customer: common_1.DimensionType.STRING,
347
- 'customer.id': common_1.DimensionType.STRING,
348
- 'customer.location': common_1.DimensionType.STRING,
349
- product: common_1.DimensionType.STRING,
350
- 'product.sku': common_1.DimensionType.STRING,
351
- 'product.price': common_1.DimensionType.NUMBER,
352
- 'product.attributes': common_1.DimensionType.STRING,
353
- 'product.attributes.colour': common_1.DimensionType.STRING,
354
- 'product.variants': common_1.DimensionType.STRING,
355
- 'product.variants.size': common_1.DimensionType.STRING,
356
- 'product.variants.stock': common_1.DimensionType.NUMBER,
357
- tags: common_1.DimensionType.STRING,
358
- scores: common_1.DimensionType.NUMBER,
359
- matrix: common_1.DimensionType.STRING,
360
- labels: common_1.DimensionType.STRING,
361
- });
362
- const shape = (path) => (0, common_1.getCatalogNestedColumnShape)(catalog, DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT', 'schema', 'transactions', path);
363
- expect(shape('customer')).toEqual({
364
- repeated: false,
365
- record: true,
366
- });
367
- expect(shape('product')).toEqual({ repeated: true, record: true });
368
- expect(shape('product.attributes')).toEqual({
369
- repeated: false,
370
- record: true,
371
- });
372
- expect(shape('product.variants')).toEqual({
373
- repeated: true,
374
- record: true,
375
- });
376
- expect(shape('tags')).toEqual({ repeated: true, record: false });
377
- expect(shape('scores')).toEqual({ repeated: true, record: false });
378
- expect(shape('matrix')).toBeUndefined();
379
- expect(shape('labels')).toBeUndefined();
380
- expect(shape('product.sku')).toBeUndefined();
381
- const domain = (path) => (0, common_1.getCatalogTimestampDomain)(catalog, DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT', 'schema', 'transactions', path);
382
- expect(domain('created_at')).toEqual('aware');
383
- expect(domain('updated_at')).toEqual('naive');
384
- expect((0, common_1.getCatalogNestedColumnsUnavailableReason)(catalog, DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT', 'schema', 'transactions')).toBeUndefined();
385
- });
386
- it('keeps the flat column list and records why when the JSON description is refused', async () => {
387
- const session = describeSession(() => Promise.reject(statusError("[PARSE_SYNTAX_ERROR] Syntax error at or near 'JSON'. SQLSTATE: 42601")), [
388
- { COLUMN_NAME: 'order_id', TYPE_NAME: 'STRING' },
389
- {
390
- COLUMN_NAME: 'product',
391
- TYPE_NAME: 'ARRAY<STRUCT<sku: STRING>>',
392
- },
393
- ]);
394
- mocks.openSession.mockResolvedValue(session);
395
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
396
- const catalog = await warehouse.getCatalog([
397
- tableRequest('transactions'),
398
- ]);
399
- const database = DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT';
400
- expect(catalog[database].schema.transactions).toEqual({
401
- order_id: common_1.DimensionType.STRING,
402
- product: common_1.DimensionType.STRING,
403
- });
404
- expect((0, common_1.getCatalogNestedColumnShape)(catalog, database, 'schema', 'transactions', 'product')).toBeUndefined();
405
- expect((0, common_1.getCatalogNestedColumnsUnavailableReason)(catalog, database, 'schema', 'transactions')).toEqual("Databricks did not describe transactions as JSON ([PARSE_SYNTAX_ERROR] Syntax error at or near 'JSON'. SQLSTATE: 42601); nested columns need a SQL warehouse or Databricks Runtime 16.2 or newer.");
406
- });
407
- it('reports no columns, and no reason, for a table that does not exist', async () => {
408
- const session = describeSession(() => Promise.reject(statusError('[TABLE_OR_VIEW_NOT_FOUND] The table or view `database`.`schema`.`missing` cannot be found.')));
409
- mocks.openSession.mockResolvedValue(session);
410
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
411
- const catalog = await warehouse.getCatalog([
412
- tableRequest('missing'),
413
- ]);
414
- const database = DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT';
415
- expect(catalog[database].schema.missing).toEqual({});
416
- expect(session.getColumns).not.toHaveBeenCalled();
417
- expect((0, common_1.getCatalogNestedColumnsUnavailableReason)(catalog, database, 'schema', 'missing')).toBeUndefined();
418
- });
419
- it('fails the fetch on errors that are not the missing JSON form', async () => {
420
- const session = describeSession(() => Promise.reject(statusError('[INSUFFICIENT_PERMISSIONS] User does not have USE SCHEMA on Schema `schema`.')));
421
- mocks.openSession.mockResolvedValue(session);
422
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
423
- await expect(warehouse.getCatalog([tableRequest('transactions')])).rejects.toThrow('INSUFFICIENT_PERMISSIONS');
424
- expect(session.getColumns).not.toHaveBeenCalled();
425
- });
426
- it('fails the fetch when the JSON description cannot be read', async () => {
427
- const session = describeSession(async () => [
428
- { json_metadata: 'not json' },
429
- ]);
430
- mocks.openSession.mockResolvedValue(session);
431
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
432
- await expect(warehouse.getCatalog([tableRequest('transactions')])).rejects.toThrow('Could not read the description of');
433
- expect(session.getColumns).not.toHaveBeenCalled();
434
- });
435
- it('resumes the remaining tables on a replacement session', async () => {
436
- const firstSession = columnsSession(() => columns('id', 'BIGINT'), new Set(['table_two', 'table_three']));
437
- const secondSession = columnsSession(() => columns('name', 'STRING'), new Set());
438
- mocks.openSession
439
- .mockResolvedValueOnce(firstSession)
440
- .mockResolvedValueOnce(secondSession);
441
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
442
- const catalog = await withTimers(() => warehouse.getCatalog([
443
- tableRequest('table_one'),
444
- tableRequest('table_two'),
445
- tableRequest('table_three'),
446
- ]));
447
- expect(catalog.DEFAULT.schema).toEqual({
448
- table_one: { id: 'number' },
449
- table_two: { name: 'string' },
450
- table_three: { name: 'string' },
451
- });
452
- expect(firstSession.executeStatement).toHaveBeenCalledTimes(3);
453
- expect(firstSession.close).toHaveBeenCalledOnce();
454
- expect(secondSession.executeStatement.mock.calls.map((call) => describedTable(call[0]))).toEqual(['table_two', 'table_three']);
455
- });
456
- it('lets the in-flight batch settle before closing the lost session', async () => {
457
- const order = [];
458
- const firstSession = createSession({
459
- executeStatement: vi.fn((sql) => {
460
- const tableName = describedTable(sql);
461
- return new Promise((resolve, reject) => {
462
- setTimeout(() => {
463
- order.push(`describe:${tableName}`);
464
- reject(sessionLostError());
465
- }, tableName === 'table_one' ? 10 : 100);
466
- });
467
- }),
468
- });
469
- firstSession.close.mockImplementation(async () => {
470
- order.push('close');
471
- });
472
- mocks.openSession
473
- .mockResolvedValueOnce(firstSession)
474
- .mockResolvedValueOnce(columnsSession(() => [], new Set()));
475
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
476
- await withTimers(() => warehouse.getCatalog([
477
- tableRequest('table_one'),
478
- tableRequest('table_two'),
479
- ]));
480
- expect(order).toEqual([
481
- 'describe:table_one',
482
- 'describe:table_two',
483
- 'close',
484
- ]);
485
- });
486
- it('shares one retry budget across the whole fetch', async () => {
487
- mocks.openSession.mockResolvedValue(columnsSession(() => [], new Set(['table_one'])));
488
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
489
- await expect(withTimers(() => warehouse.getCatalog([tableRequest('table_one')]))).rejects.toMatchObject({ message: sessionLostMessage });
490
- expect(mocks.openSession).toHaveBeenCalledTimes(24);
491
- });
492
- it('fails fast on errors that are not warehouse startup errors', async () => {
493
- const session = createSession({
494
- executeStatement: vi.fn(async (sql) => describedTable(sql) === 'table_two'
495
- ? Promise.reject(statusError('PERMISSION_DENIED on table_two'))
496
- : createOperation({
497
- fetchAll: vi.fn(async () => jsonDescription([])),
498
- })),
499
- });
500
- mocks.openSession.mockResolvedValueOnce(session);
501
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
502
- await expect(withTimers(() => warehouse.getCatalog([
503
- tableRequest('table_one'),
504
- tableRequest('table_two'),
505
- ]))).rejects.toMatchObject({
506
- message: 'PERMISSION_DENIED on table_two',
507
- });
508
- expect(mocks.openSession).toHaveBeenCalledOnce();
509
- expect(session.close).toHaveBeenCalledOnce();
510
- });
511
- });
512
- });
513
- describe('getDatabricksErrorCondition', () => {
514
- // Messages as a serverless SQL warehouse returned them on 2026-09-14.
515
- test('reads the condition Databricks prefixes to SQL errors', () => {
516
- expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(statusError('[TABLE_OR_VIEW_NOT_FOUND] The table or view `lightdash_staging`.`nested`.`odd_names` cannot be found. Verify the spelling and correctness of the schema and catalog. SQLSTATE: 42P01; line 1 pos 24'))).toBe(DatabricksWarehouseClient_1.DatabricksErrorCondition.TableOrViewNotFound);
517
- expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(statusError("[PARSE_SYNTAX_ERROR] Syntax error at or near 'JSON'. SQLSTATE: 42601 (line 1, pos 37)"))).toBe(DatabricksWarehouseClient_1.DatabricksErrorCondition.ParseSyntaxError);
518
- });
519
- test('ignores conditions we do not handle and messages without a prefix', () => {
520
- expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(statusError('[INSUFFICIENT_PERMISSIONS] User does not have USE SCHEMA on Schema `schema`.'))).toBeUndefined();
521
- expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(statusError('Query could not be scheduled: TEMPORARILY_UNAVAILABLE'))).toBeUndefined();
522
- expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(new Error('mentions TABLE_OR_VIEW_NOT_FOUND mid-sentence'))).toBeUndefined();
523
- });
524
- });
525
- describe('DatabricksSqlBuilder escaping', () => {
526
- const databricksSqlBuilder = new DatabricksWarehouseClient_1.DatabricksSqlBuilder();
527
- test('Should escape backslashes and quotes in Databricks', () => {
528
- expect(databricksSqlBuilder.escapeString("\\') OR (1=1) --")).toBe("\\\\\\') OR (1=1) ");
529
- });
530
- test('Should handle SQL injection attempts', () => {
531
- // Test with a typical SQL injection pattern
532
- const maliciousInput = "'; DROP TABLE users; --";
533
- const escaped = databricksSqlBuilder.escapeString(maliciousInput);
534
- expect(escaped).toBe("\\'; DROP TABLE users; ");
535
- // Test with another common SQL injection pattern
536
- const anotherMaliciousInput = "' OR '1'='1";
537
- const anotherEscaped = databricksSqlBuilder.escapeString(anotherMaliciousInput);
538
- expect(anotherEscaped).toBe("\\' OR \\'1\\'=\\'1");
539
- });
540
- test('Should NOT remove # comments from strings', () => {
541
- // Test that # symbols are preserved in strings (not treated as comments)
542
- const stringWithHash = 'Column name with # symbol';
543
- const escaped = databricksSqlBuilder.escapeString(stringWithHash);
544
- expect(escaped).toBe('Column name with # symbol');
545
- // Test that # at start of line is preserved
546
- const hashAtStart = '#important-tag';
547
- const escapedHashStart = databricksSqlBuilder.escapeString(hashAtStart);
548
- expect(escapedHashStart).toBe('#important-tag');
549
- // Test multiple # symbols are preserved
550
- const multipleHashes = 'value1#value2#value3';
551
- const escapedMultiple = databricksSqlBuilder.escapeString(multipleHashes);
552
- expect(escapedMultiple).toBe('value1#value2#value3');
553
- });
554
- test('Should still remove -- and /* */ comments', () => {
555
- // Test that -- comments are still removed
556
- const stringWithDashComment = 'test value -- this is a comment';
557
- const escapedDash = databricksSqlBuilder.escapeString(stringWithDashComment);
558
- expect(escapedDash).toBe('test value ');
559
- // Test that /* */ comments are still removed
560
- const stringWithBlockComment = 'test /* block comment */ value';
561
- const escapedBlock = databricksSqlBuilder.escapeString(stringWithBlockComment);
562
- expect(escapedBlock).toBe('test value');
563
- });
564
- });
565
- describe('DatabricksWarehouseClient getAllTables', () => {
566
- it('lists every relation type in Unity Catalog, not only managed tables', async () => {
567
- const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
568
- const runQuery = vi.spyOn(warehouse, 'runQuery').mockResolvedValueOnce({
569
- rows: [
570
- {
571
- table_catalog: 'main',
572
- table_schema: 'analytics',
573
- table_name: 'orders_view',
574
- table_type: 'MATERIALIZED_VIEW',
575
- },
576
- ],
577
- fields: {},
578
- });
579
- const tables = await warehouse.getAllTables();
580
- const [query] = runQuery.mock.calls[0];
581
- expect(query).not.toContain('table_type =');
582
- expect(query).toContain("table_schema <> 'information_schema'");
583
- expect(tables).toEqual([
584
- {
585
- database: 'main',
586
- schema: 'analytics',
587
- table: 'orders_view',
588
- tableType: 'materialized_view',
589
- },
590
- ]);
591
- });
592
- });
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=DatabricksWarehouseStartupRetry.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"DatabricksWarehouseStartupRetry.test.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/DatabricksWarehouseStartupRetry.test.ts"],"names":[],"mappings":""}