@lightdash/warehouses 2.209.2 → 2.211.0

package/dist/warehouseClients/DatabricksWarehouseClient.test.js

@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 /* eslint-disable prefer-arrow-callback, func-names */
 const StatusError_1 = __importDefault(require("@databricks/sql/dist/errors/StatusError"));
 const TCLIService_types_1 = require("@databricks/sql/thrift/TCLIService_types");
+const common_1 = require("@lightdash/common");
 const DatabricksWarehouseClient_1 = require("./DatabricksWarehouseClient");
 const DatabricksWarehouseClient_mock_1 = require("./DatabricksWarehouseClient.mock");
 const WarehouseClient_mock_1 = require("./WarehouseClient.mock");
@@ -163,12 +164,273 @@ describe('DatabricksWarehouseClient', () => {
         const columns = (name, type) => [
             { COLUMN_NAME: name, TYPE_NAME: type },
         ];
+        const describedTable = (sql) => /`([^`]+)` AS JSON$/.exec(sql)?.[1] ?? '';
+        const jsonDescription = (cols) => [
+            {
+                json_metadata: JSON.stringify({
+                    columns: cols.map((col) => ({
+                        name: col.COLUMN_NAME,
+                        type: { name: col.TYPE_NAME.toLowerCase() },
+                    })),
+                }),
+            },
+        ];
         const columnsSession = (resolve, lostTables) => createSession({
-            getColumns: vi.fn(async ({ tableName }) => lostTables.has(tableName)
-                ? Promise.reject(sessionLostError())
-                : createOperation({
-                    fetchAll: vi.fn(async () => resolve(tableName)),
-                })),
+            executeStatement: vi.fn(async (sql) => {
+                const table = describedTable(sql);
+                return lostTables.has(table)
+                    ? Promise.reject(sessionLostError())
+                    : createOperation({
+                        fetchAll: vi.fn(async () => jsonDescription(resolve(table))),
+                    });
+            }),
+        });
+        // Captured from DESCRIBE TABLE EXTENDED ... AS JSON on a serverless warehouse.
+        const describedTransactions = {
+            columns: [
+                {
+                    name: 'order_id',
+                    type: { name: 'string', collation: 'UTF8_BINARY' },
+                    nullable: true,
+                },
+                { name: 'amount', type: { name: 'double' }, nullable: true },
+                {
+                    name: 'created_at',
+                    type: { name: 'timestamp_ltz' },
+                    nullable: true,
+                },
+                {
+                    name: 'updated_at',
+                    type: { name: 'timestamp_ntz' },
+                    nullable: true,
+                },
+                {
+                    name: 'customer',
+                    type: {
+                        name: 'struct',
+                        fields: [
+                            {
+                                name: 'id',
+                                type: { name: 'string' },
+                                nullable: true,
+                            },
+                            {
+                                name: 'location',
+                                type: { name: 'string' },
+                                nullable: true,
+                            },
+                        ],
+                    },
+                    nullable: true,
+                },
+                {
+                    name: 'product',
+                    type: {
+                        name: 'array',
+                        element_type: {
+                            name: 'struct',
+                            fields: [
+                                {
+                                    name: 'sku',
+                                    type: { name: 'string' },
+                                    nullable: true,
+                                },
+                                {
+                                    name: 'price',
+                                    type: { name: 'double' },
+                                    nullable: true,
+                                },
+                                {
+                                    name: 'attributes',
+                                    type: {
+                                        name: 'struct',
+                                        fields: [
+                                            {
+                                                name: 'colour',
+                                                type: { name: 'string' },
+                                                nullable: true,
+                                            },
+                                        ],
+                                    },
+                                    nullable: true,
+                                },
+                                {
+                                    name: 'variants',
+                                    type: {
+                                        name: 'array',
+                                        element_type: {
+                                            name: 'struct',
+                                            fields: [
+                                                {
+                                                    name: 'size',
+                                                    type: { name: 'string' },
+                                                    nullable: true,
+                                                },
+                                                {
+                                                    name: 'stock',
+                                                    type: { name: 'int' },
+                                                    nullable: true,
+                                                },
+                                            ],
+                                        },
+                                        element_nullable: true,
+                                    },
+                                    nullable: true,
+                                },
+                            ],
+                        },
+                        element_nullable: true,
+                    },
+                    nullable: true,
+                },
+                {
+                    name: 'tags',
+                    type: {
+                        name: 'array',
+                        element_type: { name: 'string' },
+                        element_nullable: true,
+                    },
+                    nullable: true,
+                },
+                {
+                    name: 'scores',
+                    type: { name: 'array', element_type: { name: 'int' } },
+                    nullable: true,
+                },
+                {
+                    name: 'matrix',
+                    type: {
+                        name: 'array',
+                        element_type: {
+                            name: 'array',
+                            element_type: { name: 'int' },
+                        },
+                    },
+                    nullable: true,
+                },
+                {
+                    name: 'labels',
+                    type: {
+                        name: 'map',
+                        key_type: { name: 'string' },
+                        value_type: { name: 'string' },
+                        value_nullable: true,
+                    },
+                    nullable: true,
+                },
+            ],
+        };
+        const describeSession = (describe, flatColumns = []) => createSession({
+            executeStatement: vi.fn(async (sql) => createOperation({ fetchAll: vi.fn(() => describe(sql)) })),
+            getColumns: vi.fn(async () => createOperation({
+                fetchAll: vi.fn(async () => flatColumns),
+            })),
+        });
+        it('reads dotted paths and nested shapes from the JSON table description', async () => {
+            const session = describeSession(async () => [
+                { json_metadata: JSON.stringify(describedTransactions) },
+            ]);
+            mocks.openSession.mockResolvedValue(session);
+            const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
+            const catalog = await warehouse.getCatalog([
+                tableRequest('transactions'),
+            ]);
+            const table = catalog[DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT'].schema.transactions;
+            expect(session.executeStatement).toHaveBeenCalledWith('DESCRIBE TABLE EXTENDED `database`.`schema`.`transactions` AS JSON');
+            expect(session.getColumns).not.toHaveBeenCalled();
+            expect(table).toEqual({
+                order_id: common_1.DimensionType.STRING,
+                amount: common_1.DimensionType.NUMBER,
+                created_at: common_1.DimensionType.TIMESTAMP,
+                updated_at: common_1.DimensionType.TIMESTAMP,
+                customer: common_1.DimensionType.STRING,
+                'customer.id': common_1.DimensionType.STRING,
+                'customer.location': common_1.DimensionType.STRING,
+                product: common_1.DimensionType.STRING,
+                'product.sku': common_1.DimensionType.STRING,
+                'product.price': common_1.DimensionType.NUMBER,
+                'product.attributes': common_1.DimensionType.STRING,
+                'product.attributes.colour': common_1.DimensionType.STRING,
+                'product.variants': common_1.DimensionType.STRING,
+                'product.variants.size': common_1.DimensionType.STRING,
+                'product.variants.stock': common_1.DimensionType.NUMBER,
+                tags: common_1.DimensionType.STRING,
+                scores: common_1.DimensionType.NUMBER,
+                matrix: common_1.DimensionType.STRING,
+                labels: common_1.DimensionType.STRING,
+            });
+            const shape = (path) => (0, common_1.getCatalogNestedColumnShape)(catalog, DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT', 'schema', 'transactions', path);
+            expect(shape('customer')).toEqual({
+                repeated: false,
+                record: true,
+            });
+            expect(shape('product')).toEqual({ repeated: true, record: true });
+            expect(shape('product.attributes')).toEqual({
+                repeated: false,
+                record: true,
+            });
+            expect(shape('product.variants')).toEqual({
+                repeated: true,
+                record: true,
+            });
+            expect(shape('tags')).toEqual({ repeated: true, record: false });
+            expect(shape('scores')).toEqual({ repeated: true, record: false });
+            expect(shape('matrix')).toBeUndefined();
+            expect(shape('labels')).toBeUndefined();
+            expect(shape('product.sku')).toBeUndefined();
+            const domain = (path) => (0, common_1.getCatalogTimestampDomain)(catalog, DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT', 'schema', 'transactions', path);
+            expect(domain('created_at')).toEqual('aware');
+            expect(domain('updated_at')).toEqual('naive');
+            expect((0, common_1.getCatalogNestedColumnsUnavailableReason)(catalog, DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT', 'schema', 'transactions')).toBeUndefined();
+        });
+        it('keeps the flat column list and records why when the JSON description is refused', async () => {
+            const session = describeSession(() => Promise.reject(statusError("[PARSE_SYNTAX_ERROR] Syntax error at or near 'JSON'. SQLSTATE: 42601")), [
+                { COLUMN_NAME: 'order_id', TYPE_NAME: 'STRING' },
+                {
+                    COLUMN_NAME: 'product',
+                    TYPE_NAME: 'ARRAY<STRUCT<sku: STRING>>',
+                },
+            ]);
+            mocks.openSession.mockResolvedValue(session);
+            const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
+            const catalog = await warehouse.getCatalog([
+                tableRequest('transactions'),
+            ]);
+            const database = DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT';
+            expect(catalog[database].schema.transactions).toEqual({
+                order_id: common_1.DimensionType.STRING,
+                product: common_1.DimensionType.STRING,
+            });
+            expect((0, common_1.getCatalogNestedColumnShape)(catalog, database, 'schema', 'transactions', 'product')).toBeUndefined();
+            expect((0, common_1.getCatalogNestedColumnsUnavailableReason)(catalog, database, 'schema', 'transactions')).toEqual("Databricks did not describe transactions as JSON ([PARSE_SYNTAX_ERROR] Syntax error at or near 'JSON'. SQLSTATE: 42601); nested columns need a SQL warehouse or Databricks Runtime 16.2 or newer.");
+        });
+        it('reports no columns, and no reason, for a table that does not exist', async () => {
+            const session = describeSession(() => Promise.reject(statusError('[TABLE_OR_VIEW_NOT_FOUND] The table or view `database`.`schema`.`missing` cannot be found.')));
+            mocks.openSession.mockResolvedValue(session);
+            const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
+            const catalog = await warehouse.getCatalog([
+                tableRequest('missing'),
+            ]);
+            const database = DatabricksWarehouseClient_mock_1.credentials.catalog ?? 'DEFAULT';
+            expect(catalog[database].schema.missing).toEqual({});
+            expect(session.getColumns).not.toHaveBeenCalled();
+            expect((0, common_1.getCatalogNestedColumnsUnavailableReason)(catalog, database, 'schema', 'missing')).toBeUndefined();
+        });
+        it('fails the fetch on errors that are not the missing JSON form', async () => {
+            const session = describeSession(() => Promise.reject(statusError('[INSUFFICIENT_PERMISSIONS] User does not have USE SCHEMA on Schema `schema`.')));
+            mocks.openSession.mockResolvedValue(session);
+            const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
+            await expect(warehouse.getCatalog([tableRequest('transactions')])).rejects.toThrow('INSUFFICIENT_PERMISSIONS');
+            expect(session.getColumns).not.toHaveBeenCalled();
+        });
+        it('fails the fetch when the JSON description cannot be read', async () => {
+            const session = describeSession(async () => [
+                { json_metadata: 'not json' },
+            ]);
+            mocks.openSession.mockResolvedValue(session);
+            const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
+            await expect(warehouse.getCatalog([tableRequest('transactions')])).rejects.toThrow('Could not read the description of');
+            expect(session.getColumns).not.toHaveBeenCalled();
         });
         it('resumes the remaining tables on a replacement session', async () => {
             const firstSession = columnsSession(() => columns('id', 'BIGINT'), new Set(['table_two', 'table_three']));
@@ -187,34 +449,37 @@ describe('DatabricksWarehouseClient', () => {
                 table_two: { name: 'string' },
                 table_three: { name: 'string' },
             });
-            expect(firstSession.getColumns).toHaveBeenCalledTimes(3);
+            expect(firstSession.executeStatement).toHaveBeenCalledTimes(3);
             expect(firstSession.close).toHaveBeenCalledOnce();
-            expect(secondSession.getColumns.mock.calls.map((call) => call[0].tableName)).toEqual(['table_two', 'table_three']);
+            expect(secondSession.executeStatement.mock.calls.map((call) => describedTable(call[0]))).toEqual(['table_two', 'table_three']);
         });
         it('lets the in-flight batch settle before closing the lost session', async () => {
             const order = [];
             const firstSession = createSession({
-                getColumns: vi.fn(({ tableName }) => new Promise((resolve, reject) => {
-                    setTimeout(() => {
-                        order.push(`getColumns:${tableName}`);
-                        reject(sessionLostError());
-                    }, tableName === 'table_one' ? 10 : 100);
-                })),
+                executeStatement: vi.fn((sql) => {
+                    const tableName = describedTable(sql);
+                    return new Promise((resolve, reject) => {
+                        setTimeout(() => {
+                            order.push(`describe:${tableName}`);
+                            reject(sessionLostError());
+                        }, tableName === 'table_one' ? 10 : 100);
+                    });
+                }),
             });
             firstSession.close.mockImplementation(async () => {
                 order.push('close');
             });
             mocks.openSession
                 .mockResolvedValueOnce(firstSession)
-                .mockResolvedValueOnce(createSession());
+                .mockResolvedValueOnce(columnsSession(() => [], new Set()));
             const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
             await withTimers(() => warehouse.getCatalog([
                 tableRequest('table_one'),
                 tableRequest('table_two'),
             ]));
             expect(order).toEqual([
-                'getColumns:table_one',
-                'getColumns:table_two',
+                'describe:table_one',
+                'describe:table_two',
                 'close',
             ]);
         });
@@ -226,9 +491,11 @@ describe('DatabricksWarehouseClient', () => {
         });
         it('fails fast on errors that are not warehouse startup errors', async () => {
             const session = createSession({
-                getColumns: vi.fn(async ({ tableName }) => tableName === 'table_two'
+                executeStatement: vi.fn(async (sql) => describedTable(sql) === 'table_two'
                     ? Promise.reject(statusError('PERMISSION_DENIED on table_two'))
-                    : createOperation()),
+                    : createOperation({
+                        fetchAll: vi.fn(async () => jsonDescription([])),
+                    })),
             });
             mocks.openSession.mockResolvedValueOnce(session);
             const warehouse = new DatabricksWarehouseClient_1.DatabricksWarehouseClient(DatabricksWarehouseClient_mock_1.credentials);
@@ -243,6 +510,18 @@ describe('DatabricksWarehouseClient', () => {
         });
     });
 });
+describe('getDatabricksErrorCondition', () => {
+    // Messages as a serverless SQL warehouse returned them on 2026-09-14.
+    test('reads the condition Databricks prefixes to SQL errors', () => {
+        expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(statusError('[TABLE_OR_VIEW_NOT_FOUND] The table or view `lightdash_staging`.`nested`.`odd_names` cannot be found. Verify the spelling and correctness of the schema and catalog. SQLSTATE: 42P01; line 1 pos 24'))).toBe(DatabricksWarehouseClient_1.DatabricksErrorCondition.TableOrViewNotFound);
+        expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(statusError("[PARSE_SYNTAX_ERROR] Syntax error at or near 'JSON'. SQLSTATE: 42601 (line 1, pos 37)"))).toBe(DatabricksWarehouseClient_1.DatabricksErrorCondition.ParseSyntaxError);
+    });
+    test('ignores conditions we do not handle and messages without a prefix', () => {
+        expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(statusError('[INSUFFICIENT_PERMISSIONS] User does not have USE SCHEMA on Schema `schema`.'))).toBeUndefined();
+        expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(statusError('Query could not be scheduled: TEMPORARILY_UNAVAILABLE'))).toBeUndefined();
+        expect((0, DatabricksWarehouseClient_1.getDatabricksErrorCondition)(new Error('mentions TABLE_OR_VIEW_NOT_FOUND mid-sentence'))).toBeUndefined();
+    });
+});
 describe('DatabricksSqlBuilder escaping', () => {
     const databricksSqlBuilder = new DatabricksWarehouseClient_1.DatabricksSqlBuilder();
     test('Should escape backslashes and quotes in Databricks', () => {

package/dist/warehouseClients/WarehouseBaseClient.d.ts

@@ -1,4 +1,4 @@
-import { AnyType, CreateWarehouseCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseCatalog, WarehouseResults, WarehouseSqlBuilder, WarehouseTables, WeekDay, type TimestampDomain, type WarehouseExecuteAsyncQuery, type WarehouseExecuteAsyncQueryArgs, type WarehouseQueryPhase } from '@lightdash/common';
+import { AnyType, CreateWarehouseCredentials, DimensionType, Metric, SupportedDbtAdapter, TimeIntervalUnit, UnnestSql, UnnestSqlArgs, WarehouseCatalog, WarehouseResults, WarehouseSqlBuilder, WarehouseTables, WeekDay, type TimestampDomain, type WarehouseExecuteAsyncQuery, type WarehouseExecuteAsyncQueryArgs, type WarehouseQueryPhase } from '@lightdash/common';
 import { type WarehouseClient } from '../types';
 export default abstract class WarehouseBaseClient<T extends CreateWarehouseCredentials> implements WarehouseClient {
     credentials: T;
@@ -9,6 +9,7 @@ export default abstract class WarehouseBaseClient<T extends CreateWarehouseCrede
     getStringQuoteChar(): string;
     getEscapeStringQuoteChar(): string;
     getFieldQuoteChar(): string;
+    getUnnestSql(args: UnnestSqlArgs): UnnestSql | null;
     getFloatingType(): string;
     getNullSafeEqualSql(left: string, right: string): string;
     getNullSafeEqualJoinSql(left: string, right: string): string;

package/dist/warehouseClients/WarehouseBaseClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"WarehouseBaseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/WarehouseBaseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,0BAA0B,EAC1B,aAAa,EACb,MAAM,EAEN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,OAAO,EACP,KAAK,eAAe,EACpB,KAAK,0BAA0B,EAC/B,KAAK,8BAA8B,EAEnC,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,UAAU,CAAC;AAEhD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,mBAAmB,CAC7C,CAAC,SAAS,0BAA0B,CACtC,YAAW,eAAe;IACxB,WAAW,EAAE,CAAC,CAAC;IAEf,SAAS,CAAC,UAAU,EAAE,mBAAmB,CAAC;IAE1C,SAAS,aAAa,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAGpE;IAED,cAAc,IAAI,mBAAmB,CAEpC;IAED,0BAA0B,IAAI,OAAO,CAEpC;IAED,kBAAkB,IAAI,MAAM,CAE3B;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,eAAe,IAAI,MAAM,CAExB;IAED,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3D;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAE1D;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAErC;IAED,QAAQ,CAAC,UAAU,CACf,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE7B,QAAQ,CAAC,WAAW,CAChB,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAAC;IAEX,iBAAiB,CACnB,EACI,GAAG,EACH,MAAM,EACN,WAAW,EACX,IAAI,EACJ,QAAQ,GACX,EAAE,8BAA8B,EACjC,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAC1B,OAAO,CAAC,0BAA0B,CAAC,CA+BrC;IAEK,QAAQ,CACV,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7B,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;;OAoBxC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEhD;IAED,cAAc,IAAI,OAAO,GAAG,IAAI,GAAG,SAAS,CAE3C;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAGK,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAEjD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;IAED,QAAQ,CAAC,YAAY,CACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5B,QAAQ,CAAC,SAAS,CACd,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE7B,qBAAqB,CACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,aAAa,EAC7C,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,eAAe,GAAG,SAAS,GACnE,gBAAgB,CAiClB;IAED,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAE9B;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAElC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAE7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAEvC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAE5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAKR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErC;CACJ"}
+{"version":3,"file":"WarehouseBaseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/WarehouseBaseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,OAAO,EACP,0BAA0B,EAC1B,aAAa,EACb,MAAM,EAEN,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,aAAa,EACb,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,OAAO,EACP,KAAK,eAAe,EACpB,KAAK,0BAA0B,EAC/B,KAAK,8BAA8B,EAEnC,KAAK,mBAAmB,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,KAAK,eAAe,EAAE,MAAM,UAAU,CAAC;AAEhD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,mBAAmB,CAC7C,CAAC,SAAS,0BAA0B,CACtC,YAAW,eAAe;IACxB,WAAW,EAAE,CAAC,CAAC;IAEf,SAAS,CAAC,UAAU,EAAE,mBAAmB,CAAC;IAE1C,SAAS,aAAa,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAGpE;IAED,cAAc,IAAI,mBAAmB,CAEpC;IAED,0BAA0B,IAAI,OAAO,CAEpC;IAED,kBAAkB,IAAI,MAAM,CAE3B;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,YAAY,CAAC,IAAI,EAAE,aAAa,GAAG,SAAS,GAAG,IAAI,CAElD;IAED,eAAe,IAAI,MAAM,CAExB;IAED,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3D;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAE1D;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAErC;IAED,QAAQ,CAAC,UAAU,CACf,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,GAC9D,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE7B,QAAQ,CAAC,WAAW,CAChB,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,CACZ,KAAK,EAAE,mBAAmB,EAC1B,UAAU,EAAE,MAAM,KACjB,IAAI,CAAC;KACb,GACF,OAAO,CAAC,IAAI,CAAC,CAAC;IAEX,iBAAiB,CACnB,EACI,GAAG,EACH,MAAM,EACN,WAAW,EACX,IAAI,EACJ,QAAQ,GACX,EAAE,8BAA8B,EACjC,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAC1B,OAAO,CAAC,0BAA0B,CAAC,CA+BrC;IAEK,QAAQ,CACV,GAAG,EAAE,MAAM,EACX,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7B,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,OAAO,EAAE,EAClB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;;OAoBxC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEhD;IAED,cAAc,IAAI,OAAO,GAAG,IAAI,GAAG,SAAS,CAE3C;IAEK,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAE1B;IAGK,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAEjD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;IAED,QAAQ,CAAC,YAAY,CACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,eAAe,CAAC,CAAC;IAE5B,QAAQ,CAAC,SAAS,CACd,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE7B,qBAAqB,CACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,aAAa,EAC7C,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,eAAe,GAAG,SAAS,GACnE,gBAAgB,CAiClB;IAED,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAE9B;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAElC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAE7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAEvC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAE5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAKR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAErC;CACJ"}

package/dist/warehouseClients/WarehouseBaseClient.js

@@ -21,6 +21,9 @@ class WarehouseBaseClient {
     getFieldQuoteChar() {
         return this.sqlBuilder.getFieldQuoteChar();
     }
+    getUnnestSql(args) {
+        return this.sqlBuilder.getUnnestSql(args);
+    }
     getFloatingType() {
         return this.sqlBuilder.getFloatingType();
     }

package/dist/warehouseClients/WarehouseBaseSqlBuilder.d.ts

@@ -1,4 +1,4 @@
-import { Metric, SupportedDbtAdapter, TimeIntervalUnit, WarehouseSqlBuilder, WeekDay } from '@lightdash/common';
+import { Metric, SupportedDbtAdapter, TimeIntervalUnit, UnnestSql, UnnestSqlArgs, WarehouseSqlBuilder, WeekDay } from '@lightdash/common';
 export default abstract class WarehouseBaseSqlBuilder implements WarehouseSqlBuilder {
     protected startOfWeek: WeekDay | null | undefined;
     constructor(startOfWeek?: WeekDay | null);
@@ -23,5 +23,6 @@ export default abstract class WarehouseBaseSqlBuilder implements WarehouseSqlBui
     getMedianSql(valueSql: string): string;
     buildArray(elements: string[]): string;
     buildArrayAgg(expression: string, orderBy?: string): string;
+    getUnnestSql(_args: UnnestSqlArgs): UnnestSql | null;
 }
 //# sourceMappingURL=WarehouseBaseSqlBuilder.d.ts.map

package/dist/warehouseClients/WarehouseBaseSqlBuilder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"WarehouseBaseSqlBuilder.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/WarehouseBaseSqlBuilder.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,MAAM,EACN,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EAEnB,OAAO,EACV,MAAM,mBAAmB,CAAC;AAG3B,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,uBAAwB,YAAW,mBAAmB;IAChF,SAAS,CAAC,WAAW,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IAElD,YAAY,WAAW,CAAC,EAAE,OAAO,GAAG,IAAI,EAEvC;IAED,cAAc,IAAI,OAAO,GAAG,IAAI,GAAG,SAAS,CAE3C;IAED,QAAQ,CAAC,cAAc,IAAI,mBAAmB,CAAC;IAK/C,0BAA0B,IAAI,OAAO,CAEpC;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,kBAAkB,IAAI,MAAM,CAE3B;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3D;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEhD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAIlC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAE7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGvC;IAED,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CACnD,gBAAgB,EAChB,MAAM,CACT,CAQC;IAEF,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAM1D;CACJ"}
+{"version":3,"file":"WarehouseBaseSqlBuilder.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/WarehouseBaseSqlBuilder.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,MAAM,EACN,mBAAmB,EACnB,gBAAgB,EAChB,SAAS,EACT,aAAa,EACb,mBAAmB,EAEnB,OAAO,EACV,MAAM,mBAAmB,CAAC;AAG3B,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,uBAAwB,YAAW,mBAAmB;IAChF,SAAS,CAAC,WAAW,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;IAElD,YAAY,WAAW,CAAC,EAAE,OAAO,GAAG,IAAI,EAEvC;IAED,cAAc,IAAI,OAAO,GAAG,IAAI,GAAG,SAAS,CAE3C;IAED,QAAQ,CAAC,cAAc,IAAI,mBAAmB,CAAC;IAK/C,0BAA0B,IAAI,OAAO,CAEpC;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,kBAAkB,IAAI,MAAM,CAE3B;IAED,wBAAwB,IAAI,MAAM,CAEjC;IAED,eAAe,IAAI,MAAM,CAExB;IAED,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEvD;IAED,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE3D;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEhD;IAED,YAAY,CAAC,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAEtC;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAIlC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAE7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGvC;IAED,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,EAAE,MAAM,CACnD,gBAAgB,EAChB,MAAM,CACT,CAQC;IAEF,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAM1D;IAGD,YAAY,CAAC,KAAK,EAAE,aAAa,GAAG,SAAS,GAAG,IAAI,CAEnD;CACJ"}

package/dist/warehouseClients/WarehouseBaseSqlBuilder.js

@@ -90,6 +90,10 @@ class WarehouseBaseSqlBuilder {
         }
         return `ARRAY_AGG(${expression})`;
     }
+    // Repeated columns stay opaque unless the warehouse can unnest them.
+    getUnnestSql(_args) {
+        return null;
+    }
 }
 WarehouseBaseSqlBuilder.intervalUnitsSingular = {
     [common_1.TimeIntervalUnit.SECOND]: 'SECOND',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/warehouses",
-  "version": "2.209.2",
+  "version": "2.211.0",
   "description": "Warehouse connectors for Lightdash",
   "license": "MIT",
   "repository": {
@@ -21,7 +21,7 @@
     "@databricks/sql": "1.13.0",
     "@duckdb/node-api": "1.5.2-r.2",
     "@google-cloud/bigquery": "8.3.1",
-    "@lightdash/common": "2.209.2",
+    "@lightdash/common": "2.211.0",
     "node-fetch": "2.7.0",
     "pg": "8.13.1",
     "pg-cursor": "2.10.0",