@lightdash/warehouses 0.3455.0 → 0.3457.0

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export * from './ssh/sshTunnel';
 export * from './types';
+export * from './validateWarehouseColumnReferences';
 export * from './warehouseClientFromCredentials';
 export * from './warehouseClients/AthenaWarehouseClient';
 export * from './warehouseClients/BigqueryWarehouseClient';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,4CAA4C,CAAC;AAC3D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,6CAA6C,CAAC;AAC5D,cAAc,yCAAyC,CAAC;AACxD,cAAc,+BAA+B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC;AACxB,cAAc,qCAAqC,CAAC;AACpD,cAAc,kCAAkC,CAAC;AACjD,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,8CAA8C,CAAC;AAC7D,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,4CAA4C,CAAC;AAC3D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,6CAA6C,CAAC;AAC5D,cAAc,yCAAyC,CAAC;AACxD,cAAc,+BAA+B,CAAC"}

package/dist/index.js

@@ -16,6 +16,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./ssh/sshTunnel"), exports);
 __exportStar(require("./types"), exports);
+__exportStar(require("./validateWarehouseColumnReferences"), exports);
 __exportStar(require("./warehouseClientFromCredentials"), exports);
 __exportStar(require("./warehouseClients/AthenaWarehouseClient"), exports);
 __exportStar(require("./warehouseClients/BigqueryWarehouseClient"), exports);

package/dist/validateWarehouseColumnReferences.d.ts

@@ -0,0 +1,9 @@
+import { type Explore, type ExploreError, type WarehouseClient } from '@lightdash/common';
+type WarehouseColumnValidationClient = Pick<WarehouseClient, 'getFieldQuoteChar' | 'runQuery'>;
+export declare const validateWarehouseColumnReferences: ({ explores, client, tags, }: {
+    explores: (Explore | ExploreError)[];
+    client: WarehouseColumnValidationClient;
+    tags: Record<string, string>;
+}) => Promise<(Explore | ExploreError)[]>;
+export {};
+//# sourceMappingURL=validateWarehouseColumnReferences.d.ts.map

package/dist/validateWarehouseColumnReferences.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validateWarehouseColumnReferences.d.ts","sourceRoot":"","sources":["../src/validateWarehouseColumnReferences.ts"],"names":[],"mappings":"AAAA,OAAO,EAKH,KAAK,OAAO,EACZ,KAAK,YAAY,EACjB,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAG3B,KAAK,+BAA+B,GAAG,IAAI,CACvC,eAAe,EACf,mBAAmB,GAAG,UAAU,CACnC,CAAC;AAmKF,eAAO,MAAM,iCAAiC,GAAU,6BAIrD;IACC,QAAQ,EAAE,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CAAC;IACrC,MAAM,EAAE,+BAA+B,CAAC;IACxC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC,KAAG,OAAO,CAAC,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CA0CrC,CAAC"}

package/dist/validateWarehouseColumnReferences.js

@@ -0,0 +1,143 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateWarehouseColumnReferences = void 0;
+const common_1 = require("@lightdash/common");
+const processPromisesInBatches_1 = require("./utils/processPromisesInBatches");
+const getTableReferences = (explores) => {
+    const referencesByTable = new Map();
+    explores.forEach((explore) => {
+        if ((0, common_1.isExploreError)(explore))
+            return;
+        Object.values(explore.tables).forEach((table) => {
+            if (table.sqlTable.includes('${'))
+                return;
+            const key = `${table.name}\0${table.sqlTable}`;
+            const tableReference = referencesByTable.get(key) ?? {
+                modelName: table.name,
+                sqlTable: table.sqlTable,
+                columnReferences: new Set(),
+            };
+            [
+                ...Object.values(table.dimensions),
+                ...Object.values(table.metrics),
+            ]
+                .flatMap((field) => (0, common_1.getTableColumnReferences)(field.sql))
+                .forEach((columnReference) => tableReference.columnReferences.add(columnReference));
+            referencesByTable.set(key, tableReference);
+        });
+    });
+    return [...referencesByTable.values()].filter(({ columnReferences }) => columnReferences.size > 0);
+};
+const getProbeSql = ({ sqlTable, modelName, columnReferences, fieldQuoteChar, }) => {
+    const tableAlias = `${fieldQuoteChar}${modelName}${fieldQuoteChar}`;
+    const select = columnReferences.length === 0
+        ? '1'
+        : columnReferences
+            .map((reference) => `${tableAlias}.${reference}`)
+            .join(', ');
+    return `SELECT ${select} FROM ${sqlTable} AS ${tableAlias} WHERE 1 = 0`;
+};
+const VALIDATION_CONCURRENCY = 5;
+const findInvalidColumnReferences = async ({ client, tableReferences, tags, }) => {
+    const fieldQuoteChar = client.getFieldQuoteChar();
+    const invalidReferencesByTable = await (0, processPromisesInBatches_1.processPromisesInBatches)(tableReferences, VALIDATION_CONCURRENCY, async (tableReference) => {
+        const sharedProbeArgs = {
+            sqlTable: tableReference.sqlTable,
+            modelName: tableReference.modelName,
+            fieldQuoteChar,
+        };
+        const columnReferences = [...tableReference.columnReferences];
+        try {
+            await client.runQuery(getProbeSql({
+                ...sharedProbeArgs,
+                columnReferences,
+            }), tags);
+            return [];
+        }
+        catch {
+            // Probe separately only when the batched query fails.
+        }
+        try {
+            await client.runQuery(getProbeSql({
+                ...sharedProbeArgs,
+                columnReferences: [],
+            }), tags);
+        }
+        catch {
+            // Relation failures do not prove that its columns are invalid.
+            return [];
+        }
+        const invalidReferences = await (0, processPromisesInBatches_1.processPromisesInBatches)(columnReferences, VALIDATION_CONCURRENCY, async (columnReference) => {
+            try {
+                await client.runQuery(getProbeSql({
+                    ...sharedProbeArgs,
+                    columnReferences: [columnReference],
+                }), tags);
+                return null;
+            }
+            catch (error) {
+                try {
+                    await client.runQuery(getProbeSql({
+                        ...sharedProbeArgs,
+                        columnReferences: [],
+                    }), tags);
+                    return {
+                        modelName: tableReference.modelName,
+                        columnReference,
+                        error: (0, common_1.getErrorMessage)(error),
+                    };
+                }
+                catch {
+                    // The relation became unavailable mid-probe.
+                    return null;
+                }
+            }
+        });
+        return invalidReferences.filter((reference) => reference !== null);
+    });
+    return invalidReferencesByTable.flat();
+};
+const validateWarehouseColumnReferences = async ({ explores, client, tags, }) => {
+    const invalidReferences = await findInvalidColumnReferences({
+        client,
+        tableReferences: getTableReferences(explores),
+        tags,
+    });
+    if (invalidReferences.length === 0)
+        return explores;
+    const warningExploreByModel = new Map();
+    explores.forEach((explore) => {
+        if ((0, common_1.isExploreError)(explore))
+            return;
+        Object.values(explore.tables).forEach((table) => {
+            const current = warningExploreByModel.get(table.name);
+            if (current === undefined || explore.name === table.name) {
+                warningExploreByModel.set(table.name, explore.name);
+            }
+        });
+    });
+    const warningsByExplore = new Map();
+    invalidReferences.forEach(({ modelName, columnReference, error }) => {
+        const exploreName = warningExploreByModel.get(modelName);
+        if (exploreName === undefined)
+            return;
+        const warnings = warningsByExplore.get(exploreName) ?? [];
+        warnings.push({
+            type: common_1.InlineErrorType.WAREHOUSE_COLUMN_ERROR,
+            message: `Warehouse rejected column reference \${TABLE}.${columnReference} in model "${modelName}": ${error}`,
+        });
+        warningsByExplore.set(exploreName, warnings);
+    });
+    return explores.map((explore) => {
+        if ((0, common_1.isExploreError)(explore))
+            return explore;
+        const warnings = warningsByExplore.get(explore.name);
+        if (warnings === undefined)
+            return explore;
+        return {
+            ...explore,
+            warnings: [...(explore.warnings ?? []), ...warnings],
+        };
+    });
+};
+exports.validateWarehouseColumnReferences = validateWarehouseColumnReferences;

package/dist/validateWarehouseColumnReferences.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=validateWarehouseColumnReferences.test.d.ts.map

package/dist/validateWarehouseColumnReferences.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validateWarehouseColumnReferences.test.d.ts","sourceRoot":"","sources":["../src/validateWarehouseColumnReferences.test.ts"],"names":[],"mappings":""}

package/dist/validateWarehouseColumnReferences.test.js

@@ -0,0 +1,168 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const common_1 = require("@lightdash/common");
+const validateWarehouseColumnReferences_1 = require("./validateWarehouseColumnReferences");
+const explore = {
+    name: 'orders',
+    label: 'Orders',
+    tags: [],
+    baseTable: 'orders',
+    joinedTables: [],
+    targetDatabase: common_1.SupportedDbtAdapter.POSTGRES,
+    tables: {
+        orders: {
+            name: 'orders',
+            label: 'Orders',
+            database: 'postgres',
+            schema: 'jaffle',
+            sqlTable: '"postgres"."jaffle"."orders"',
+            lineageGraph: {},
+            dimensions: {
+                amount: {
+                    fieldType: common_1.FieldType.DIMENSION,
+                    type: common_1.DimensionType.NUMBER,
+                    name: 'amount',
+                    label: 'Amount',
+                    table: 'orders',
+                    tableLabel: 'Orders',
+                    // eslint-disable-next-line no-template-curly-in-string
+                    sql: '${TABLE}.amount',
+                    compiledSql: '"orders".amount',
+                    tablesReferences: ['orders'],
+                    hidden: false,
+                },
+            },
+            metrics: {
+                stale_metric: {
+                    fieldType: common_1.FieldType.METRIC,
+                    type: common_1.MetricType.AVERAGE,
+                    name: 'stale_metric',
+                    label: 'Stale metric',
+                    table: 'orders',
+                    tableLabel: 'Orders',
+                    // eslint-disable-next-line no-template-curly-in-string
+                    sql: '${TABLE}.rolling_30d_avg_sales',
+                    compiledSql: 'AVG("orders".rolling_30d_avg_sales)',
+                    tablesReferences: ['orders'],
+                    hidden: false,
+                },
+            },
+        },
+    },
+};
+const queryResult = { fields: {}, rows: [] };
+const tags = { query_context: 'cli' };
+describe('validateWarehouseColumnReferences', () => {
+    it('returns explores unchanged when the warehouse accepts all references', async () => {
+        const runQuery = vi.fn(async () => queryResult);
+        const result = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
+            explores: [explore],
+            client: { getFieldQuoteChar: () => '"', runQuery },
+            tags,
+        });
+        expect(result).toEqual([explore]);
+        expect(runQuery).toHaveBeenCalledTimes(1);
+        expect(runQuery).toHaveBeenNthCalledWith(1, 'SELECT "orders".amount, "orders".rolling_30d_avg_sales FROM "postgres"."jaffle"."orders" AS "orders" WHERE 1 = 0', tags);
+    });
+    it('attaches one warning for each reference rejected by the warehouse', async () => {
+        const runQuery = vi.fn(async (sql) => {
+            if (sql.includes('rolling_30d_avg_sales')) {
+                throw new Error('column orders.rolling_30d_avg_sales does not exist');
+            }
+            return queryResult;
+        });
+        const [result] = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
+            explores: [explore],
+            client: { getFieldQuoteChar: () => '"', runQuery },
+            tags,
+        });
+        expect(result).toEqual({
+            ...explore,
+            warnings: [
+                {
+                    type: common_1.InlineErrorType.WAREHOUSE_COLUMN_ERROR,
+                    message: 
+                    // eslint-disable-next-line no-template-curly-in-string
+                    'Warehouse rejected column reference ${TABLE}.rolling_30d_avg_sales in model "orders": column orders.rolling_30d_avg_sales does not exist',
+                },
+            ],
+        });
+        expect(runQuery).toHaveBeenCalledTimes(5);
+    });
+    it('skips column validation when the relation probe fails', async () => {
+        const runQuery = vi.fn(async () => {
+            throw new Error('warehouse unavailable');
+        });
+        const result = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
+            explores: [explore],
+            client: { getFieldQuoteChar: () => '"', runQuery },
+            tags,
+        });
+        expect(result).toEqual([explore]);
+        expect(runQuery).toHaveBeenCalledTimes(2);
+    });
+    it('preserves pre-existing warnings and appends warehouse warnings after them', async () => {
+        const existingWarning = {
+            type: common_1.InlineErrorType.FIELD_ERROR,
+            message: 'Dimension "broken_dimension" failed to compile',
+        };
+        const exploreWithExistingWarning = {
+            ...explore,
+            warnings: [existingWarning],
+        };
+        const runQuery = vi.fn(async (sql) => {
+            if (sql.includes('rolling_30d_avg_sales')) {
+                throw new Error('column orders.rolling_30d_avg_sales does not exist');
+            }
+            return queryResult;
+        });
+        const [result] = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
+            explores: [exploreWithExistingWarning],
+            client: { getFieldQuoteChar: () => '"', runQuery },
+            tags,
+        });
+        expect(result).toEqual({
+            ...exploreWithExistingWarning,
+            warnings: [
+                existingWarning,
+                {
+                    type: common_1.InlineErrorType.WAREHOUSE_COLUMN_ERROR,
+                    message: 
+                    // eslint-disable-next-line no-template-curly-in-string
+                    'Warehouse rejected column reference ${TABLE}.rolling_30d_avg_sales in model "orders": column orders.rolling_30d_avg_sales does not exist',
+                },
+            ],
+        });
+    });
+    it('skips explore errors and templated table references without probing', async () => {
+        const exploreError = {
+            name: 'broken_explore',
+            label: 'Broken explore',
+            errors: [
+                {
+                    type: common_1.InlineErrorType.METADATA_PARSE_ERROR,
+                    message: 'Failed to compile explore',
+                },
+            ],
+        };
+        const templatedExplore = {
+            ...explore,
+            name: 'templated_orders',
+            tables: {
+                orders: {
+                    ...explore.tables.orders,
+                    // eslint-disable-next-line no-template-curly-in-string
+                    sqlTable: '"postgres"."${ld.schema}"."orders"',
+                },
+            },
+        };
+        const runQuery = vi.fn(async () => queryResult);
+        const result = await (0, validateWarehouseColumnReferences_1.validateWarehouseColumnReferences)({
+            explores: [exploreError, templatedExplore],
+            client: { getFieldQuoteChar: () => '"', runQuery },
+            tags,
+        });
+        expect(result).toEqual([exploreError, templatedExplore]);
+        expect(runQuery).not.toHaveBeenCalled();
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/warehouses",
-  "version": "0.3455.0",
+  "version": "0.3457.0",
   "description": "Warehouse connectors for Lightdash",
   "license": "MIT",
   "repository": {
@@ -30,7 +30,7 @@
     "snowflake-sdk": "2.3.4",
     "ssh2": "1.14.0",
     "trino-client": "0.2.9",
-    "@lightdash/common": "0.3455.0"
+    "@lightdash/common": "0.3457.0"
   },
   "devDependencies": {
     "@types/big.js": "6.2.2",