@lightdash/cli 2.209.2 → 2.211.0

package/dist/dbt/models.d.ts

@@ -25,11 +25,24 @@ type TableRef = {
  * inside a repeated node.
  */
 export declare const isGeneratableColumn: (catalog: WarehouseCatalog, { database, schema, table }: TableRef, columnPath: string) => boolean;
-export declare const getWarehouseTableForModel: ({ model, warehouseClient, preserveColumnCase, }: GetDatabaseTableForModelArgs) => Promise<WarehouseTableSchema>;
+export type WarehouseModelColumns = {
+    /** Columns `generate` writes to YAML: scalars, struct leaves and arrays of scalars. */
+    table: WarehouseTableSchema;
+    /** Every column path the warehouse reports, containers and nested leaves included. */
+    warehouseColumnPaths: string[];
+};
+export declare const getWarehouseTableForModel: ({ model, warehouseClient, preserveColumnCase, }: GetDatabaseTableForModelArgs) => Promise<WarehouseModelColumns>;
+/**
+ * Existing YAML columns the warehouse no longer has. Nested leaves and
+ * containers are not generated, but they are real warehouse paths, so they
+ * are never reported as missing.
+ */
+export declare const getColumnsMissingFromWarehouse: (existingColumnNames: string[], warehouseColumnPaths: string[]) => string[];
 export declare const isDocBlock: (text?: string | undefined) => boolean;
 type FindAndUpdateModelYamlArgs = {
     model: CompiledModel;
     table: WarehouseTableSchema;
+    warehouseColumnPaths: string[];
     docs: Record<string, DbtDoc>;
     includeMeta: boolean;
     projectDir: string;
@@ -37,7 +50,7 @@ type FindAndUpdateModelYamlArgs = {
     assumeYes: boolean;
     dbtVersion?: SupportedDbtVersions;
 };
-export declare const findAndUpdateModelYaml: ({ model, table, docs, includeMeta, projectDir, projectName, assumeYes, dbtVersion, }: FindAndUpdateModelYamlArgs) => Promise<{
+export declare const findAndUpdateModelYaml: ({ model, table, warehouseColumnPaths, docs, includeMeta, projectDir, projectName, assumeYes, dbtVersion, }: FindAndUpdateModelYamlArgs) => Promise<{
     updatedYml: DbtSchemaEditor;
     outputFilePath: string;
 }>;

package/dist/dbt/models.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../../src/dbt/models.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,MAAM,EACN,YAAY,EACZ,eAAe,EAKf,oBAAoB,EACpB,KAAK,gBAAgB,EACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAQ9E,KAAK,aAAa,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAChC,KAAK,EAAE,aAAa,CAAC;IACrB,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC;CAC/B,CAAC;AACF,KAAK,QAAQ,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,YACnB,gBAAgB,+BACI,QAAQ,cACzB,MAAM,KACnB,OAqBF,CAAC;AAEF,eAAO,MAAM,yBAAyB,oDAInC,4BAA4B,KAAG,OAAO,CAAC,oBAAoB,CAgC7D,CAAC;AAyDF,eAAO,MAAM,UAAU,UAAU,MAAM,GAAG,SAAS,KAAQ,OACX,CAAC;AA8BjD,KAAK,0BAA0B,GAAG;IAC9B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,oBAAoB,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,oBAAoB,CAAC;CACrC,CAAC;AACF,eAAO,MAAM,sBAAsB,yFAShC,0BAA0B,KAAG,OAAO,CAAC;IACpC,UAAU,EAAE,eAAe,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;CAC1B,CAoKA,CAAC;AAEF,eAAO,MAAM,iBAAiB,WAClB,YAAY,EAAE,QAChB;IACF,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC9B,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,KACF,OAAO,CAAC,aAAa,EAAE,CAoEzB,CAAC"}
+{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../../src/dbt/models.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,MAAM,EACN,YAAY,EACZ,eAAe,EAKf,oBAAoB,EACpB,KAAK,gBAAgB,EACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAQ9E,KAAK,aAAa,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAChC,KAAK,EAAE,aAAa,CAAC;IACrB,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC;CAC/B,CAAC;AACF,KAAK,QAAQ,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,YACnB,gBAAgB,+BACI,QAAQ,cACzB,MAAM,KACnB,OAqBF,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAChC,uFAAuF;IACvF,KAAK,EAAE,oBAAoB,CAAC;IAC5B,sFAAsF;IACtF,oBAAoB,EAAE,MAAM,EAAE,CAAC;CAClC,CAAC;AAEF,eAAO,MAAM,yBAAyB,oDAInC,4BAA4B,KAAG,OAAO,CAAC,qBAAqB,CAoC9D,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,8BAA8B,wBAClB,MAAM,EAAE,wBACP,MAAM,EAAE,KAC/B,MAAM,EAKR,CAAC;AAyDF,eAAO,MAAM,UAAU,UAAU,MAAM,GAAG,SAAS,KAAQ,OACX,CAAC;AA8BjD,KAAK,0BAA0B,GAAG;IAC9B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,oBAAoB,CAAC;IAC5B,oBAAoB,EAAE,MAAM,EAAE,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,oBAAoB,CAAC;CACrC,CAAC;AACF,eAAO,MAAM,sBAAsB,+GAUhC,0BAA0B,KAAG,OAAO,CAAC;IACpC,UAAU,EAAE,eAAe,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;CAC1B,CA8JA,CAAC;AAEF,eAAO,MAAM,iBAAiB,WAClB,YAAY,EAAE,QAChB;IACF,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC9B,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,KACF,OAAO,CAAC,aAAa,EAAE,CAoEzB,CAAC"}

package/dist/dbt/models.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getCompiledModels = exports.findAndUpdateModelYaml = exports.isDocBlock = exports.getWarehouseTableForModel = exports.isGeneratableColumn = void 0;
+exports.getCompiledModels = exports.findAndUpdateModelYaml = exports.isDocBlock = exports.getColumnsMissingFromWarehouse = exports.getWarehouseTableForModel = exports.isGeneratableColumn = void 0;
 const tslib_1 = require("tslib");
 const common_1 = require("@lightdash/common");
 const execa_1 = tslib_1.__importDefault(require("execa"));
@@ -46,16 +46,29 @@ const getWarehouseTableForModel = async ({ model, warehouseClient, preserveColum
             (table === undefined && `table ${tableRef.table}`);
         throw new common_1.ParseError(`Expected to find materialised model at ${tableRef.database}.${tableRef.schema}.${tableRef.table} but couldn't find (or cannot access) ${missing}`);
     }
-    return Object.entries(table).reduce((accumulator, [key, value]) => {
-        if (!(0, exports.isGeneratableColumn)(catalog, tableRef, key)) {
+    const columnName = (key) => preserveColumnCase ? key : key.toLowerCase();
+    return {
+        table: Object.entries(table).reduce((accumulator, [key, value]) => {
+            if (!(0, exports.isGeneratableColumn)(catalog, tableRef, key)) {
+                return accumulator;
+            }
+            accumulator[columnName(key)] = value;
             return accumulator;
-        }
-        const columnName = preserveColumnCase ? key : key.toLowerCase();
-        accumulator[columnName] = value;
-        return accumulator;
-    }, {});
+        }, {}),
+        warehouseColumnPaths: Object.keys(table).map(columnName),
+    };
 };
 exports.getWarehouseTableForModel = getWarehouseTableForModel;
+/**
+ * Existing YAML columns the warehouse no longer has. Nested leaves and
+ * containers are not generated, but they are real warehouse paths, so they
+ * are never reported as missing.
+ */
+const getColumnsMissingFromWarehouse = (existingColumnNames, warehouseColumnPaths) => {
+    const known = new Set(warehouseColumnPaths.map((columnPath) => columnPath.toLowerCase()));
+    return existingColumnNames.filter((name) => !known.has(name.toLowerCase()));
+};
+exports.getColumnsMissingFromWarehouse = getColumnsMissingFromWarehouse;
 // Check if we should use dbt 1.10+ metadata structure
 const wrapInConfig = (dbtVersion, meta) => {
     const useDbt110Metadata = dbtVersion &&
@@ -112,7 +125,7 @@ const askOverwriteDescription = async (columnName, existingDescription, newDescr
         return newDescription;
     return existingDescription;
 };
-const findAndUpdateModelYaml = async ({ model, table, docs, includeMeta, projectDir, projectName, assumeYes, dbtVersion, }) => {
+const findAndUpdateModelYaml = async ({ model, table, warehouseColumnPaths, docs, includeMeta, projectDir, projectName, assumeYes, dbtVersion, }) => {
     const generatedModel = generateModelYml({
         model,
         table,
@@ -185,11 +198,7 @@ const findAndUpdateModelYaml = async ({ model, table, docs, includeMeta, project
         newColumns.forEach((column) => {
             schemaEditor.addColumn(model.name, column);
         });
-        // Delete columns that no longer exist in the warehouse (case-insensitive comparison)
-        const generatedColumnNamesLower = generatedModel.columns.map((gc) => gc.name.toLowerCase());
-        const deletedColumnNames = existingColumns
-            .filter((c) => !generatedColumnNamesLower.includes(c.name.toLowerCase()))
-            .map((c) => c.name);
+        const deletedColumnNames = (0, exports.getColumnsMissingFromWarehouse)(existingColumns.map((c) => c.name), warehouseColumnPaths);
         if (deletedColumnNames.length > 0 && !globalState_1.default.isNonInteractive()) {
             let answers = { isConfirm: assumeYes };
             if (!assumeYes) {

package/dist/dbt/models.test.js

@@ -43,6 +43,85 @@ describe('Models', () => {
             expect((0, models_1.isGeneratableColumn)(catalog, ref, 'hits.page.pagePath')).toBe(false);
         });
     });
+    describe('getWarehouseTableForModel', () => {
+        const catalog = {
+            db: {
+                ds: {
+                    orders: {
+                        order_id: common_1.DimensionType.STRING,
+                        product: common_1.DimensionType.STRING,
+                        'product.sku': common_1.DimensionType.STRING,
+                        'product.price': common_1.DimensionType.NUMBER,
+                        'product.variants': common_1.DimensionType.STRING,
+                        'product.variants.size': common_1.DimensionType.STRING,
+                        customer: common_1.DimensionType.STRING,
+                        'customer.location': common_1.DimensionType.STRING,
+                        tags: common_1.DimensionType.STRING,
+                    },
+                },
+            },
+        };
+        Object.entries({
+            product: { repeated: true, record: true },
+            'product.variants': { repeated: true, record: true },
+            customer: { repeated: false, record: true },
+            tags: { repeated: true, record: false },
+        }).forEach(([path, shape]) => (0, common_1.setCatalogNestedColumnShape)(catalog, 'db', 'ds', 'orders', path, shape));
+        const warehouseClient = {
+            getCatalog: vi.fn(async () => catalog),
+        };
+        const model = {
+            name: 'orders',
+            schema: 'ds',
+            database: 'db',
+            alias: 'orders',
+            originalFilePath: 'models/orders.sql',
+            patchPath: null,
+            packageName: 'p',
+            uniqueId: 'model.p.orders',
+        };
+        test('generates scalars, struct leaves and arrays of scalars but reports every warehouse path', async () => {
+            const result = await (0, models_1.getWarehouseTableForModel)({
+                model,
+                warehouseClient,
+                preserveColumnCase: false,
+            });
+            expect(Object.keys(result.table).sort()).toEqual([
+                'customer.location',
+                'order_id',
+                'tags',
+            ]);
+            expect(result.warehouseColumnPaths.sort()).toEqual([
+                'customer',
+                'customer.location',
+                'order_id',
+                'product',
+                'product.price',
+                'product.sku',
+                'product.variants',
+                'product.variants.size',
+                'tags',
+            ]);
+        });
+    });
+    describe('getColumnsMissingFromWarehouse', () => {
+        test('keeps documented nested leaves and containers, drops only paths the warehouse lacks', () => {
+            expect((0, models_1.getColumnsMissingFromWarehouse)([
+                'order_id',
+                'Product',
+                'product.sku',
+                'product.variants.size',
+                'legacy_status',
+                'product.colour',
+            ], [
+                'order_id',
+                'product',
+                'product.sku',
+                'product.variants',
+                'product.variants.size',
+            ])).toEqual(['legacy_status', 'product.colour']);
+        });
+    });
     describe('isDocBlock', () => {
         test('should match all doc block variations', () => {
             expect((0, models_1.isDocBlock)("{{doc('user_id')}}")).toBe(true); // single quote

package/dist/handlers/generate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../src/handlers/generate.ts"],"names":[],"mappings":"AAqBA,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAMlD,KAAK,sBAAsB,GAAG,qBAAqB,GAAG;IAClD,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;IACnB,WAAW,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,kBAAkB,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,eAAO,MAAM,eAAe,YAAmB,sBAAsB,kBA6KpE,CAAC"}
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../src/handlers/generate.ts"],"names":[],"mappings":"AAqBA,OAAO,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAMlD,KAAK,sBAAsB,GAAG,qBAAqB,GAAG;IAClD,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;IACnB,WAAW,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,kBAAkB,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,eAAO,MAAM,eAAe,YAAmB,sBAAsB,kBA+KpE,CAAC"}

package/dist/handlers/generate.js

@@ -82,7 +82,7 @@ const generateHandler = async (options) => {
     for await (const compiledModel of compiledModels) {
         const spinner = globalState_1.default.startSpinner(`  Generating .yml for model ${styles.bold(compiledModel.name)}`);
         try {
-            const table = await (0, models_1.getWarehouseTableForModel)({
+            const { table, warehouseColumnPaths } = await (0, models_1.getWarehouseTableForModel)({
                 model: compiledModel,
                 warehouseClient,
                 preserveColumnCase: options.preserveColumnCase,
@@ -95,6 +95,7 @@ const generateHandler = async (options) => {
             const { updatedYml, outputFilePath } = await (0, models_1.findAndUpdateModelYaml)({
                 model: compiledModel,
                 table,
+                warehouseColumnPaths,
                 docs: manifest.docs,
                 includeMeta: !options.excludeMeta,
                 projectDir: absoluteProjectPath,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/cli",
-  "version": "2.209.2",
+  "version": "2.211.0",
   "description": "Lightdash CLI tool",
   "license": "MIT",
   "repository": {
@@ -22,8 +22,8 @@
     "@actions/core": "1.11.1",
     "@aws-sdk/credential-providers": "3.995.0",
     "@casl/ability": "6.8.0",
-    "@lightdash/common": "2.209.2",
-    "@lightdash/warehouses": "2.209.2",
+    "@lightdash/common": "2.211.0",
+    "@lightdash/warehouses": "2.211.0",
     "@sidvind/better-ajv-errors": "7.0.0",
     "@types/columnify": "1.5.1",
     "ajv": "8.18.0",