@lightdash/cli 2.152.0 → 2.153.0

package/dist/dbt/models.d.ts

@@ -1,4 +1,4 @@
-import { DbtDoc, DbtModelNode, DbtSchemaEditor, SupportedDbtVersions } from '@lightdash/common';
+import { DbtDoc, DbtModelNode, DbtSchemaEditor, SupportedDbtVersions, type WarehouseCatalog } from '@lightdash/common';
 import { WarehouseClient, WarehouseTableSchema } from '@lightdash/warehouses';
 type CompiledModel = {
     name: string;
@@ -14,6 +14,17 @@ type GetDatabaseTableForModelArgs = {
     warehouseClient: WarehouseClient;
     preserveColumnCase: boolean;
 };
+type TableRef = {
+    database: string;
+    schema: string;
+    table: string;
+};
+/**
+ * Only scalar leaves reachable through non-repeated structs can be queried as
+ * plain columns today, so generated YAML skips container nodes and anything
+ * inside a repeated node.
+ */
+export declare const isGeneratableColumn: (catalog: WarehouseCatalog, { database, schema, table }: TableRef, columnPath: string) => boolean;
 export declare const getWarehouseTableForModel: ({ model, warehouseClient, preserveColumnCase, }: GetDatabaseTableForModelArgs) => Promise<WarehouseTableSchema>;
 export declare const isDocBlock: (text?: string | undefined) => boolean;
 type FindAndUpdateModelYamlArgs = {

package/dist/dbt/models.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../../src/dbt/models.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,MAAM,EACN,YAAY,EACZ,eAAe,EAIf,oBAAoB,EACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAQ9E,KAAK,aAAa,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAChC,KAAK,EAAE,aAAa,CAAC;IACrB,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC;CAC/B,CAAC;AACF,eAAO,MAAM,yBAAyB,oDAInC,4BAA4B,KAAG,OAAO,CAAC,oBAAoB,CA6B7D,CAAC;AAyDF,eAAO,MAAM,UAAU,UAAU,MAAM,GAAG,SAAS,KAAQ,OACX,CAAC;AA8BjD,KAAK,0BAA0B,GAAG;IAC9B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,oBAAoB,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,oBAAoB,CAAC;CACrC,CAAC;AACF,eAAO,MAAM,sBAAsB,yFAShC,0BAA0B,KAAG,OAAO,CAAC;IACpC,UAAU,EAAE,eAAe,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;CAC1B,CAoKA,CAAC;AAEF,eAAO,MAAM,iBAAiB,WAClB,YAAY,EAAE,QAChB;IACF,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC9B,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,KACF,OAAO,CAAC,aAAa,EAAE,CAoEzB,CAAC"}
+{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../../src/dbt/models.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,MAAM,EACN,YAAY,EACZ,eAAe,EAKf,oBAAoB,EACpB,KAAK,gBAAgB,EACxB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAQ9E,KAAK,aAAa,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,4BAA4B,GAAG;IAChC,KAAK,EAAE,aAAa,CAAC;IACrB,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC;CAC/B,CAAC;AACF,KAAK,QAAQ,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,YACnB,gBAAgB,+BACI,QAAQ,cACzB,MAAM,KACnB,OAqBF,CAAC;AAEF,eAAO,MAAM,yBAAyB,oDAInC,4BAA4B,KAAG,OAAO,CAAC,oBAAoB,CAgC7D,CAAC;AAyDF,eAAO,MAAM,UAAU,UAAU,MAAM,GAAG,SAAS,KAAQ,OACX,CAAC;AA8BjD,KAAK,0BAA0B,GAAG;IAC9B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,oBAAoB,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,oBAAoB,CAAC;CACrC,CAAC;AACF,eAAO,MAAM,sBAAsB,yFAShC,0BAA0B,KAAG,OAAO,CAAC;IACpC,UAAU,EAAE,eAAe,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;CAC1B,CAoKA,CAAC;AAEF,eAAO,MAAM,iBAAiB,WAClB,YAAY,EAAE,QAChB;IACF,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC7B,OAAO,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC9B,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5B,KACF,OAAO,CAAC,aAAa,EAAE,CAoEzB,CAAC"}

package/dist/dbt/models.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getCompiledModels = exports.findAndUpdateModelYaml = exports.isDocBlock = exports.getWarehouseTableForModel = void 0;
+exports.getCompiledModels = exports.findAndUpdateModelYaml = exports.isDocBlock = exports.getWarehouseTableForModel = exports.isGeneratableColumn = void 0;
 const tslib_1 = require("tslib");
 const common_1 = require("@lightdash/common");
 const execa_1 = tslib_1.__importDefault(require("execa"));
@@ -9,6 +9,27 @@ const path = tslib_1.__importStar(require("path"));
 const globalState_1 = tslib_1.__importDefault(require("../globalState"));
 const styles = tslib_1.__importStar(require("../styles"));
 const schema_1 = require("./schema");
+/**
+ * Only scalar leaves reachable through non-repeated structs can be queried as
+ * plain columns today, so generated YAML skips container nodes and anything
+ * inside a repeated node.
+ */
+const isGeneratableColumn = (catalog, { database, schema, table }, columnPath) => {
+    const segments = columnPath.split('.');
+    return segments.every((_, index) => {
+        const prefix = segments.slice(0, index + 1).join('.');
+        const shape = (0, common_1.getCatalogNestedColumnShape)(catalog, database, schema, table, prefix);
+        const isLeaf = index === segments.length - 1;
+        // An array of scalars is exposed through its container entry, so it
+        // is generated like a leaf; every other container and anything under
+        // an array is left for the user to document by hand.
+        const isScalarArray = shape?.repeated === true && shape.record === false;
+        return isLeaf
+            ? shape === undefined || isScalarArray
+            : shape?.repeated === false;
+    });
+};
+exports.isGeneratableColumn = isGeneratableColumn;
 const getWarehouseTableForModel = async ({ model, warehouseClient, preserveColumnCase, }) => {
     const tableRef = {
         database: model.database,
@@ -26,6 +47,9 @@ const getWarehouseTableForModel = async ({ model, warehouseClient, preserveColum
         throw new common_1.ParseError(`Expected to find materialised model at ${tableRef.database}.${tableRef.schema}.${tableRef.table} but couldn't find (or cannot access) ${missing}`);
     }
     return Object.entries(table).reduce((accumulator, [key, value]) => {
+        if (!(0, exports.isGeneratableColumn)(catalog, tableRef, key)) {
+            return accumulator;
+        }
         const columnName = preserveColumnCase ? key : key.toLowerCase();
         accumulator[columnName] = value;
         return accumulator;

package/dist/dbt/models.test.js

@@ -1,7 +1,48 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const common_1 = require("@lightdash/common");
 const models_1 = require("./models");
 describe('Models', () => {
+    describe('isGeneratableColumn', () => {
+        const ref = { database: 'db', schema: 'ds', table: 'sessions' };
+        const catalog = {
+            db: {
+                ds: {
+                    sessions: {
+                        visitId: common_1.DimensionType.NUMBER,
+                        totals: common_1.DimensionType.STRING,
+                        'totals.pageviews': common_1.DimensionType.NUMBER,
+                        hits: common_1.DimensionType.STRING,
+                        'hits.page': common_1.DimensionType.STRING,
+                        'hits.page.pagePath': common_1.DimensionType.STRING,
+                        tags: common_1.DimensionType.STRING,
+                    },
+                },
+            },
+        };
+        const shapes = {
+            totals: { repeated: false, record: true },
+            hits: { repeated: true, record: true },
+            'hits.page': { repeated: false, record: true },
+            tags: { repeated: true, record: false },
+        };
+        Object.entries(shapes).forEach(([path, shape]) => (0, common_1.setCatalogNestedColumnShape)(catalog, ref.database, ref.schema, ref.table, path, shape));
+        test('keeps scalar columns and leaves under plain structs', () => {
+            expect((0, models_1.isGeneratableColumn)(catalog, ref, 'visitId')).toBe(true);
+            expect((0, models_1.isGeneratableColumn)(catalog, ref, 'totals.pageviews')).toBe(true);
+        });
+        test('keeps an array of scalars, exposed through its container entry', () => {
+            expect((0, models_1.isGeneratableColumn)(catalog, ref, 'tags')).toBe(true);
+        });
+        test('skips struct and repeated record containers', () => {
+            expect((0, models_1.isGeneratableColumn)(catalog, ref, 'totals')).toBe(false);
+            expect((0, models_1.isGeneratableColumn)(catalog, ref, 'hits')).toBe(false);
+        });
+        test('skips leaves beneath a repeated node', () => {
+            expect((0, models_1.isGeneratableColumn)(catalog, ref, 'hits.page')).toBe(false);
+            expect((0, models_1.isGeneratableColumn)(catalog, ref, 'hits.page.pagePath')).toBe(false);
+        });
+    });
     describe('isDocBlock', () => {
         test('should match all doc block variations', () => {
             expect((0, models_1.isDocBlock)("{{doc('user_id')}}")).toBe(true); // single quote

package/dist/handlers/compile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/handlers/compile.ts"],"names":[],"mappings":"AAAA,OAAO,EAOH,OAAO,EACP,YAAY,EAgBf,MAAM,mBAAmB,CAAC;AAqB3B,OAAO,EAAE,iBAAiB,EAA8B,MAAM,eAAe,CAAC;AAI9E,MAAM,MAAM,qBAAqB,GAAG,iBAAiB,GAAG;IACpD,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,QAAQ,EAAE,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CAAC;IACrC,iBAAiB,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEF,eAAO,MAAM,uBAAuB,YACvB,OAAO,GAAG,YAAY,KAChC,OAIW,CAAC;AAEf,eAAO,MAAM,0BAA0B,YAC1B,OAAO,GAAG,YAAY,KAChC,OAAO,GAAG,YAiBZ,CAAC;AA6TF,eAAO,MAAM,cAAc,YACd,qBAAqB,KAC/B,OAAO,CAAC,oBAAoB,CA0a9B,CAAC;AAEF,eAAO,MAAM,OAAO,YACP,qBAAqB,KAC/B,OAAO,CAAC,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CACO,CAAC;AAE7C,eAAO,MAAM,cAAc,oBACN,qBAAqB,kBAmCzC,CAAC"}
+{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/handlers/compile.ts"],"names":[],"mappings":"AAAA,OAAO,EAOH,OAAO,EACP,YAAY,EAkBf,MAAM,mBAAmB,CAAC;AAqB3B,OAAO,EAAE,iBAAiB,EAA8B,MAAM,eAAe,CAAC;AAsB9E,MAAM,MAAM,qBAAqB,GAAG,iBAAiB,GAAG;IACpD,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IAC/B,QAAQ,EAAE,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CAAC;IACrC,iBAAiB,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEF,eAAO,MAAM,uBAAuB,YACvB,OAAO,GAAG,YAAY,KAChC,OAIW,CAAC;AAEf,eAAO,MAAM,0BAA0B,YAC1B,OAAO,GAAG,YAAY,KAChC,OAAO,GAAG,YAiBZ,CAAC;AA6TF,eAAO,MAAM,cAAc,YACd,qBAAqB,KAC/B,OAAO,CAAC,oBAAoB,CA2a9B,CAAC;AAEF,eAAO,MAAM,OAAO,YACP,qBAAqB,KAC/B,OAAO,CAAC,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,CACO,CAAC;AAE7C,eAAO,MAAM,cAAc,oBACN,qBAAqB,kBAmCzC,CAAC"}

package/dist/handlers/compile.js

@@ -19,6 +19,22 @@ const apiClient_1 = require("./dbt/apiClient");
 const compile_1 = require("./dbt/compile");
 const getDbtVersion_1 = require("./dbt/getDbtVersion");
 const getWarehouseClient_1 = tslib_1.__importDefault(require("./dbt/getWarehouseClient"));
+// Compile-time behaviour the server gates behind a feature flag. Unreachable
+// servers and older ones that don't know the flag both mean "off".
+const getUnnestRepeatedColumns = async () => {
+    try {
+        const flag = await (0, apiClient_1.lightdashApi)({
+            method: 'GET',
+            url: `/api/v2/feature-flag/${common_1.FeatureFlags.UnnestRepeatedColumns}`,
+            body: undefined,
+        });
+        return flag.enabled;
+    }
+    catch (e) {
+        globalState_1.default.debug(`> Could not read the ${common_1.FeatureFlags.UnnestRepeatedColumns} flag, compiling without it: ${(0, common_1.getErrorMessage)(e)}`);
+        return false;
+    }
+};
 const hasBlockingCompileError = (explore) => (0, common_1.isExploreError)(explore) ||
     explore.warnings?.some((warning) => warning.type === common_1.InlineErrorType.WAREHOUSE_COLUMN_ERROR) === true;
 exports.hasBlockingCompileError = hasBlockingCompileError;
@@ -414,6 +430,7 @@ const compileProject = async (options) => {
             disableTimestampConversion: options.disableTimestampConversion,
             allowPartialCompilation,
             postProcessors: [common_1.preAggregatePostProcessor],
+            unnestRepeatedColumns: await getUnnestRepeatedColumns(),
         });
         const validatedExplores = options.validateWarehouseColumns === true &&
             validationWarehouseClient

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/cli",
-  "version": "2.152.0",
+  "version": "2.153.0",
   "description": "Lightdash CLI tool",
   "license": "MIT",
   "repository": {
@@ -45,8 +45,8 @@
     "unique-names-generator": "4.7.1",
     "uuid": "11.1.1",
     "yaml": "2.8.3",
-    "@lightdash/common": "2.152.0",
-    "@lightdash/warehouses": "2.152.0"
+    "@lightdash/warehouses": "2.153.0",
+    "@lightdash/common": "2.153.0"
   },
   "devDependencies": {
     "@types/inquirer": "8.2.4",