@lightdash/warehouses 2.152.0 → 2.153.0

package/dist/warehouseClients/BigqueryWarehouseClient.d.ts

@@ -28,7 +28,7 @@ export declare const getBigqueryTimestampDomain: (type: string | undefined) => T
 type TableSchema = {
     fields: SchemaFields[];
 };
-type SchemaFields = Required<Pick<bigquery.ITableFieldSchema, 'name' | 'type'>>;
+type SchemaFields = Required<Pick<bigquery.ITableFieldSchema, 'name' | 'type'>> & Pick<bigquery.ITableFieldSchema, 'mode' | 'fields'>;
 type BigqueryError = {
     errors: bigquery.IErrorProto[];
 };

package/dist/warehouseClients/BigqueryWarehouseClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAGP,eAAe,EACf,eAAe,EACf,6BAA6B,EAE7B,yBAAyB,EAGzB,MAAM,EAEN,eAAe,EAKf,mBAAmB,EACnB,gBAAgB,EAEhB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EACvB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACH,gBAAgB,EAChB,0BAA0B,EAC1B,8BAA8B,EAEjC,MAAM,UAAU,CAAC;AAOlB,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,iBAAiB;IACzB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,OAAO,YAAY;IACnB,UAAU,eAAe;IACzB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,KAAK,UAAU;CAClB;AAiDD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,GAAG,SAAS,KACzB,eAAe,GAAG,SASpB,CAAC;AA0BF,KAAK,WAAW,GAAG;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;CAC1B,CAAC;AAEF,KAAK,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;AAehF,KAAK,aAAa,GAAG;IACjB,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;CAClC,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,QAAQ,CAAC,IAAI,2BAA2B;IAExC,cAAc,IAAI,mBAAmB,CAEpC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAWhD;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGlC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAG7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGvC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAM1D;CACJ;AAqCD,qBAAa,uBAAwB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACvF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAM;IAExC,MAAM,EAAE,QAAQ,CAAC;IAEjB,YAAY,WAAW,EAAE,yBAAyB,EAyBjD;IAED,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAE7D;IAED,OAAO,CAAC,oBAAoB;IAQ5B,OAAO,CAAC,8BAA8B;IAsBtC,OAAO,CAAC,4BAA4B;IAOpC;;;;;;;;;;OAUG;IACH,MAAM,CAAC,wBAAwB,CAC3B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAoBpC;IAED,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,SAAS,oEAYtE;YAEa,SAAS;YA+BT,MAAM;YAWN,qBAAqB;YAUrB,aAAa;IAoCrB,WAAW,CACb,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC,CAuBf;IAED,OAAa,gBAAgB,CACzB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAShD;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,6BAiDN;IAEK,YAAY;;;;;SAOjB;YAEa,sBAAsB;IA+D9B,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAsB3B;IAED,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,GAAE,MAAW,uBA+FzD;IAEK,iBAAiB,CACnB,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,8BAA8B,EACvD,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GACV,OAAO,CAAC,0BAA0B,CAAC,CA+DrC;YAEa,kBAAkB;IAWhC,OAAa,YAAY,CACrB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,cAAc,GAAE;QACZ,WAAW,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KASvD,GACH,OAAO,CAAC,eAAe,EAAE,CAAC,CA0B5B;IAED,OAAa,wBAAwB,CACjC,QAAQ,EAAE,eAAe,EAAE,EAC3B,YAAY,EAAE,MAAM,EACpB,qBAAqB,GAAE,CAAC,SAAS,EAAE,MAAM,KAAK;QAC1C,WAAW,EAAE,CACT,OAAO,EAAE,kBAAkB,KAC1B,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/B,KAAK,EAAE,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAgBpC,GACF,OAAO,CAAC,6BAA6B,CAAC,CAuFxC;IAED,OAAa,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA0DxE;CACJ"}
+{"version":3,"file":"BigqueryWarehouseClient.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,QAAQ,EAKR,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAGlB,iBAAiB,EACpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,QAAQ,MAAM,wCAAwC,CAAC;AAC9D,OAAO,EACH,OAAO,EAGP,eAAe,EACf,eAAe,EACf,6BAA6B,EAE7B,yBAAyB,EAGzB,MAAM,EAEN,eAAe,EAMf,mBAAmB,EACnB,gBAAgB,EAEhB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,KAAK,eAAe,EAEvB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACH,gBAAgB,EAChB,0BAA0B,EAC1B,8BAA8B,EAEjC,MAAM,UAAU,CAAC;AAOlB,OAAO,mBAAmB,MAAM,uBAAuB,CAAC;AACxD,OAAO,uBAAuB,MAAM,2BAA2B,CAAC;AAKhE,oBAAY,iBAAiB;IACzB,MAAM,WAAW;IACjB,OAAO,YAAY;IACnB,KAAK,UAAU;IACf,KAAK,UAAU;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,OAAO,YAAY;IACnB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,OAAO,YAAY;IACnB,UAAU,eAAe;IACzB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,KAAK,UAAU;CAClB;AAqFD,eAAO,MAAM,0BAA0B,SAC7B,MAAM,GAAG,SAAS,KACzB,eAAe,GAAG,SASpB,CAAC;AA0BF,KAAK,WAAW,GAAG;IACf,MAAM,EAAE,YAAY,EAAE,CAAC;CAC1B,CAAC;AAEF,KAAK,YAAY,GAAG,QAAQ,CACxB,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,MAAM,CAAC,CACpD,GACG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,QAAQ,CAAC,CAAC;AAkDxD,KAAK,aAAa,GAAG;IACjB,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;CAClC,CAAC;AAEF,qBAAa,kBAAmB,SAAQ,uBAAuB;IAC3D,QAAQ,CAAC,IAAI,2BAA2B;IAExC,cAAc,IAAI,mBAAmB,CAEpC;IAED,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAWhD;IAED,iBAAiB,IAAI,MAAM,CAE1B;IAED,eAAe,IAAI,MAAM,CAExB;IAED,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAiBlC;IAED,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGlC;IAED,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAG7B;IAED,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAGvC;IAED,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAI5D;IAED,uBAAuB,CACnB,iBAAiB,EAAE,MAAM,EACzB,eAAe,EAAE,MAAM,GACxB,MAAM,CAGR;IAED,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAGrC;IAED,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAGrC;IAED,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAM1D;CACJ;AAqCD,qBAAa,uBAAwB,SAAQ,mBAAmB,CAAC,yBAAyB,CAAC;IACvF,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAM;IAExC,MAAM,EAAE,QAAQ,CAAC;IAEjB,YAAY,WAAW,EAAE,yBAAyB,EAyBjD;IAED,MAAM,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAE7D;IAED,OAAO,CAAC,oBAAoB;IAQ5B,OAAO,CAAC,8BAA8B;IAsBtC,OAAO,CAAC,4BAA4B;IAOpC;;;;;;;;;;OAUG;IACH,MAAM,CAAC,wBAAwB,CAC3B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAoBpC;IAED,MAAM,CAAC,qBAAqB,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,SAAS,oEAYtE;YAEa,SAAS;YA+BT,MAAM;YAWN,qBAAqB;YAUrB,aAAa;IAoCrB,WAAW,CACb,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAChE,OAAO,EAAE;QACL,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,GACF,OAAO,CAAC,IAAI,CAAC,CAuBf;IAED,OAAa,gBAAgB,CACzB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC,CAShD;IAEK,UAAU,CACZ,QAAQ,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;KACjB,EAAE,6BA0DN;IAEK,YAAY;;;;;SAOjB;YAEa,sBAAsB;IA+D9B,SAAS,CACX,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAmC3B;IAED,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,GAAE,MAAW,uBA+FzD;IAEK,iBAAiB,CACnB,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,8BAA8B,EACvD,qBAAqB,CAAC,EAAE,CACpB,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,KACjC,IAAI,GACV,OAAO,CAAC,0BAA0B,CAAC,CA+DrC;YAEa,kBAAkB;IAWhC,OAAa,YAAY,CACrB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,cAAc,GAAE;QACZ,WAAW,EAAE,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KASvD,GACH,OAAO,CAAC,eAAe,EAAE,CAAC,CA0B5B;IAED,OAAa,wBAAwB,CACjC,QAAQ,EAAE,eAAe,EAAE,EAC3B,YAAY,EAAE,MAAM,EACpB,qBAAqB,GAAE,CAAC,SAAS,EAAE,MAAM,KAAK;QAC1C,WAAW,EAAE,CACT,OAAO,EAAE,kBAAkB,KAC1B,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC/B,KAAK,EAAE,CAAC,OAAO,EAAE;YACb,KAAK,EAAE,MAAM,CAAC;YACd,QAAQ,EAAE,MAAM,CAAC;SACpB,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;KAgBpC,GACF,OAAO,CAAC,6BAA6B,CAAC,CAuFxC;IAED,OAAa,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA0DxE;CACJ"}

package/dist/warehouseClients/BigqueryWarehouseClient.js

@@ -47,6 +47,34 @@ const isBigqueryDecimal = (value) => typeof value === 'object' &&
     Array.isArray(value.c) &&
     value.c.length > 0 &&
     value.c.every((digit) => typeof digit === 'number');
+const isBigqueryTemporal = (cell) => cell instanceof bigquery_1.BigQueryDate ||
+    cell instanceof bigquery_1.BigQueryTimestamp ||
+    cell instanceof bigquery_1.BigQueryDatetime ||
+    cell instanceof bigquery_1.BigQueryTime;
+const isPlainObject = (value) => typeof value === 'object' &&
+    value !== null &&
+    Object.getPrototypeOf(value) === Object.prototype;
+// STRUCT and ARRAY values arrive as plain objects/arrays whose leaves are the
+// same SDK wrappers as top-level cells, so leaves are normalised before the
+// whole value is serialised.
+const normaliseNestedValue = (value) => {
+    if (Array.isArray(value)) {
+        return value.map(normaliseNestedValue);
+    }
+    if (isBigqueryTemporal(value)) {
+        return value.value;
+    }
+    if (isBigqueryDecimal(value)) {
+        return Number(value.toFixed());
+    }
+    if (isPlainObject(value)) {
+        return Object.fromEntries(Object.entries(value).map(([key, nested]) => [
+            key,
+            normaliseNestedValue(nested),
+        ]));
+    }
+    return value;
+};
 const parseCell = (cell) => {
     if (cell === undefined ||
         cell === null ||
@@ -54,15 +82,15 @@ const parseCell = (cell) => {
         typeof cell === 'number') {
         return cell;
     }
-    if (cell instanceof bigquery_1.BigQueryDate ||
-        cell instanceof bigquery_1.BigQueryTimestamp ||
-        cell instanceof bigquery_1.BigQueryDatetime ||
-        cell instanceof bigquery_1.BigQueryTime) {
+    if (isBigqueryTemporal(cell)) {
         return new Date(cell.value);
     }
     if (isBigqueryDecimal(cell)) {
         return Number(cell.toFixed());
     }
+    if (Array.isArray(cell) || isPlainObject(cell)) {
+        return JSON.stringify(normaliseNestedValue(cell));
+    }
     return `${cell}`;
 };
 const getBigqueryTimestampDomain = (type) => {
@@ -101,6 +129,28 @@ const mapFieldType = (type) => {
 };
 const isSchemaFields = (rawSchemaFields) => rawSchemaFields.every((field) => field.type && field.name);
 const isTableSchema = (schema) => !!schema && !!schema.fields && isSchemaFields(schema.fields);
+const BIGQUERY_REPEATED_MODE = 'REPEATED';
+const isRecordType = (type) => type === BigqueryFieldType.RECORD || type === BigqueryFieldType.STRUCT;
+/**
+ * Walks nested RECORD fields depth-first, emitting every node under its dotted
+ * path. A record node is kept alongside its children so the container column
+ * still resolves; the shape says whether it is a struct, an array, or both.
+ */
+const flattenSchemaFields = (fields, prefix = '') => fields.flatMap((field) => {
+    if (!field.name || !field.type)
+        return [];
+    const path = prefix ? `${prefix}.${field.name}` : field.name;
+    const repeated = field.mode === BIGQUERY_REPEATED_MODE;
+    const record = isRecordType(field.type);
+    const node = {
+        path,
+        type: field.type,
+        shape: repeated || record ? { repeated, record } : undefined,
+    };
+    return record
+        ? [node, ...flattenSchemaFields(field.fields ?? [], path)]
+        : [node];
+});
 const parseRow = (row) => Object.fromEntries(Object.entries(row).map(([name, value]) => [name, parseCell(value)]));
 class BigquerySqlBuilder extends WarehouseBaseSqlBuilder_1.default {
     constructor() {
@@ -408,11 +458,10 @@ class BigqueryWarehouseClient extends WarehouseBaseClient_1.default {
                 acc[database] = acc[database] || {};
                 acc[database][schema] = acc[database][schema] || {};
                 acc[database][schema][table] = {};
-                tableSchema.fields.forEach(({ name, type }) => {
-                    if (name === undefined)
-                        return;
-                    acc[database][schema][table][name] = mapFieldType(type);
-                    (0, common_1.setCatalogTimestampDomain)(acc, database, schema, table, name, (0, exports.getBigqueryTimestampDomain)(type));
+                flattenSchemaFields(tableSchema.fields).forEach(({ path, type, shape }) => {
+                    acc[database][schema][table][path] = mapFieldType(type);
+                    (0, common_1.setCatalogTimestampDomain)(acc, database, schema, table, path, (0, exports.getBigqueryTimestampDomain)(type));
+                    (0, common_1.setCatalogNestedColumnShape)(acc, database, schema, table, path, shape);
                 });
             }
             return acc;
@@ -474,17 +523,20 @@ class BigqueryWarehouseClient extends WarehouseBaseClient_1.default {
         const dataset = new bigquery_1.Dataset(this.client, schema, {
             projectId: database,
         });
-        const schemas = await BigqueryWarehouseClient.getTableMetadata(dataset, tableName).catch((e) => {
+        const [tableCatalog, tableSchema, table, metadataSchema] = await BigqueryWarehouseClient.getTableMetadata(dataset, tableName).catch((e) => {
             this.throwIfGoogleOauthTokenError(e);
             throw e;
         });
-        return this.parseWarehouseCatalog(schemas[3].fields.map((column) => ({
-            table_catalog: schemas[0],
-            table_schema: schemas[1],
-            table_name: schemas[2],
-            column_name: column.name,
-            data_type: column.type,
+        const flattenedFields = flattenSchemaFields(metadataSchema.fields);
+        const catalog = this.parseWarehouseCatalog(flattenedFields.map(({ path, type }) => ({
+            table_catalog: tableCatalog,
+            table_schema: tableSchema,
+            table_name: table,
+            column_name: path,
+            data_type: type,
         })), mapFieldType, exports.getBigqueryTimestampDomain);
+        flattenedFields.forEach(({ path, shape }) => (0, common_1.setCatalogNestedColumnShape)(catalog, tableCatalog, tableSchema, table, path, shape));
+        return catalog;
     }
     parseError(error, query = '') {
         // if the error has no reason, return a generic error

package/dist/warehouseClients/BigqueryWarehouseClient.mock.d.ts

@@ -9,10 +9,35 @@ export declare const getTableResponse: {
     };
     getMetadata: import("vitest").Mock<() => {
         schema: {
-            fields: {
+            fields: ({
                 name: string;
                 type: BigqueryFieldType;
-            }[];
+                mode?: undefined;
+                fields?: undefined;
+            } | {
+                name: string;
+                type: BigqueryFieldType;
+                mode: string;
+                fields?: undefined;
+            } | {
+                name: string;
+                type: BigqueryFieldType;
+                mode: string;
+                fields: ({
+                    mode?: undefined;
+                    fields?: undefined;
+                    name: string;
+                    type: BigqueryFieldType;
+                } | {
+                    name: string;
+                    type: BigqueryFieldType;
+                    mode: string;
+                    fields: {
+                        name: string;
+                        type: BigqueryFieldType;
+                    }[];
+                })[];
+            })[];
         };
     }[]>;
 };
@@ -21,10 +46,35 @@ declare const mockStreamRow: () => Readable;
 export declare const createJobResponse: {
     getQueryResults: import("vitest").Mock<() => (Record<string, any>[] | {
         schema: {
-            fields: {
+            fields: ({
+                name: string;
+                type: BigqueryFieldType;
+                mode?: undefined;
+                fields?: undefined;
+            } | {
+                name: string;
+                type: BigqueryFieldType;
+                mode: string;
+                fields?: undefined;
+            } | {
                 name: string;
                 type: BigqueryFieldType;
-            }[];
+                mode: string;
+                fields: ({
+                    mode?: undefined;
+                    fields?: undefined;
+                    name: string;
+                    type: BigqueryFieldType;
+                } | {
+                    name: string;
+                    type: BigqueryFieldType;
+                    mode: string;
+                    fields: {
+                        name: string;
+                        type: BigqueryFieldType;
+                    }[];
+                })[];
+            })[];
         };
     } | undefined)[]>;
     getQueryResultsStream: import("vitest").Mock<typeof mockStreamRow>;

package/dist/warehouseClients/BigqueryWarehouseClient.mock.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"BigqueryWarehouseClient.mock.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.mock.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,yBAAyB,EAE5B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,eAAO,MAAM,WAAW,EAAE,yBAUzB,CAAC;AAyCF,eAAO,MAAM,gBAAgB;IACzB,EAAE;IACF,QAAQ;QAAI,SAAS;;IACrB,WAAW;;;;;;;;CACd,CAAC;AAwBF,eAAO,MAAM,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAUzC,CAAC;AAEF,QAAA,MAAM,aAAa,gBAOb,CAAC;AAEP,eAAO,MAAM,iBAAiB;;;;;;;;;;GAK7B,CAAC"}
+{"version":3,"file":"BigqueryWarehouseClient.mock.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/BigqueryWarehouseClient.mock.ts"],"names":[],"mappings":"AAKA,OAAO,EACH,OAAO,EACP,yBAAyB,EAE5B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,eAAO,MAAM,WAAW,EAAE,yBAUzB,CAAC;AA+DF,eAAO,MAAM,gBAAgB;IACzB,EAAE;IACF,QAAQ;QAAI,SAAS;;IACrB,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CACd,CAAC;AAwBF,eAAO,MAAM,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAgBzC,CAAC;AAEF,QAAA,MAAM,aAAa,gBAOb,CAAC;AAEP,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAK7B,CAAC"}

package/dist/warehouseClients/BigqueryWarehouseClient.mock.js

@@ -51,6 +51,28 @@ const metadata = {
                 name: 'myObjectColumn',
                 type: BigqueryWarehouseClient_1.BigqueryFieldType.STRUCT,
             },
+            {
+                name: 'myRepeatedColumn',
+                type: BigqueryWarehouseClient_1.BigqueryFieldType.STRING,
+                mode: 'REPEATED',
+            },
+            {
+                name: 'myRecordColumn',
+                type: BigqueryWarehouseClient_1.BigqueryFieldType.RECORD,
+                mode: 'NULLABLE',
+                fields: [
+                    { name: 'id', type: BigqueryWarehouseClient_1.BigqueryFieldType.INT64 },
+                    { name: 'createdAt', type: BigqueryWarehouseClient_1.BigqueryFieldType.TIMESTAMP },
+                    {
+                        name: 'tags',
+                        type: BigqueryWarehouseClient_1.BigqueryFieldType.RECORD,
+                        mode: 'REPEATED',
+                        fields: [
+                            { name: 'label', type: BigqueryWarehouseClient_1.BigqueryFieldType.STRING },
+                        ],
+                    },
+                ],
+            },
         ],
     },
 };
@@ -85,6 +107,12 @@ exports.rows = [
         myBooleanColumn: false,
         myArrayColumn: ['1', '2', '3'],
         myObjectColumn: { test: '1', toFixed: () => '123' },
+        myRepeatedColumn: ['a', 'b'],
+        myRecordColumn: {
+            id: 7,
+            createdAt: new bigquery_1.BigQueryTimestamp('1990-03-02T08:30:00.010Z'),
+            tags: [{ label: 'x' }, { label: 'y' }],
+        },
     },
 ];
 const mockStreamRow = () => new stream_1.Readable({

package/dist/warehouseClients/BigqueryWarehouseClient.test.js

@@ -13,11 +13,18 @@ describe('BigqueryWarehouseClient', () => {
         expect(results.fields).toEqual({
             ...WarehouseClient_mock_1.expectedFields,
             myBigNumberColumn: { type: 'number' },
+            myRepeatedColumn: { type: 'string' },
+            myRecordColumn: { type: 'string' },
         });
         expect(results.rows[0]).toEqual({
             ...WarehouseClient_mock_1.expectedRow,
             myNumberColumn: 100.25,
             myBigNumberColumn: 200.5,
+            // BigQuery serialises nested values as JSON; other clients pass them through
+            myArrayColumn: '["1","2","3"]',
+            myObjectColumn: '{"test":"1"}',
+            myRepeatedColumn: '["a","b"]',
+            myRecordColumn: '{"id":7,"createdAt":"1990-03-02T08:30:00.010Z","tags":[{"label":"x"},{"label":"y"}]}',
         });
         expect(warehouse.client.createQueryJob).toHaveBeenCalledTimes(1);
     });
@@ -46,12 +53,37 @@ describe('BigqueryWarehouseClient', () => {
         bigquery_1.Dataset.prototype.table = getTableMock;
         const warehouse = new BigqueryWarehouseClient_1.BigqueryWarehouseClient(BigqueryWarehouseClient_mock_1.credentials);
         const expectedCatalog = structuredClone(WarehouseClient_mock_1.expectedWarehouseSchemaWithAwareTimestamp);
-        expectedCatalog.myDatabase.mySchema.myTable.myBigNumberColumn =
-            common_1.DimensionType.NUMBER;
+        Object.assign(expectedCatalog.myDatabase.mySchema.myTable, {
+            myBigNumberColumn: common_1.DimensionType.NUMBER,
+            myRepeatedColumn: common_1.DimensionType.STRING,
+            myRecordColumn: common_1.DimensionType.STRING,
+            'myRecordColumn.id': common_1.DimensionType.NUMBER,
+            'myRecordColumn.createdAt': common_1.DimensionType.TIMESTAMP,
+            'myRecordColumn.tags': common_1.DimensionType.STRING,
+            'myRecordColumn.tags.label': common_1.DimensionType.STRING,
+        });
+        (0, common_1.setCatalogTimestampDomain)(expectedCatalog, 'myDatabase', 'mySchema', 'myTable', 'myRecordColumn.createdAt', 'aware');
+        const nestedShapes = {
+            myObjectColumn: { repeated: false, record: true },
+            myRepeatedColumn: { repeated: true, record: false },
+            myRecordColumn: { repeated: false, record: true },
+            'myRecordColumn.tags': { repeated: true, record: true },
+        };
+        Object.entries(nestedShapes).forEach(([path, shape]) => (0, common_1.setCatalogNestedColumnShape)(expectedCatalog, 'myDatabase', 'mySchema', 'myTable', path, shape));
         expect(await warehouse.getCatalog(WarehouseClient_mock_1.config)).toEqual(expectedCatalog);
         expect(getTableMock).toHaveBeenCalledTimes(1);
         expect(BigqueryWarehouseClient_mock_1.getTableResponse.getMetadata).toHaveBeenCalledTimes(1);
     });
+    it('expect getFields to flatten nested columns with the same shapes as getCatalog', async () => {
+        bigquery_1.Dataset.prototype.table = vi
+            .fn()
+            .mockImplementationOnce(() => BigqueryWarehouseClient_mock_1.getTableResponse);
+        const warehouse = new BigqueryWarehouseClient_1.BigqueryWarehouseClient(BigqueryWarehouseClient_mock_1.credentials);
+        const fields = await warehouse.getFields('myTable', 'mySchema', 'myDatabase');
+        expect(fields.myDatabase.mySchema.myTable['myRecordColumn.id']).toBe(common_1.DimensionType.NUMBER);
+        expect((0, common_1.getCatalogNestedColumnShape)(fields, 'myDatabase', 'mySchema', 'myTable', 'myRecordColumn.tags')).toEqual({ repeated: true, record: true });
+        expect((0, common_1.getCatalogNestedColumnShape)(fields, 'myDatabase', 'mySchema', 'myTable', 'myStringColumn')).toBeUndefined();
+    });
 });
 describe('BigqueryWarehouseClient.getDatabases', () => {
     const createDataset = (datasetId) => new bigquery_1.Dataset(new bigquery_1.BigQuery({ projectId: 'test-project' }), datasetId, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/warehouses",
-  "version": "2.152.0",
+  "version": "2.153.0",
   "description": "Warehouse connectors for Lightdash",
   "license": "MIT",
   "repository": {
@@ -28,7 +28,7 @@
     "snowflake-sdk": "2.3.4",
     "ssh2": "1.14.0",
     "trino-client": "0.2.9",
-    "@lightdash/common": "2.152.0"
+    "@lightdash/common": "2.153.0"
   },
   "devDependencies": {
     "oxlint": "1.75.0",