@lightdash/warehouses 2.162.1 → 2.162.3

package/dist/warehouseClients/TrinoWarehouseClient.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TrinoWarehouseClient = exports.TrinoSqlBuilder = exports.getTrinoTimestampDomain = exports.TrinoTypes = void 0;
+exports.TrinoWarehouseClient = exports.TrinoSqlBuilder = exports.getTrinoNumericKind = exports.getTrinoTimestampDomain = exports.TrinoTypes = void 0;
 const common_1 = require("@lightdash/common");
 const trino_client_1 = require("trino-client");
 const coerceTagToString_1 = require("../utils/coerceTagToString");
@@ -77,6 +77,26 @@ const getTrinoTimestampDomain = (type) => {
     }
 };
 exports.getTrinoTimestampDomain = getTrinoTimestampDomain;
+// The column type string carries the decimal scale: decimal(p,s)
+const getTrinoNumericKind = (type) => {
+    switch (removeNumericTypeParameters(type)) {
+        case TrinoTypes.TINYINT:
+        case TrinoTypes.SMALLINT:
+        case TrinoTypes.INTEGER:
+        case TrinoTypes.BIGINT:
+            return { kind: 'integer' };
+        case TrinoTypes.REAL:
+        case TrinoTypes.DOUBLE:
+            return { kind: 'float' };
+        case TrinoTypes.DECIMAL: {
+            const scale = type.match(/\(\s*\d+\s*,\s*(\d+)\s*\)/);
+            return scale ? { kind: 'decimal', scale: Number(scale[1]) } : null;
+        }
+        default:
+            return null;
+    }
+};
+exports.getTrinoNumericKind = getTrinoNumericKind;
 const convertDataTypeToDimensionType = (type) => {
     const typeWithoutParameters = removeNumericTypeParameters(type);
     switch (typeWithoutParameters) {
@@ -263,12 +283,20 @@ class TrinoWarehouseClient extends WarehouseBaseClient_1.default {
                     'Unexpected error in query execution');
             }
             const schema = queryResult.value.columns ?? [];
-            const fields = schema.reduce((acc, column) => ({
-                ...acc,
-                [normalizeColumnName(column.name)]: {
-                    type: convertDataTypeToDimensionType(column.typeSignature.rawType ?? TrinoTypes.VARCHAR),
-                },
-            }), {});
+            const fields = schema.reduce((acc, column) => {
+                // The full type string carries the decimal scale that rawType drops
+                const type = column.type ??
+                    column.typeSignature.rawType ??
+                    TrinoTypes.VARCHAR;
+                const numericKind = (0, exports.getTrinoNumericKind)(type);
+                return {
+                    ...acc,
+                    [normalizeColumnName(column.name)]: {
+                        type: convertDataTypeToDimensionType(type),
+                        ...(numericKind ? { numericKind } : {}),
+                    },
+                };
+            }, {});
             // stream initial data, if available
             if (queryResult.value.data) {
                 await streamCallback({

package/dist/warehouseClients/TrinoWarehouseClient.test.js

@@ -52,6 +52,11 @@ describe('TrinoWarehouseClient', () => {
         acc[key.toLowerCase()] = warehouseClient.expectedFields[key];
         return acc;
     }, {});
+    // The mock's number column is a Trino integer, which reports its kind
+    lowerCaseFields.mynumbercolumn = {
+        ...lowerCaseFields.mynumbercolumn,
+        numericKind: { kind: 'integer' },
+    };
     const lowerCaseRow = Object.keys(warehouseClient.expectedRow).reduce((acc, key) => {
         acc[key.toLowerCase()] = warehouseClient.expectedRow[key];
         return acc;

package/dist/warehouseClients/numericKinds.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=numericKinds.test.d.ts.map

package/dist/warehouseClients/numericKinds.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"numericKinds.test.d.ts","sourceRoot":"","sources":["../../src/warehouseClients/numericKinds.test.ts"],"names":[],"mappings":""}

package/dist/warehouseClients/numericKinds.test.js

@@ -0,0 +1,168 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const TCLIService_types_1 = require("@databricks/sql/thrift/TCLIService_types");
+const vitest_1 = require("vitest");
+const AthenaWarehouseClient_1 = require("./AthenaWarehouseClient");
+const BigqueryWarehouseClient_1 = require("./BigqueryWarehouseClient");
+const ClickhouseWarehouseClient_1 = require("./ClickhouseWarehouseClient");
+const DatabricksWarehouseClient_1 = require("./DatabricksWarehouseClient");
+const SnowflakeWarehouseClient_1 = require("./SnowflakeWarehouseClient");
+const TrinoWarehouseClient_1 = require("./TrinoWarehouseClient");
+// The typed read binds a NUMBER column by the kind its driver reports; a
+// column without one binds as DOUBLE and loses digits above 2^53
+(0, vitest_1.describe)('numeric kinds reported by warehouse drivers', () => {
+    (0, vitest_1.describe)('Snowflake', () => {
+        (0, vitest_1.it)('reads NUMBER with scale 0 as an integer and with a scale as a decimal', () => {
+            (0, vitest_1.expect)((0, SnowflakeWarehouseClient_1.getSnowflakeNumericKind)('FIXED', 0)).toEqual({
+                kind: 'integer',
+            });
+            (0, vitest_1.expect)((0, SnowflakeWarehouseClient_1.getSnowflakeNumericKind)('NUMBER', 4)).toEqual({
+                kind: 'decimal',
+                scale: 4,
+            });
+            (0, vitest_1.expect)((0, SnowflakeWarehouseClient_1.getSnowflakeNumericKind)('fixed', 2)).toEqual({
+                kind: 'decimal',
+                scale: 2,
+            });
+        });
+        (0, vitest_1.it)('reads REAL as a float and reports nothing without a scale or for other types', () => {
+            (0, vitest_1.expect)((0, SnowflakeWarehouseClient_1.getSnowflakeNumericKind)('REAL', 0)).toEqual({
+                kind: 'float',
+            });
+            (0, vitest_1.expect)((0, SnowflakeWarehouseClient_1.getSnowflakeNumericKind)('FIXED', undefined)).toBeNull();
+            (0, vitest_1.expect)((0, SnowflakeWarehouseClient_1.getSnowflakeNumericKind)('TEXT', 0)).toBeNull();
+        });
+    });
+    (0, vitest_1.describe)('BigQuery', () => {
+        (0, vitest_1.it)('reads INT64 as an integer and FLOAT64 as a float', () => {
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({ type: 'INTEGER' })).toEqual({
+                kind: 'integer',
+            });
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({ type: 'INT64' })).toEqual({
+                kind: 'integer',
+            });
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({ type: 'FLOAT64' })).toEqual({
+                kind: 'float',
+            });
+        });
+        (0, vitest_1.it)('reads NUMERIC at its declared scale, else at the default of 9', () => {
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({ type: 'NUMERIC', scale: '2' })).toEqual({ kind: 'decimal', scale: 2 });
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({ type: 'NUMERIC' })).toEqual({
+                kind: 'decimal',
+                scale: 9,
+            });
+        });
+        (0, vitest_1.it)('reads BIGNUMERIC only when its declared precision fits a DuckDB decimal', () => {
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({
+                type: 'BIGNUMERIC',
+                precision: '30',
+                scale: '4',
+            })).toEqual({ kind: 'decimal', scale: 4 });
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({ type: 'BIGNUMERIC' })).toBeNull();
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({ type: 'BIGNUMERIC', scale: '4' })).toBeNull();
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({
+                type: 'BIGNUMERIC',
+                precision: '60',
+                scale: '4',
+            })).toBeNull();
+            (0, vitest_1.expect)((0, BigqueryWarehouseClient_1.getBigqueryNumericKind)({ type: 'STRING' })).toBeNull();
+        });
+    });
+    (0, vitest_1.describe)('Trino', () => {
+        (0, vitest_1.it)('reads the kind and the decimal scale from the column type string', () => {
+            (0, vitest_1.expect)((0, TrinoWarehouseClient_1.getTrinoNumericKind)('bigint')).toEqual({ kind: 'integer' });
+            (0, vitest_1.expect)((0, TrinoWarehouseClient_1.getTrinoNumericKind)('tinyint')).toEqual({
+                kind: 'integer',
+            });
+            (0, vitest_1.expect)((0, TrinoWarehouseClient_1.getTrinoNumericKind)('double')).toEqual({ kind: 'float' });
+            (0, vitest_1.expect)((0, TrinoWarehouseClient_1.getTrinoNumericKind)('real')).toEqual({ kind: 'float' });
+            (0, vitest_1.expect)((0, TrinoWarehouseClient_1.getTrinoNumericKind)('decimal(10,2)')).toEqual({
+                kind: 'decimal',
+                scale: 2,
+            });
+            (0, vitest_1.expect)((0, TrinoWarehouseClient_1.getTrinoNumericKind)('decimal(38, 0)')).toEqual({
+                kind: 'decimal',
+                scale: 0,
+            });
+        });
+        (0, vitest_1.it)('reports nothing for a decimal without a scale or for other types', () => {
+            (0, vitest_1.expect)((0, TrinoWarehouseClient_1.getTrinoNumericKind)('decimal')).toBeNull();
+            (0, vitest_1.expect)((0, TrinoWarehouseClient_1.getTrinoNumericKind)('varchar')).toBeNull();
+        });
+    });
+    (0, vitest_1.describe)('Databricks', () => {
+        (0, vitest_1.it)('reads integers, floats and a decimal with its scale qualifier', () => {
+            (0, vitest_1.expect)((0, DatabricksWarehouseClient_1.getDatabricksNumericKind)({
+                type: TCLIService_types_1.TTypeId.BIGINT_TYPE,
+            })).toEqual({ kind: 'integer' });
+            (0, vitest_1.expect)((0, DatabricksWarehouseClient_1.getDatabricksNumericKind)({
+                type: TCLIService_types_1.TTypeId.DOUBLE_TYPE,
+            })).toEqual({ kind: 'float' });
+            (0, vitest_1.expect)((0, DatabricksWarehouseClient_1.getDatabricksNumericKind)({
+                type: TCLIService_types_1.TTypeId.DECIMAL_TYPE,
+                typeQualifiers: {
+                    qualifiers: {
+                        precision: { i32Value: 18 },
+                        scale: { i32Value: 3 },
+                    },
+                },
+            })).toEqual({ kind: 'decimal', scale: 3 });
+        });
+        (0, vitest_1.it)('reports nothing for a decimal without a scale qualifier or for other types', () => {
+            (0, vitest_1.expect)((0, DatabricksWarehouseClient_1.getDatabricksNumericKind)({
+                type: TCLIService_types_1.TTypeId.DECIMAL_TYPE,
+            })).toBeNull();
+            (0, vitest_1.expect)((0, DatabricksWarehouseClient_1.getDatabricksNumericKind)({
+                type: TCLIService_types_1.TTypeId.STRING_TYPE,
+            })).toBeNull();
+        });
+    });
+    (0, vitest_1.describe)('ClickHouse', () => {
+        (0, vitest_1.it)('reads integers and floats through Nullable and LowCardinality wrappers', () => {
+            (0, vitest_1.expect)((0, ClickhouseWarehouseClient_1.getClickhouseNumericKind)('Int64')).toEqual({
+                kind: 'integer',
+            });
+            (0, vitest_1.expect)((0, ClickhouseWarehouseClient_1.getClickhouseNumericKind)('Nullable(UInt64)')).toEqual({
+                kind: 'integer',
+            });
+            (0, vitest_1.expect)((0, ClickhouseWarehouseClient_1.getClickhouseNumericKind)('LowCardinality(Nullable(Float64))')).toEqual({ kind: 'float' });
+        });
+        (0, vitest_1.it)('reads the scale of Decimal(P, S), the spelling the server sends for every decimal', () => {
+            (0, vitest_1.expect)((0, ClickhouseWarehouseClient_1.getClickhouseNumericKind)('Decimal(18, 4)')).toEqual({
+                kind: 'decimal',
+                scale: 4,
+            });
+            (0, vitest_1.expect)((0, ClickhouseWarehouseClient_1.getClickhouseNumericKind)('Nullable(Decimal(10,2))')).toEqual({ kind: 'decimal', scale: 2 });
+            (0, vitest_1.expect)((0, ClickhouseWarehouseClient_1.getClickhouseNumericKind)('Decimal(38, 10)')).toEqual({
+                kind: 'decimal',
+                scale: 10,
+            });
+        });
+        (0, vitest_1.it)('reports nothing for a decimal wider than 38 digits or for other types', () => {
+            (0, vitest_1.expect)((0, ClickhouseWarehouseClient_1.getClickhouseNumericKind)('Decimal(76, 10)')).toBeNull();
+            (0, vitest_1.expect)((0, ClickhouseWarehouseClient_1.getClickhouseNumericKind)('String')).toBeNull();
+        });
+    });
+    (0, vitest_1.describe)('Athena', () => {
+        (0, vitest_1.it)('reads integers, floats and a decimal with the scale ColumnInfo reports', () => {
+            (0, vitest_1.expect)((0, AthenaWarehouseClient_1.getAthenaNumericKind)('bigint', 0)).toEqual({
+                kind: 'integer',
+            });
+            (0, vitest_1.expect)((0, AthenaWarehouseClient_1.getAthenaNumericKind)('double', 0)).toEqual({
+                kind: 'float',
+            });
+            (0, vitest_1.expect)((0, AthenaWarehouseClient_1.getAthenaNumericKind)('decimal', 2)).toEqual({
+                kind: 'decimal',
+                scale: 2,
+            });
+            (0, vitest_1.expect)((0, AthenaWarehouseClient_1.getAthenaNumericKind)('decimal(10,2)', 2)).toEqual({
+                kind: 'decimal',
+                scale: 2,
+            });
+        });
+        (0, vitest_1.it)('reports nothing for a decimal without a scale or for other types', () => {
+            (0, vitest_1.expect)((0, AthenaWarehouseClient_1.getAthenaNumericKind)('decimal', undefined)).toBeNull();
+            (0, vitest_1.expect)((0, AthenaWarehouseClient_1.getAthenaNumericKind)('varchar', undefined)).toBeNull();
+        });
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/warehouses",
-  "version": "2.162.1",
+  "version": "2.162.3",
   "description": "Warehouse connectors for Lightdash",
   "license": "MIT",
   "repository": {
@@ -28,7 +28,7 @@
     "snowflake-sdk": "2.3.4",
     "ssh2": "1.14.0",
     "trino-client": "0.2.9",
-    "@lightdash/common": "2.162.1"
+    "@lightdash/common": "2.162.3"
   },
   "devDependencies": {
     "oxlint": "1.75.0",