@lightdash/warehouses 0.2019.0 → 0.2021.0

package/dist/ssh/sshTunnel.js

@@ -205,6 +205,7 @@ class SshTunnel {
                 case common_1.WarehouseTypes.TRINO:
                 case common_1.WarehouseTypes.SNOWFLAKE:
                 case common_1.WarehouseTypes.BIGQUERY:
+                case common_1.WarehouseTypes.CLICKHOUSE:
                     break;
                 default:
                     (0, common_1.assertUnreachable)(type, new Error('Unknown warehouse type'));

package/dist/warehouseClientFromCredentials.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.warehouseClientFromCredentials = void 0;
 const common_1 = require("@lightdash/common");
 const BigqueryWarehouseClient_1 = require("./warehouseClients/BigqueryWarehouseClient");
+const ClickhouseWarehouseClient_1 = require("./warehouseClients/ClickhouseWarehouseClient");
 const DatabricksWarehouseClient_1 = require("./warehouseClients/DatabricksWarehouseClient");
 const PostgresWarehouseClient_1 = require("./warehouseClients/PostgresWarehouseClient");
 const RedshiftWarehouseClient_1 = require("./warehouseClients/RedshiftWarehouseClient");
@@ -22,6 +23,8 @@ const warehouseClientFromCredentials = (credentials) => {
             return new DatabricksWarehouseClient_1.DatabricksWarehouseClient(credentials);
         case common_1.WarehouseTypes.TRINO:
             return new TrinoWarehouseClient_1.TrinoWarehouseClient(credentials);
+        case common_1.WarehouseTypes.CLICKHOUSE:
+            return new ClickhouseWarehouseClient_1.ClickhouseWarehouseClient(credentials);
         default:
             const never = credentials;
             throw new common_1.UnexpectedServerError('Warehouse credentials type were not recognised');

package/dist/warehouseClients/ClickhouseWarehouseClient.d.ts

@@ -0,0 +1,71 @@
+import { type ClickHouseClient } from '@clickhouse/client';
+import { AnyType, CreateClickhouseCredentials, Metric, SupportedDbtAdapter, WarehouseResults, WarehouseTypes } from '@lightdash/common';
+import { WarehouseCatalog } from '../types';
+import WarehouseBaseClient from './WarehouseBaseClient';
+import WarehouseBaseSqlBuilder from './WarehouseBaseSqlBuilder';
+export declare enum ClickhouseTypes {
+    UINT8 = "UInt8",
+    UINT16 = "UInt16",
+    UINT32 = "UInt32",
+    UINT64 = "UInt64",
+    INT8 = "Int8",
+    INT16 = "Int16",
+    INT32 = "Int32",
+    INT64 = "Int64",
+    FLOAT32 = "Float32",
+    FLOAT64 = "Float64",
+    DECIMAL = "Decimal",
+    DECIMAL32 = "Decimal32",
+    DECIMAL64 = "Decimal64",
+    DECIMAL128 = "Decimal128",
+    DECIMAL256 = "Decimal256",
+    BOOL = "Bool",
+    STRING = "String",
+    FIXEDSTRING = "FixedString",
+    UUID = "UUID",
+    DATE = "Date",
+    DATE32 = "Date32",
+    DATETIME = "DateTime",
+    DATETIME64 = "DateTime64",
+    ENUM8 = "Enum8",
+    ENUM16 = "Enum16",
+    ARRAY = "Array",
+    TUPLE = "Tuple",
+    MAP = "Map",
+    NULLABLE = "Nullable",
+    LOWCARDINALITY = "LowCardinality",
+    IPV4 = "IPv4",
+    IPV6 = "IPv6"
+}
+interface TableInfo {
+    database: string;
+    schema?: string;
+    table: string;
+}
+export declare class ClickhouseSqlBuilder extends WarehouseBaseSqlBuilder {
+    readonly type = WarehouseTypes.CLICKHOUSE;
+    getAdapterType(): SupportedDbtAdapter;
+    getEscapeStringQuoteChar(): string;
+    getMetricSql(sql: string, metric: Metric): string;
+    escapeString(value: string): string;
+}
+export declare class ClickhouseWarehouseClient extends WarehouseBaseClient<CreateClickhouseCredentials> {
+    client: ClickHouseClient;
+    constructor(credentials: CreateClickhouseCredentials);
+    streamQuery(sql: string, streamCallback: (data: WarehouseResults) => void, options: {
+        queryParams?: Record<string, AnyType>;
+        tags?: Record<string, string>;
+        timezone?: string;
+    }): Promise<void>;
+    getCatalog(requests: TableInfo[]): Promise<WarehouseCatalog>;
+    getTables(schema?: string, tags?: Record<string, string>): Promise<WarehouseCatalog>;
+    getFields(tableName: string, schema?: string, database?: string, // always empty string in clickhouse
+    tags?: Record<string, string>): Promise<WarehouseCatalog>;
+    getAllTables(): Promise<{
+        database: any;
+        schema: any;
+        table: any;
+    }[]>;
+    test(): Promise<void>;
+}
+export {};

package/dist/warehouseClients/ClickhouseWarehouseClient.js

@@ -0,0 +1,313 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClickhouseWarehouseClient = exports.ClickhouseSqlBuilder = exports.ClickhouseTypes = void 0;
+const client_1 = require("@clickhouse/client");
+const common_1 = require("@lightdash/common");
+const processPromisesInBatches_1 = require("../utils/processPromisesInBatches");
+const sql_1 = require("../utils/sql");
+const WarehouseBaseClient_1 = __importDefault(require("./WarehouseBaseClient"));
+const WarehouseBaseSqlBuilder_1 = __importDefault(require("./WarehouseBaseSqlBuilder"));
+var ClickhouseTypes;
+(function (ClickhouseTypes) {
+    ClickhouseTypes["UINT8"] = "UInt8";
+    ClickhouseTypes["UINT16"] = "UInt16";
+    ClickhouseTypes["UINT32"] = "UInt32";
+    ClickhouseTypes["UINT64"] = "UInt64";
+    ClickhouseTypes["INT8"] = "Int8";
+    ClickhouseTypes["INT16"] = "Int16";
+    ClickhouseTypes["INT32"] = "Int32";
+    ClickhouseTypes["INT64"] = "Int64";
+    ClickhouseTypes["FLOAT32"] = "Float32";
+    ClickhouseTypes["FLOAT64"] = "Float64";
+    ClickhouseTypes["DECIMAL"] = "Decimal";
+    ClickhouseTypes["DECIMAL32"] = "Decimal32";
+    ClickhouseTypes["DECIMAL64"] = "Decimal64";
+    ClickhouseTypes["DECIMAL128"] = "Decimal128";
+    ClickhouseTypes["DECIMAL256"] = "Decimal256";
+    ClickhouseTypes["BOOL"] = "Bool";
+    ClickhouseTypes["STRING"] = "String";
+    ClickhouseTypes["FIXEDSTRING"] = "FixedString";
+    ClickhouseTypes["UUID"] = "UUID";
+    ClickhouseTypes["DATE"] = "Date";
+    ClickhouseTypes["DATE32"] = "Date32";
+    ClickhouseTypes["DATETIME"] = "DateTime";
+    ClickhouseTypes["DATETIME64"] = "DateTime64";
+    ClickhouseTypes["ENUM8"] = "Enum8";
+    ClickhouseTypes["ENUM16"] = "Enum16";
+    ClickhouseTypes["ARRAY"] = "Array";
+    ClickhouseTypes["TUPLE"] = "Tuple";
+    ClickhouseTypes["MAP"] = "Map";
+    ClickhouseTypes["NULLABLE"] = "Nullable";
+    ClickhouseTypes["LOWCARDINALITY"] = "LowCardinality";
+    ClickhouseTypes["IPV4"] = "IPv4";
+    ClickhouseTypes["IPV6"] = "IPv6";
+})(ClickhouseTypes || (exports.ClickhouseTypes = ClickhouseTypes = {}));
+const convertDataTypeToDimensionType = (type) => {
+    // Remove nullable wrapper and low cardinality wrapper
+    const cleanType = type
+        .replace(/^Nullable\((.+)\)$/, '$1')
+        .replace(/^LowCardinality\((.+)\)$/, '$1')
+        .replace(/\(\d+\)/, ''); // Remove precision from decimals and fixed strings
+    switch (cleanType) {
+        case ClickhouseTypes.BOOL:
+            return common_1.DimensionType.BOOLEAN;
+        case ClickhouseTypes.UINT8:
+        case ClickhouseTypes.UINT16:
+        case ClickhouseTypes.UINT32:
+        case ClickhouseTypes.UINT64:
+        case ClickhouseTypes.INT8:
+        case ClickhouseTypes.INT16:
+        case ClickhouseTypes.INT32:
+        case ClickhouseTypes.INT64:
+        case ClickhouseTypes.FLOAT32:
+        case ClickhouseTypes.FLOAT64:
+        case ClickhouseTypes.DECIMAL:
+        case ClickhouseTypes.DECIMAL32:
+        case ClickhouseTypes.DECIMAL64:
+        case ClickhouseTypes.DECIMAL128:
+        case ClickhouseTypes.DECIMAL256:
+            return common_1.DimensionType.NUMBER;
+        case ClickhouseTypes.DATE:
+        case ClickhouseTypes.DATE32:
+            return common_1.DimensionType.DATE;
+        case ClickhouseTypes.DATETIME:
+        case ClickhouseTypes.DATETIME64:
+            return common_1.DimensionType.TIMESTAMP;
+        default:
+            return common_1.DimensionType.STRING;
+    }
+};
+const catalogToSchema = (results) => {
+    const warehouseCatalog = {};
+    results.forEach((result) => {
+        result.forEach((row) => {
+            // Map column names from snake_case to camelCase
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            const { table_catalog: database, table_schema: tableSchema, table_name: tableName, column_name: columnName, data_type: dataType, } = row;
+            warehouseCatalog[database] = warehouseCatalog[database] || {};
+            warehouseCatalog[database][tableSchema || 'default'] =
+                warehouseCatalog[database][tableSchema || 'default'] || {};
+            warehouseCatalog[database][tableSchema || 'default'][tableName] =
+                warehouseCatalog[database][tableSchema || 'default'][tableName] || {};
+            warehouseCatalog[database][tableSchema || 'default'][tableName][columnName] = convertDataTypeToDimensionType(dataType);
+        });
+    });
+    return warehouseCatalog;
+};
+class ClickhouseSqlBuilder extends WarehouseBaseSqlBuilder_1.default {
+    constructor() {
+        super(...arguments);
+        this.type = common_1.WarehouseTypes.CLICKHOUSE;
+    }
+    getAdapterType() {
+        return common_1.SupportedDbtAdapter.CLICKHOUSE;
+    }
+    getEscapeStringQuoteChar() {
+        return "'";
+    }
+    getMetricSql(sql, metric) {
+        switch (metric.type) {
+            case common_1.MetricType.PERCENTILE:
+                return `quantile(${(metric.percentile ?? 50) / 100})(${sql})`;
+            case common_1.MetricType.MEDIAN:
+                return `median(${sql})`;
+            default:
+                return super.getMetricSql(sql, metric);
+        }
+    }
+    escapeString(value) {
+        if (typeof value !== 'string') {
+            return value;
+        }
+        return ((0, sql_1.normalizeUnicode)(value)
+            // ClickHouse uses single quote doubling like PostgreSQL
+            .replaceAll("'", "''")
+            // Escape backslashes
+            .replaceAll('\\', '\\\\')
+            // Remove SQL comments
+            .replace(/--.*$/gm, '')
+            .replace(/\/\*[\s\S]*?\*\//g, '')
+            // Remove null bytes
+            .replaceAll('\0', ''));
+    }
+}
+exports.ClickhouseSqlBuilder = ClickhouseSqlBuilder;
+class ClickhouseWarehouseClient extends WarehouseBaseClient_1.default {
+    constructor(credentials) {
+        super(credentials, new ClickhouseSqlBuilder(credentials.startOfWeek));
+        const protocol = credentials.secure ? 'https' : 'http';
+        const url = `${protocol}://${credentials.host}:${credentials.port}`;
+        this.client = (0, client_1.createClient)({
+            url,
+            username: credentials.user,
+            password: credentials.password,
+            database: credentials.schema, // In clickhouse schema = database
+            request_timeout: (credentials.timeoutSeconds || 30) * 1000,
+        });
+    }
+    async streamQuery(sql, streamCallback, options) {
+        try {
+            let alteredQuery = sql;
+            if (options?.tags) {
+                alteredQuery = `${alteredQuery}\n-- ${JSON.stringify(options?.tags)}`;
+            }
+            const resultSet = await this.client.query({
+                query: alteredQuery,
+                format: 'JSONCompactEachRowWithNamesAndTypes',
+                query_params: options?.queryParams,
+                clickhouse_settings: options?.timezone
+                    ? { timezone: options.timezone }
+                    : undefined,
+            });
+            const columnNames = [];
+            const fields = {};
+            const stream = resultSet.stream();
+            stream.on('data', (rows) => {
+                // handle the rest of the rows with results
+                rows.forEach((r) => {
+                    const row = r.json();
+                    // handle first row with column names
+                    if (columnNames.length === 0) {
+                        row.map((c) => {
+                            if (typeof c === 'string') {
+                                return columnNames.push(c);
+                            }
+                            return columnNames.push(String(c));
+                        });
+                    }
+                    else if (Object.keys(fields).length === 0) {
+                        // handle second row with column types
+                        columnNames.forEach((c, index) => {
+                            fields[c] = {
+                                type: convertDataTypeToDimensionType(String(row[index])),
+                            };
+                        });
+                        streamCallback({
+                            fields,
+                            rows: [],
+                        });
+                    }
+                    else {
+                        streamCallback({
+                            fields,
+                            rows: [
+                                // convert value array to object
+                                columnNames.reduce((acc, c, index) => {
+                                    acc[c] = row[index];
+                                    return acc;
+                                }, {}),
+                            ],
+                        });
+                    }
+                });
+            });
+            await new Promise((resolve, reject) => {
+                stream.on('end', () => {
+                    resolve();
+                });
+                stream.on('error', reject);
+            });
+        }
+        catch (e) {
+            throw new common_1.WarehouseQueryError((0, common_1.getErrorMessage)(e));
+        }
+    }
+    async getCatalog(requests) {
+        let results;
+        const query = `SELECT 
+                            '' as "table_catalog",
+                            database as "table_schema",
+                            table as "table_name",
+                            name as "column_name",
+                            type as "data_type"
+                        FROM system.columns
+                        WHERE database = {databaseName: String}
+                          AND table = {tableName: String}
+                        ORDER BY position`;
+        try {
+            results = await (0, processPromisesInBatches_1.processPromisesInBatches)(requests, processPromisesInBatches_1.DEFAULT_BATCH_SIZE, async (request) => {
+                const { rows } = await this.runQuery(query, {}, undefined, undefined, {
+                    databaseName: request.schema,
+                    tableName: request.table,
+                });
+                return rows;
+            });
+        }
+        catch (e) {
+            throw new common_1.WarehouseQueryError((0, common_1.getErrorMessage)(e));
+        }
+        return catalogToSchema(results);
+    }
+    async getTables(schema, tags) {
+        const databaseName = schema || this.credentials.schema; // In clickhouse schema = database
+        const query = `
+            SELECT 
+                '' as "table_catalog",
+                database as "table_schema",
+                name as "table_name"
+            FROM system.tables
+            WHERE database = {databaseName: String}
+            ORDER BY database, name
+        `;
+        const { rows } = await this.runQuery(query, tags, undefined, undefined, {
+            databaseName,
+        });
+        return this.parseWarehouseCatalog(rows, convertDataTypeToDimensionType);
+    }
+    async getFields(tableName, schema, database, // always empty string in clickhouse
+    tags) {
+        const dbName = schema || this.credentials.schema; // In clickhouse schema = database
+        const query = `
+            SELECT 
+                '' as "table_catalog",
+                database as "table_schema",
+                table as "table_name",
+                name as "column_name",
+                type as "data_type"
+            FROM system.columns
+            WHERE database = {databaseName: String}
+              AND table = {tableName: String}
+            ORDER BY position
+        `;
+        const { rows } = await this.runQuery(query, tags, undefined, undefined, {
+            databaseName: dbName,
+            tableName,
+        });
+        return this.parseWarehouseCatalog(rows, convertDataTypeToDimensionType);
+    }
+    async getAllTables() {
+        const databaseName = this.credentials.schema; // In clickhouse schema = database
+        const query = `
+            SELECT 
+                '' as "table_catalog",
+                database as "table_schema",
+                name as "table_name"
+            FROM system.tables
+            WHERE database = {databaseName: String}
+            ORDER BY database, name
+        `;
+        const { rows } = await this.runQuery(query, {}, undefined, undefined, {
+            databaseName,
+        });
+        return rows.map((row) => ({
+            database: row.table_catalog,
+            schema: row.table_schema || 'default',
+            table: row.table_name,
+        }));
+    }
+    async test() {
+        try {
+            await this.client.query({
+                query: 'SELECT 1 as test',
+                format: 'JSON',
+            });
+        }
+        catch (e) {
+            throw new common_1.WarehouseConnectionError((0, common_1.getErrorMessage)(e));
+        }
+    }
+}
+exports.ClickhouseWarehouseClient = ClickhouseWarehouseClient;

package/dist/warehouseClients/WarehouseBaseClient.d.ts

@@ -16,11 +16,12 @@ export default abstract class WarehouseBaseClient<T extends CreateWarehouseCrede
     getAsyncQueryResults<TFormattedRow extends Record<string, unknown>>(_args: WarehouseGetAsyncQueryResultsArgs, _rowFormatter?: (row: Record<string, unknown>) => TFormattedRow): Promise<WarehouseGetAsyncQueryResults<TFormattedRow>>;
     abstract streamQuery(query: string, streamCallback: (data: WarehouseResults) => void, options: {
         values?: AnyType[];
+        queryParams?: Record<string, AnyType>;
         tags?: Record<string, string>;
         timezone?: string;
     }): Promise<void>;
-    executeAsyncQuery({ sql, values, tags, timezone }: WarehouseExecuteAsyncQueryArgs, resultsStreamCallback: (rows: WarehouseResults['rows'], fields: WarehouseResults['fields']) => void): Promise<WarehouseExecuteAsyncQuery>;
-    runQuery(sql: string, tags?: Record<string, string>, timezone?: string, values?: AnyType[]): Promise<{
+    executeAsyncQuery({ sql, values, queryParams, tags, timezone, }: WarehouseExecuteAsyncQueryArgs, resultsStreamCallback: (rows: WarehouseResults['rows'], fields: WarehouseResults['fields']) => void): Promise<WarehouseExecuteAsyncQuery>;
+    runQuery(sql: string, tags?: Record<string, string>, timezone?: string, values?: AnyType[], queryParams?: Record<string, AnyType>): Promise<{
         fields: Record<string, {
             type: DimensionType;
         }>;

package/dist/warehouseClients/WarehouseBaseClient.js

@@ -21,7 +21,7 @@ class WarehouseBaseClient {
     async getAsyncQueryResults(_args, _rowFormatter) {
         throw new common_1.NotImplementedError(`Paginated query results are not supported for warehouse type: ${this.getAdapterType()}`);
     }
-    async executeAsyncQuery({ sql, values, tags, timezone }, resultsStreamCallback) {
+    async executeAsyncQuery({ sql, values, queryParams, tags, timezone, }, resultsStreamCallback) {
         let rowCount = 0;
         const startTime = performance.now();
         await this.streamQuery(sql, ({ rows, fields }) => {
@@ -29,6 +29,7 @@ class WarehouseBaseClient {
             resultsStreamCallback(rows, fields);
         }, {
             values,
+            queryParams,
             tags,
             timezone,
         });
@@ -40,7 +41,7 @@ class WarehouseBaseClient {
             totalRows: rowCount,
         };
     }
-    async runQuery(sql, tags, timezone, values) {
+    async runQuery(sql, tags, timezone, values, queryParams) {
         let fields = {};
         const rows = [];
         await this.streamQuery(sql, (data) => {
@@ -48,6 +49,7 @@ class WarehouseBaseClient {
             rows.push(...data.rows);
         }, {
             values,
+            queryParams,
             tags,
             timezone,
         });

package/dist/warehouseSqlBuilderFromType.d.ts

@@ -1,7 +1,8 @@
 import { SupportedDbtAdapter } from '@lightdash/common';
 import { BigquerySqlBuilder } from './warehouseClients/BigqueryWarehouseClient';
+import { ClickhouseSqlBuilder } from './warehouseClients/ClickhouseWarehouseClient';
 import { DatabricksSqlBuilder } from './warehouseClients/DatabricksWarehouseClient';
 import { PostgresSqlBuilder } from './warehouseClients/PostgresWarehouseClient';
 import { SnowflakeSqlBuilder } from './warehouseClients/SnowflakeWarehouseClient';
 import { TrinoSqlBuilder } from './warehouseClients/TrinoWarehouseClient';
-export declare const warehouseSqlBuilderFromType: (adapterType: string | SupportedDbtAdapter, startOfWeek?: import("@lightdash/common").WeekDay | null | undefined) => BigquerySqlBuilder | DatabricksSqlBuilder | PostgresSqlBuilder | SnowflakeSqlBuilder | TrinoSqlBuilder;
+export declare const warehouseSqlBuilderFromType: (adapterType: string | SupportedDbtAdapter, startOfWeek?: import("@lightdash/common").WeekDay | null | undefined) => BigquerySqlBuilder | ClickhouseSqlBuilder | DatabricksSqlBuilder | PostgresSqlBuilder | SnowflakeSqlBuilder | TrinoSqlBuilder;

package/dist/warehouseSqlBuilderFromType.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.warehouseSqlBuilderFromType = void 0;
 const common_1 = require("@lightdash/common");
 const BigqueryWarehouseClient_1 = require("./warehouseClients/BigqueryWarehouseClient");
+const ClickhouseWarehouseClient_1 = require("./warehouseClients/ClickhouseWarehouseClient");
 const DatabricksWarehouseClient_1 = require("./warehouseClients/DatabricksWarehouseClient");
 const PostgresWarehouseClient_1 = require("./warehouseClients/PostgresWarehouseClient");
 const RedshiftWarehouseClient_1 = require("./warehouseClients/RedshiftWarehouseClient");
@@ -15,6 +16,8 @@ const warehouseSqlBuilderFromType = (adapterType, ...args) => {
     switch (adapterType) {
         case common_1.SupportedDbtAdapter.BIGQUERY:
             return new BigqueryWarehouseClient_1.BigquerySqlBuilder(...args);
+        case common_1.SupportedDbtAdapter.CLICKHOUSE:
+            return new ClickhouseWarehouseClient_1.ClickhouseSqlBuilder(...args);
         case common_1.SupportedDbtAdapter.DATABRICKS:
             return new DatabricksWarehouseClient_1.DatabricksSqlBuilder(...args);
         case common_1.SupportedDbtAdapter.POSTGRES:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/warehouses",
-  "version": "0.2019.0",
+  "version": "0.2021.0",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -8,6 +8,7 @@
     "dist/**/*"
   ],
   "dependencies": {
+    "@clickhouse/client": "^1.12.1",
     "@databricks/sql": "1.8.4",
     "@google-cloud/bigquery": "^7.9.2",
     "lodash": "^4.17.21",
@@ -16,7 +17,7 @@
     "snowflake-sdk": "~2.1.3",
     "ssh2": "^1.14.0",
     "trino-client": "0.2.6",
-    "@lightdash/common": "0.2019.0"
+    "@lightdash/common": "0.2021.0"
   },
   "devDependencies": {
     "@types/pg": "^8.11.10",