@cleverbrush/knex-clickhouse 1.1.6

package/README.md

@@ -0,0 +1,3 @@
+# Knex client for ClickHouse.
+
+This is a library to use [knex](https://knexjs.org/) query builder against ClickHouse. It uses `@clickhouse/client` library to connect to ClickHouse.

package/dist/index.d.ts

@@ -0,0 +1,24 @@
+import Knex from 'knex';
+import ClickhouseDriver from '@clickhouse/client';
+export declare class ClickhouseKnexClient extends Knex.Client {
+    #private;
+    _driver(): typeof ClickhouseDriver;
+    constructor(config?: {});
+    _escapeBinding(value: any): any;
+    /**
+     *
+     * @param {import('@clickhouse/client').ClickHouseClient} connection
+     * @param {import('@clickhouse/client').QueryParams} obj
+     * @returns
+     */
+    _query(connection: any, obj: any): Promise<any>;
+    /**
+     *
+     * @param {import('@clickhouse/client').ResultSet} obj
+     * @param {*} runner
+     * @returns
+     */
+    processResponse(obj: any, runner: any): any;
+    acquireRawConnection(): Promise<any>;
+    destroyRawConnection(connection: any): Promise<void>;
+}

package/dist/index.js

@@ -0,0 +1,218 @@
+import Knex from 'knex';
+import ClickhouseDriver from '@clickhouse/client';
+import { makeEscape } from './makeEscape.js';
+const parseDate = (value) => new Date(Date.parse(value));
+// this function is taken from knex npm package
+function arrayString(arr, esc) {
+    let result = '';
+    for (let i = 0; i < arr.length; i++) {
+        if (i > 0)
+            result += ',';
+        const val = arr[i];
+        if (val === null || typeof val === 'undefined') {
+            result += 'NULL';
+        }
+        else if (Array.isArray(val)) {
+            result += arrayString(val, esc);
+        }
+        else if (typeof val === 'number') {
+            result += val;
+        }
+        else {
+            result += esc(val);
+        }
+    }
+    return `[${result}]`;
+}
+export class ClickhouseKnexClient extends Knex.Client {
+    _driver() {
+        return ClickhouseDriver;
+    }
+    constructor(config = {}) {
+        super(config);
+    }
+    #typeParsers = {
+        Date: parseDate,
+        DateTime: parseDate,
+        'LowCardinality(Date)': parseDate,
+        'LowCardinality(DateTime)': parseDate,
+        'Nullable(Date)': parseDate,
+        'Nullable(DateTime)': parseDate,
+        "DateTime('UTC')": parseDate,
+        "Nullable(DateTime('UTC'))": parseDate,
+        'LowCardinality(Nullable(Date))': parseDate,
+        'LowCardinality(Nullable(DateTime))': parseDate,
+        UInt64: (value) => Number(value),
+        UInt32: (value) => Number(value)
+    };
+    #selectQueryRegex = /^(\s+)?(select|with)/i;
+    _escapeBinding(value) {
+        const escapeBinding = makeEscape({
+            escapeArray(val, esc) {
+                return arrayString(val, esc);
+            },
+            escapeString(str) {
+                let escaped = "'";
+                for (let i = 0; i < str.length; i++) {
+                    const c = str[i];
+                    if (c === "'") {
+                        escaped += c + c;
+                    }
+                    else if (c === '\\') {
+                        escaped += c + c;
+                    }
+                    else {
+                        escaped += c;
+                    }
+                }
+                escaped += "'";
+                return escaped;
+            },
+            escapeObject(val, prepareValue, timezone, seen = []) {
+                if (val && typeof val.toPostgres === 'function') {
+                    seen = seen || [];
+                    if (seen.indexOf(val) !== -1) {
+                        throw new Error(`circular reference detected while preparing "${val}" for query`);
+                    }
+                    seen.push(val);
+                    return prepareValue(val.toPostgres(prepareValue), seen);
+                }
+                return JSON.stringify(val);
+            }
+        });
+        const result = escapeBinding(value);
+        if (typeof result === 'string') {
+            return result.replace(/\\/g, '\\\\');
+        }
+        return result;
+    }
+    /**
+     *
+     * @param {import('@clickhouse/client').ClickHouseClient} connection
+     * @param {import('@clickhouse/client').QueryParams} obj
+     * @returns
+     */
+    async _query(connection, obj) {
+        if (!obj || typeof obj === 'string') {
+            obj = { sql: obj };
+        }
+        else if (!obj.sql) {
+            throw new Error('The query is empty');
+        }
+        let { method } = obj;
+        if (method === 'raw' && this.#selectQueryRegex.test(obj.sql)) {
+            method = 'select';
+        }
+        const query = Array.isArray(obj.bindings) && obj.bindings.length > 0
+            ? obj.sql
+                .split('?')
+                .map((x, i) => `${x}${obj.bindings.length > i
+                ? this._escapeBinding(obj.bindings[i])
+                : ''}`)
+                .join('')
+            : obj.sql;
+        let response;
+        const queryParams = {
+            clickhouse_settings: {
+                exact_rows_before_limit: true
+            },
+            query,
+            format: 'JSONCompact'
+        };
+        obj.finalQuery = query;
+        switch (method) {
+            case 'select':
+            case 'first':
+            case 'pluck':
+                {
+                    const p = await connection.query(queryParams);
+                    response = await p.json();
+                    const { data, meta, ...rest } = response;
+                    obj.response = [data, meta, { ...rest }];
+                }
+                break;
+            default:
+                response = await connection.exec(queryParams);
+        }
+        return obj;
+    }
+    /**
+     *
+     * @param {import('@clickhouse/client').ResultSet} obj
+     * @param {*} runner
+     * @returns
+     */
+    processResponse(obj, runner) {
+        if (obj == null)
+            return null;
+        const { response } = obj;
+        const [rows, fields, meta] = response || [[]];
+        const parsers = Array.isArray(fields)
+            ? fields.map((field) => typeof this.#typeParsers[field.type] === 'function'
+                ? this.#typeParsers[field.type]
+                : (x) => x)
+            : fields;
+        const rowToObj = (row) => fields.reduce((acc, field, index) => ({
+            ...acc,
+            [field.name]: parsers[index](row[index])
+        }), {});
+        if (obj.output)
+            return obj.output.call(runner, rows, fields);
+        let { method } = obj;
+        if (method === 'raw' && this.#selectQueryRegex.test(obj.sql)) {
+            method = 'select';
+        }
+        switch (method) {
+            case 'select':
+                return [rows.map((r) => rowToObj(r)), fields, meta];
+            case 'first':
+                return [rowToObj(rows[0]), fields, meta];
+            case 'pluck': {
+                const pluckIndex = fields.findIndex((val) => val.name === obj.pluck);
+                if (pluckIndex === -1)
+                    return rows;
+                return [rows.map((row) => row[pluckIndex]), obj.pluck, meta];
+            }
+            case 'insert':
+            case 'del':
+            case 'update':
+            case 'counter':
+                return response;
+            default:
+                return response;
+        }
+    }
+    /**
+     * @type {import('@clickhouse/client').ClickHouseClient}
+     */
+    #connection;
+    // Get a raw connection, called by the `pool` whenever a new
+    // connection needs to be added to the pool.
+    async acquireRawConnection() {
+        if (!this.#connection) {
+            const connection = ClickhouseDriver.createClient({
+                // @ts-ignore
+                url: this.config.connection.url,
+                // @ts-ignore
+                username: this.config.connection.user,
+                // @ts-ignore
+                password: this.config.connection.password,
+                // @ts-ignore
+                database: this.config.connection.database,
+                compression: { request: true, response: true }
+            });
+            // @ts-ignore
+            connection.connected = true;
+            this.#connection = connection;
+        }
+        return this.#connection;
+    }
+    async destroyRawConnection(connection) {
+        connection.close();
+    }
+}
+Object.assign(ClickhouseKnexClient.prototype, {
+    dialect: 'clikhouse',
+    driverName: 'clikhouse',
+    canCancelQuery: true
+});

package/dist/makeEscape.d.ts

@@ -0,0 +1 @@
+export declare const makeEscape: (config?: any) => any;

package/dist/makeEscape.js

@@ -0,0 +1,158 @@
+/*
+ *  NOTE: functions below are taken from knex npm package,
+ *  unfortunately they are not exported, so we have to copy them here
+ *  because they are useful when it comes to write your own knex dialect
+ */
+const charsRegex = /[\0\b\t\n\r\x1a"'\\]/g; // eslint-disable-line no-control-regex
+const charsMap = {
+    '\0': '\\0',
+    '\b': '\\b',
+    '\t': '\\t',
+    '\n': '\\n',
+    '\r': '\\r',
+    '\x1a': '\\Z',
+    '"': '\\"',
+    "'": "\\'",
+    '\\': '\\\\'
+};
+const wrapEscape = (escapeFn) => function finalEscape(val, ctx = {}) {
+    return escapeFn(val, finalEscape, ctx);
+};
+const escapeObject = (val, finalEscape, ctx) => {
+    if (val && typeof val.toSQL === 'function') {
+        return val.toSQL(ctx);
+    }
+    return JSON.stringify(val);
+};
+// eslint-disable-next-line
+const escapeString = (val) => {
+    // eslint-disable-next-line
+    let chunkIndex = (charsRegex.lastIndex = 0);
+    let escapedVal = '';
+    let match;
+    // eslint-disable-next-line
+    while ((match = charsRegex.exec(val))) {
+        escapedVal += val.slice(chunkIndex, match.index) + charsMap[match[0]];
+        chunkIndex = charsRegex.lastIndex;
+    }
+    if (chunkIndex === 0) {
+        // Nothing was escaped
+        return `'${val}'`;
+    }
+    if (chunkIndex < val.length) {
+        return `'${escapedVal}${val.slice(chunkIndex)}'`;
+    }
+    return `'${escapedVal}'`;
+};
+const bufferToString = (buffer) => `X${escapeString(buffer.toString('hex'))}`;
+const arrayToList = (array, finalEscape, ctx) => {
+    let sql = '';
+    for (let i = 0; i < array.length; i++) {
+        const val = array[i];
+        if (Array.isArray(val)) {
+            sql +=
+                // eslint-disable-next-line
+                (i === 0 ? '' : ', ') +
+                    '(' +
+                    arrayToList(val, finalEscape, ctx) +
+                    ')';
+        }
+        else {
+            sql += (i === 0 ? '' : ', ') + finalEscape(val, ctx);
+        }
+    }
+    return sql;
+};
+const zeroPad = (number, length) => {
+    number = number.toString();
+    while (number.length < length) {
+        number = `0${number}`;
+    }
+    return number;
+};
+const convertTimezone = (tz) => {
+    if (tz === 'Z') {
+        return 0;
+    }
+    const m = tz.match(/([+\-\s])(\d\d):?(\d\d)?/);
+    if (m) {
+        return (
+        // eslint-disable-next-line
+        (m[1] == '-' ? -1 : 1) *
+            (parseInt(m[2], 10) + (m[3] ? parseInt(m[3], 10) : 0) / 60) *
+            60);
+    }
+    return false;
+};
+const dateToString = (date, finalEscape, ctx = {}) => {
+    // @ts-ignore
+    const timeZone = ctx.timeZone || 'local';
+    const dt = new Date(date);
+    let year;
+    let month;
+    let day;
+    let hour;
+    let minute;
+    let second;
+    let millisecond;
+    if (timeZone === 'local') {
+        year = dt.getFullYear();
+        month = dt.getMonth() + 1;
+        day = dt.getDate();
+        hour = dt.getHours();
+        minute = dt.getMinutes();
+        second = dt.getSeconds();
+        millisecond = dt.getMilliseconds();
+    }
+    else {
+        const tz = convertTimezone(timeZone);
+        if (tz !== false && tz !== 0) {
+            dt.setTime(dt.getTime() + tz * 60000);
+        }
+        year = dt.getUTCFullYear();
+        month = dt.getUTCMonth() + 1;
+        day = dt.getUTCDate();
+        hour = dt.getUTCHours();
+        minute = dt.getUTCMinutes();
+        second = dt.getUTCSeconds();
+        millisecond = dt.getUTCMilliseconds();
+    }
+    // YYYY-MM-DD HH:mm:ss.mmm
+    return `${zeroPad(year, 4)}-${zeroPad(month, 2)}-${zeroPad(day, 2)} ${zeroPad(hour, 2)}:${zeroPad(minute, 2)}:${zeroPad(second, 2)}.${zeroPad(millisecond, 3)}`;
+};
+export const makeEscape = (config = {}) => {
+    const finalEscapeDate = config.escapeDate || dateToString;
+    const finalEscapeArray = config.escapeArray || arrayToList;
+    const finalEscapeBuffer = config.escapeBuffer || bufferToString;
+    const finalEscapeString = config.escapeString || escapeString;
+    const finalEscapeObject = config.escapeObject || escapeObject;
+    const finalWrap = config.wrap || wrapEscape;
+    function escapeFn(val, finalEscape, ctx) {
+        if (val === undefined || val === null) {
+            return 'NULL';
+        }
+        // eslint-disable-next-line
+        switch (typeof val) {
+            case 'boolean':
+                return val ? 'true' : 'false';
+            case 'number':
+                // eslint-disable-next-line
+                return val + '';
+            case 'object':
+                if (val instanceof Date) {
+                    val = finalEscapeDate(val, finalEscape, ctx);
+                }
+                else if (Array.isArray(val)) {
+                    return finalEscapeArray(val, finalEscape, ctx);
+                }
+                else if (Buffer.isBuffer(val)) {
+                    return finalEscapeBuffer(val, finalEscape, ctx);
+                }
+                else {
+                    return finalEscapeObject(val, finalEscape, ctx);
+                }
+        }
+        return finalEscapeString(val, finalEscape, ctx);
+    }
+    return finalWrap ? finalWrap(escapeFn) : escapeFn;
+};

package/package.json

@@ -0,0 +1,39 @@
+{
+    "author": "Andrew Zolotukhin <andrew_zol@cleverbrush.com>",
+    "bugs": {
+        "url": "https://github.com/cleverbrush/framework/issues",
+        "email": "andrew_zol@cleverbrush.com"
+    },
+    "dependencies": {
+        "@clickhouse/client": "^1.0.1"
+    },
+    "peerDependencies": {
+        "knex": ">=3.1.0"
+    },
+    "description": "Knex Query Builder for ClickHouse",
+    "files": [
+        "dist"
+    ],
+    "homepage": "CHANGE ME",
+    "keywords": [
+        "knex",
+        "query builder",
+        "clickhouse",
+        "sql"
+    ],
+    "license": "BSD 3-Clause",
+    "main": "./dist/index.js",
+    "name": "@cleverbrush/knex-clickhouse",
+    "readme": "https://github.com/cleverbrush/framework/tree/master/libs/knex-clickhouse#readme",
+    "repository": {
+        "type": "git",
+        "url": "github:cleverbrush/framework"
+    },
+    "scripts": {
+        "watch": "tsc --watch",
+        "build": "tsc"
+    },
+    "type": "module",
+    "types": "./dist/index.d.ts",
+    "version": "1.1.6"
+}