clickhouse-migrations 0.1.0

package/README.md

@@ -0,0 +1,35 @@
+
+# clickhouse-migrations
+
+> CLI NodeJS ClickHouse Migrations.
+
+## Install
+
+```sh
+npm install --global clickhouse-migrations
+```
+
+## Usage
+
+Create a directory, where migrations will be stored. It will be used as the value for the `--migrations-home` option.
+
+Create migrations in the directory, which should look like `1_`some_text`.sql`, `2_`other_text`.sql`, ... `10_`more_test`.sql`. Here, is important: migration versions followed by underscore - `1_`, `2_`, ...`10_`..., and file extension `.sql`. Any text can follow by version number. The version number should increase for every next migration.
+
+For migrations' content should be used correct SQL ClickHouse queries. A few queries can be used in one migration file. Every query should end with `;`. The queries could be idempotent - for example: `CREATE TABLE IF NOT EXISTS table ...;` Comments should start with `--- `.
+
+```
+  Usage
+    $ clickhouse-migrations migrate <options>
+
+  Required options
+      --host=<name>             Clickhouse hostname in format https://clichouse
+      --port=<number>           Port
+      --user=<name>             Username
+      --pass=<password>         Password
+      --db-name=<database>      Database name
+      --migrations-home=<dir>   Migrations' directory
+
+  Examples
+      clickhouse-migrations migrate --host=http://localhost --port=8123 --user=default --pass='' --db-name=analytics --migrations-home=/app/clickhouse/migrations
+
+```

package/lib/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/lib/cli.js

@@ -0,0 +1,340 @@
+#!/usr/bin/env node
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (_) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var createClient = require('@clickhouse/client').createClient;
+var Command = require('commander').Command;
+var fs = require('fs');
+var crypto = require('crypto');
+// Extract SQL queries from migrations.
+function sql_queries(content) {
+    var queries = content
+        .replace(/\s+/g, ' ')
+        .replace(/(\n|\r\n|\r)/gm, ' ')
+        .split(';')
+        .map(Function.prototype.call, String.prototype.trim)
+        .filter(function (el) {
+        // remove empty and comments
+        return el.length != 0 && !el.startsWith('-- ');
+    });
+    return queries;
+}
+function log(type, message, error) {
+    if (type === void 0) { type = 'info'; }
+    if (type === 'info') {
+        console.log('\x1b[36m', "clickhouse-migrations :", '\x1b[0m', message);
+    }
+    else {
+        console.error('\x1b[36m', "clickhouse-migrations :", '\x1b[31m', "Error: ".concat(message), error ? "\n\n ".concat(error) : '');
+    }
+}
+function connect(host, port, username, password, db_name) {
+    var db_params = {
+        url: host,
+        port: parseInt(port),
+        debug: false,
+        basicAuth: {
+            username: username,
+            password: password,
+        },
+        isUseGzip: false,
+        format: 'json',
+        raw: false,
+        config: {
+            session_timeout: 60,
+            output_format_json_quote_64bit_integers: 0,
+            enable_http_compression: 0,
+        },
+    };
+    if (db_name) {
+        db_params.database = db_name;
+    }
+    return new createClient(db_params);
+}
+function create_db(host, port, username, password, db_name) {
+    return __awaiter(this, void 0, void 0, function () {
+        var client, q, e_1;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    client = connect(host, port, username, password);
+                    q = "CREATE DATABASE IF NOT EXISTS ".concat(db_name, " ENGINE = Atomic");
+                    _a.label = 1;
+                case 1:
+                    _a.trys.push([1, 3, , 4]);
+                    return [4 /*yield*/, client.exec({
+                            query: q,
+                        })];
+                case 2:
+                    _a.sent();
+                    return [3 /*break*/, 4];
+                case 3:
+                    e_1 = _a.sent();
+                    log('error', "can't create the database ".concat(db_name, "."), e_1.message);
+                    process.exit(1);
+                    return [3 /*break*/, 4];
+                case 4: return [4 /*yield*/, client.close()];
+                case 5:
+                    _a.sent();
+                    return [2 /*return*/];
+            }
+        });
+    });
+}
+function init_migration_table(client) {
+    return __awaiter(this, void 0, void 0, function () {
+        var q, e_2;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    q = "CREATE TABLE IF NOT EXISTS _migrations (\n      uid UUID DEFAULT generateUUIDv4(), \n      version UInt32,\n      checksum String, \n      migration_name String, \n      applied_at DateTime DEFAULT now()\n    ) \n    ENGINE = MergeTree \n    ORDER BY tuple(applied_at)";
+                    _a.label = 1;
+                case 1:
+                    _a.trys.push([1, 3, , 4]);
+                    return [4 /*yield*/, client.exec({
+                            query: q,
+                        })];
+                case 2:
+                    _a.sent();
+                    return [3 /*break*/, 4];
+                case 3:
+                    e_2 = _a.sent();
+                    log('error', "can't create the _migrations table.", e_2.message);
+                    process.exit(1);
+                    return [3 /*break*/, 4];
+                case 4: return [2 /*return*/];
+            }
+        });
+    });
+}
+function get_migrations(migrations_home) {
+    var files;
+    try {
+        files = fs.readdirSync(migrations_home);
+    }
+    catch (e) {
+        log('error', "no migration directory ".concat(migrations_home, ". Please create it."));
+        process.exit(1);
+    }
+    var migrations = [];
+    files.forEach(function (file) {
+        var version = Number(file.split('_')[0]);
+        // Manage only .sql files.
+        if (!file.endsWith('.sql'))
+            return;
+        migrations.push({
+            version: version,
+            file: file,
+        });
+    });
+    if (!migrations) {
+        log('error', "no migrations in the ".concat(migrations_home, " migrations directory"));
+    }
+    // Order by version.
+    migrations.sort(function (m1, m2) { return m1.version - m2.version; });
+    return migrations;
+}
+function apply_migrations(client, migrations, migrations_home) {
+    return __awaiter(this, void 0, void 0, function () {
+        var migration_query_result, resultSet, e_3, migrations_applied, applied_migrations, _i, migrations_1, migration_1, content, checksum, queries, _a, queries_1, query, e_4, e_5;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    migration_query_result = [];
+                    _b.label = 1;
+                case 1:
+                    _b.trys.push([1, 4, , 5]);
+                    return [4 /*yield*/, client.query({
+                            query: "SELECT version, checksum, migration_name FROM _migrations ORDER BY version",
+                            format: 'JSONEachRow',
+                        })];
+                case 2:
+                    resultSet = _b.sent();
+                    return [4 /*yield*/, resultSet.json()];
+                case 3:
+                    migration_query_result = _b.sent();
+                    return [3 /*break*/, 5];
+                case 4:
+                    e_3 = _b.sent();
+                    log('error', "can't select data from the _migrations table.", e_3.message);
+                    process.exit(1);
+                    return [3 /*break*/, 5];
+                case 5:
+                    migrations_applied = {};
+                    migration_query_result.forEach(function (row) {
+                        migrations_applied[row.version] = {
+                            checksum: row.checksum,
+                            migration_name: row.migration_name,
+                        };
+                        // Check if migration file was not removed after apply.
+                        var migration_exist = migrations.find(function (_a) {
+                            var version = _a.version;
+                            return version === row.version;
+                        });
+                        if (!migration_exist) {
+                            log('error', "a migration file shouldn't be removed after apply. Please, restore the migration ".concat(migrations_applied.version, " version."));
+                            process.exit(1);
+                        }
+                    });
+                    applied_migrations = '';
+                    _i = 0, migrations_1 = migrations;
+                    _b.label = 6;
+                case 6:
+                    if (!(_i < migrations_1.length)) return [3 /*break*/, 17];
+                    migration_1 = migrations_1[_i];
+                    content = fs.readFileSync(migrations_home + '/' + migration_1.file).toString();
+                    checksum = crypto.createHash('md5').update(content).digest('hex');
+                    if (migrations_applied[migration_1.version]) {
+                        // Check if migration file was not changed after apply.
+                        if (migrations_applied[migration_1.version].checksum !== checksum) {
+                            log('error', "a migration file should't be changed after apply. Please, restore content of the ".concat(migrations_applied[migration_1.version].migration_name, " migrations."));
+                            process.exit(1);
+                        }
+                        // Skip if a migration is already applied.
+                        return [3 /*break*/, 16];
+                    }
+                    queries = sql_queries(content);
+                    _a = 0, queries_1 = queries;
+                    _b.label = 7;
+                case 7:
+                    if (!(_a < queries_1.length)) return [3 /*break*/, 12];
+                    query = queries_1[_a];
+                    _b.label = 8;
+                case 8:
+                    _b.trys.push([8, 10, , 11]);
+                    return [4 /*yield*/, client.exec({
+                            query: query,
+                        })];
+                case 9:
+                    _b.sent();
+                    return [3 /*break*/, 11];
+                case 10:
+                    e_4 = _b.sent();
+                    if (applied_migrations) {
+                        log('info', "The migration(s) ".concat(applied_migrations, " was successfully applied!"));
+                    }
+                    log('error', "the migrations ".concat(migration_1.file, " has an error. Please, fix it (be sure that already executed parts of the migration would not be run second time) and re-run migration script."), e_4.message);
+                    process.exit(1);
+                    return [3 /*break*/, 11];
+                case 11:
+                    _a++;
+                    return [3 /*break*/, 7];
+                case 12:
+                    _b.trys.push([12, 14, , 15]);
+                    return [4 /*yield*/, client.insert({
+                            table: '_migrations',
+                            values: [{ version: migration_1.version, checksum: checksum, migration_name: migration_1.file }],
+                            format: 'JSONEachRow',
+                        })];
+                case 13:
+                    _b.sent();
+                    return [3 /*break*/, 15];
+                case 14:
+                    e_5 = _b.sent();
+                    log('error', "can't insert a data into the table _migrations.", e_5.message);
+                    process.exit(1);
+                    return [3 /*break*/, 15];
+                case 15:
+                    applied_migrations = applied_migrations ? applied_migrations + ', ' + migration_1.file : migration_1.file;
+                    _b.label = 16;
+                case 16:
+                    _i++;
+                    return [3 /*break*/, 6];
+                case 17:
+                    if (applied_migrations) {
+                        log('info', "The migration(s) ".concat(applied_migrations, " was successfully applied!"));
+                    }
+                    else {
+                        log('info', "No migrations to apply.");
+                    }
+                    return [2 /*return*/];
+            }
+        });
+    });
+}
+function migration(migrations_home, host, port, username, password, db_name) {
+    return __awaiter(this, void 0, void 0, function () {
+        var migrations, client;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    migrations = get_migrations(migrations_home);
+                    return [4 /*yield*/, create_db(host, port, username, password, db_name)];
+                case 1:
+                    _a.sent();
+                    client = connect(host, port, username, password, db_name);
+                    return [4 /*yield*/, init_migration_table(client)];
+                case 2:
+                    _a.sent();
+                    return [4 /*yield*/, apply_migrations(client, migrations, migrations_home)];
+                case 3:
+                    _a.sent();
+                    return [4 /*yield*/, client.close()];
+                case 4:
+                    _a.sent();
+                    return [2 /*return*/];
+            }
+        });
+    });
+}
+function migrate() {
+    var _this = this;
+    var program = new Command();
+    program.name('clickhouse-migrations').description('ClickHouse migrations.').version('1.0.0');
+    program
+        .command('migrate')
+        .description('apply migrations.')
+        .requiredOption('--host <name>', 'clickhouse hostname in format http://clickhouse')
+        .requiredOption('--port <number>', 'port')
+        .requiredOption('--user <name>', 'username')
+        .requiredOption('--pass <password>', 'password')
+        .requiredOption('--db-name <dbname>', 'database name')
+        .requiredOption('--migrations-home <dir>', 'migrations directory')
+        .action(function (options) { return __awaiter(_this, void 0, void 0, function () {
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0: return [4 /*yield*/, migration(options.migrationsHome, options.host, options.port, options.user, options.pass, options.dbName)];
+                case 1:
+                    _a.sent();
+                    return [2 /*return*/];
+            }
+        });
+    }); });
+    program.parse();
+}
+migrate();

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "clickhouse-migrations",
+  "version": "0.1.0",
+  "description": "NodeJS ClickHouse Migrations",
+	"bin": {
+		"clickhouse-migrations": "lib/cli.js"
+	},  
+   "scripts": {
+    "build": "tsc",
+    "test": "jest --config jestconfig.json",
+    "format": "prettier --write \"src/**/*.ts\" \"src/**/*.js\"",
+    "lint": "eslint ./src/*.ts",
+    "prepare": "npm run build",
+    "prepublishOnly": "npm test && npm run lint",
+    "preversion": "npm run lint",
+    "version": "npm run format && git add -A src",
+    "postversion": "git push && git push --tags"
+  },
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@clickhouse/client": "^0.0.9",
+    "commander": "^9.4.1"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.2.2",
+    "@types/node": "^18.11.9",
+    "eslint": "^8.26.0",
+    "eslint-config-prettier": "^8.5.0",
+    "jest": "^29.2.2",
+    "prettier": "^2.7.1",
+    "ts-jest": "^29.0.3",
+    "typescript": "^4.8.4"
+  },
+  "files": [
+    "lib/**/*"
+  ]
+}