clickhouse-migrations 0.1.8 → 0.1.10

package/README.md

@@ -1,7 +1,7 @@
 
 # clickhouse-migrations
 
-> CLI NodeJS ClickHouse Migrations.
+> ClickHouse Migrations CLI
 
 ## Install
 
@@ -13,9 +13,9 @@ npm install --global clickhouse-migrations
 
 Create a directory, where migrations will be stored. It will be used as the value for the `--migrations-home` option.
 
-In the directory, create migrations - which should be named like: `1_`some_text`.sql`, `2_`other_text`.sql`, ... `10_`more_test`.sql`. What is important here: migration versions followed by underscore - `1_`, `2_`, ...`10_`..., and file extension `.sql`. Any text can follow by version number. The version number should increase for every next migration.
+In the directory, create migrations files - which should be named like: `1_`some_text`.sql`, `2_`other_text`.sql`, ... `10_`more_test`.sql`. What is important here: migration versions followed by underscore - `1_`, `2_`, ...`10_`..., and file extension `.sql`. Any text can follow by version number. The version number should increase for every next migration. 
 
-For migrations' content should be used correct SQL ClickHouse queries. A few queries can be used in one migration file. Every query should end with `;`. The queries could be idempotent - for example: `CREATE TABLE IF NOT EXISTS table ...;` For adding comments should be used `--`, `# `, `#!` at the beginning of the line.
+For migrations' content should be used correct SQL ClickHouse queries. A few queries can be used in one migration file. Every query should end with `;`. The queries could be idempotent - for example: `CREATE TABLE IF NOT EXISTS table ...;` For adding comments should be used `--`, `# `, `#!` at the beginning of the line. Can be used ClickHouse settings - that can be applied on query level, in the format: `SET allow_experimental_object_type = 1;` A migration file can't be changed or removed after applying.
 
 ```
   Usage
@@ -37,9 +37,19 @@ For migrations' content should be used correct SQL ClickHouse queries. A few que
       CH_MIGRATIONS_DB          Database name (--db)
       CH_MIGRATIONS_HOME        Migrations' directory (--migrations-home)
 
-  Examples
+  CLI examples
       clickhouse-migrations migrate --host=http://localhost:8123 
       --user=default --password='' --db=analytics 
       --migrations-home=/app/clickhouse/migrations
 
       clickhouse-migrations migrate 
+  
+  Migration file example (could be named 'init-1.sql' and stored in the
+  directory provided in --migrations-home)
+
+      SET allow_experimental_object_type = 1;
+
+      CREATE TABLE IF NOT EXISTS events (
+        event JSON
+      );
+```      

package/lib/cli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-var migrate = require('./migrate').migrate;
-migrate();
+var migrate_1 = require("./migrate");
+(0, migrate_1.migrate)();

package/lib/migrate.d.ts

@@ -1 +1,3 @@
-export declare const migrate: () => void;
+declare const migration: (migrations_home: string, host: string, username: string, password: string, db_name: string) => Promise<void>;
+declare const migrate: () => void;
+export { migrate, migration };

package/lib/migrate.js

@@ -14,7 +14,7 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
     function verb(n) { return function (v) { return step([n, v]); }; }
     function step(op) {
         if (f) throw new TypeError("Generator is already executing.");
-        while (_) try {
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
             if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
             if (y = 0, t) op = [op[0] & 2, t.value];
             switch (op[0]) {
@@ -35,23 +35,16 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
         if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
     }
 };
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.migrate = void 0;
-var createClient = require('@clickhouse/client').createClient;
-var Command = require('commander').Command;
-var fs = require('fs');
-var crypto = require('crypto');
-// Extract SQL queries from migrations.
-var sql_queries = function (content) {
-    var queries = content
-        .replace(/^(--|#!|# ).*(\n|\r\n|\r)$/gm, '')
-        .replace(/(\n|\r\n|\r)/gm, ' ')
-        .replace(/\s+/g, ' ')
-        .split(';')
-        .map(function (el) { return el.trim(); })
-        .filter(function (el) { return el.length != 0; });
-    return queries;
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
 };
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.migration = exports.migrate = void 0;
+var client_1 = require("@clickhouse/client");
+var commander_1 = require("commander");
+var fs_1 = __importDefault(require("fs"));
+var crypto_1 = __importDefault(require("crypto"));
+var sql_parse_1 = require("./sql-parse");
 var log = function (type, message, error) {
     if (type === void 0) { type = 'info'; }
     if (type === 'info') {
@@ -71,7 +64,7 @@ var connect = function (host, username, password, db_name) {
     if (db_name) {
         db_params.database = db_name;
     }
-    return new createClient(db_params);
+    return (0, client_1.createClient)(db_params);
 };
 var create_db = function (host, username, password, db_name) { return __awaiter(void 0, void 0, void 0, function () {
     var client, q, e_1;
@@ -85,6 +78,9 @@ var create_db = function (host, username, password, db_name) { return __awaiter(
                 _a.trys.push([1, 3, , 4]);
                 return [4 /*yield*/, client.exec({
                         query: q,
+                        clickhouse_settings: {
+                            wait_end_of_query: 1,
+                        },
                     })];
             case 2:
                 _a.sent();
@@ -112,6 +108,9 @@ var init_migration_table = function (client) { return __awaiter(void 0, void 0,
                 _a.trys.push([1, 3, , 4]);
                 return [4 /*yield*/, client.exec({
                         query: q,
+                        clickhouse_settings: {
+                            wait_end_of_query: 1,
+                        },
                     })];
             case 2:
                 _a.sent();
@@ -128,7 +127,7 @@ var init_migration_table = function (client) { return __awaiter(void 0, void 0,
 var get_migrations = function (migrations_home) {
     var files;
     try {
-        files = fs.readdirSync(migrations_home);
+        files = fs_1.default.readdirSync(migrations_home);
     }
     catch (e) {
         log('error', "no migration directory ".concat(migrations_home, ". Please create it."));
@@ -137,6 +136,10 @@ var get_migrations = function (migrations_home) {
     var migrations = [];
     files.forEach(function (file) {
         var version = Number(file.split('_')[0]);
+        if (!version) {
+            log('error', "a migration name should start from number, example: 1_init.sql. Please check, if the migration ".concat(file, " is named correctly"));
+            process.exit(1);
+        }
         // Manage only .sql files.
         if (!file.endsWith('.sql'))
             return;
@@ -153,7 +156,7 @@ var get_migrations = function (migrations_home) {
     return migrations;
 };
 var apply_migrations = function (client, migrations, migrations_home) { return __awaiter(void 0, void 0, void 0, function () {
-    var migration_query_result, resultSet, e_3, migrations_applied, applied_migrations, _i, migrations_1, migration_1, content, checksum, queries, _a, queries_1, query, e_4, e_5;
+    var migration_query_result, resultSet, e_3, migrations_applied, applied_migrations, _i, migrations_1, migration_1, content, checksum, queries, sets, _a, queries_1, query, e_4, e_5;
     return __generator(this, function (_b) {
         switch (_b.label) {
             case 0:
@@ -177,9 +180,10 @@ var apply_migrations = function (client, migrations, migrations_home) { return _
                 process.exit(1);
                 return [3 /*break*/, 5];
             case 5:
-                migrations_applied = {};
+                migrations_applied = [];
                 migration_query_result.forEach(function (row) {
                     migrations_applied[row.version] = {
+                        version: row.version,
                         checksum: row.checksum,
                         migration_name: row.migration_name,
                     };
@@ -199,8 +203,8 @@ var apply_migrations = function (client, migrations, migrations_home) { return _
             case 6:
                 if (!(_i < migrations_1.length)) return [3 /*break*/, 17];
                 migration_1 = migrations_1[_i];
-                content = fs.readFileSync(migrations_home + '/' + migration_1.file).toString();
-                checksum = crypto.createHash('md5').update(content).digest('hex');
+                content = fs_1.default.readFileSync(migrations_home + '/' + migration_1.file).toString();
+                checksum = crypto_1.default.createHash('md5').update(content).digest('hex');
                 if (migrations_applied[migration_1.version]) {
                     // Check if migration file was not changed after apply.
                     if (migrations_applied[migration_1.version].checksum !== checksum) {
@@ -210,7 +214,8 @@ var apply_migrations = function (client, migrations, migrations_home) { return _
                     // Skip if a migration is already applied.
                     return [3 /*break*/, 16];
                 }
-                queries = sql_queries(content);
+                queries = (0, sql_parse_1.sql_queries)(content);
+                sets = (0, sql_parse_1.sql_sets)(content);
                 _a = 0, queries_1 = queries;
                 _b.label = 7;
             case 7:
@@ -221,6 +226,7 @@ var apply_migrations = function (client, migrations, migrations_home) { return _
                 _b.trys.push([8, 10, , 11]);
                 return [4 /*yield*/, client.exec({
                         query: query,
+                        clickhouse_settings: sets,
                     })];
             case 9:
                 _b.sent();
@@ -291,9 +297,10 @@ var migration = function (migrations_home, host, username, password, db_name) {
         }
     });
 }); };
+exports.migration = migration;
 var migrate = function () {
-    var program = new Command();
-    program.name('clickhouse-migrations').description('ClickHouse migrations.').version('0.1.8');
+    var program = new commander_1.Command();
+    program.name('clickhouse-migrations').description('ClickHouse migrations.').version('0.1.10');
     program
         .command('migrate')
         .description('Apply migrations.')

package/lib/sql-parse.d.ts

@@ -0,0 +1,5 @@
+declare const sql_queries: (content: string) => string[];
+declare const sql_sets: (content: string) => {
+    [key: string]: string;
+};
+export { sql_queries, sql_sets };

package/lib/sql-parse.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sql_sets = exports.sql_queries = void 0;
+// Extract sql queries from migrations.
+var sql_queries = function (content) {
+    var queries = content
+        .replace(/^(--|#!|#\s|SET\s).*(\n|\r\n|\r|$)/gm, '')
+        .replace(/(\n|\r\n|\r)/gm, ' ')
+        .replace(/\s+/g, ' ')
+        .split(';')
+        .map(function (el) { return el.trim(); })
+        .filter(function (el) { return el.length != 0; });
+    return queries;
+};
+exports.sql_queries = sql_queries;
+// Extract query settings from migrations.
+var sql_sets = function (content) {
+    var sets = {};
+    var sets_arr = content
+        .replace(/^(?!SET\s).*(\n|\r\n|\r|$)/gm, '')
+        .replace(/^(SET\s)/gm, '')
+        .replace(/(\n|\r\n|\r)/gm, ' ')
+        .replace(/\s+/g, '')
+        .split(';');
+    sets_arr.forEach(function (set_full) {
+        var set = set_full.split('=');
+        sets[set[0]] = set[1];
+    });
+    return sets;
+};
+exports.sql_sets = sql_sets;

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "clickhouse-migrations",
-  "version": "0.1.8",
-  "description": "CLI NodeJS ClickHouse Migrations",
+  "version": "0.1.10",
+  "description": "ClickHouse Migrations",
   "bin": {
     "clickhouse-migrations": "lib/cli.js"
   },
   "scripts": {
-    "build": "tsc",
+    "build": "rm lib/*; tsc",
     "test": "jest --config jestconfig.json",
     "format": "prettier --write \"src/**/*.ts\" \"src/**/*.js\"",
-    "lint": "eslint ./src/*.ts",
+    "lint": "eslint ./src --ext .ts",
     "prepare": "npm run build",
     "prepublishOnly": "npm test && npm run lint",
     "preversion": "npm run lint",
@@ -20,19 +20,21 @@
   "license": "MIT",
   "repository": "VVVi/clickhouse-migrations",
   "dependencies": {
-    "@clickhouse/client": "^0.0.9",
-    "commander": "^9.4.1"
+    "@clickhouse/client": "^0.0.13",
+    "commander": "^10.0.0"
   },
   "devDependencies": {
-    "@types/jest": "^29.2.2",
-    "@types/node": "^18.11.9",
-    "@typescript-eslint/parser": "^5.43.0",
-    "eslint": "^8.26.0",
-    "eslint-config-prettier": "^8.5.0",
-    "jest": "^29.2.2",
-    "prettier": "^2.7.1",
-    "ts-jest": "^29.0.3",
-    "typescript": "^4.8.4"
+    "@types/jest": "^29.4.1",
+    "@types/node": "^18.15.3",
+    "@typescript-eslint/parser": "^5.55.0",
+    "@typescript-eslint/eslint-plugin": "^5.55.0",
+    "eslint-plugin-prettier": "^4.2.1",
+    "eslint": "^8.36.0",
+    "eslint-config-prettier": "^8.7.0",
+    "jest": "^29.5.0",
+    "prettier": "^2.8.4",
+    "ts-jest": "^29.0.5",
+    "typescript": "^4.9.5"
   },
   "files": [
     "lib/**/*"