clickhouse-migrations 1.3.1 → 1.4.0

package/README.md

@@ -63,6 +63,10 @@ For TLS/HTTPS connections, you can provide a custom CA certificate and optional
                                 (optional) Skip database creation,
                                   set to 'true' to enable
                                   (--skip-db-creation)
+      CH_MIGRATIONS_SUBSTITUTE_ENV
+                                (optional) Substitute ${VAR} in migration
+                                  files from the environment at apply time,
+                                  set to 'true' to enable
 
 
   CLI executions examples
@@ -99,7 +103,7 @@ For TLS/HTTPS connections, you can provide a custom CA certificate and optional
 Migration file example:
 (e.g., located at /app/clickhouse/migrations/1_init.sql)
 
-```
+```sql
 -- an example of migration file 1_init.sql
 
 SET allow_experimental_json_type = 1;
@@ -115,3 +119,34 @@ SAMPLE BY session_id
 ORDER BY (session_id)
 SETTINGS index_granularity = 8192;
 ```
+
+### Environment variable substitution (optional)
+
+Some migrations need a value that differs per environment, for example: the host of a Postgres instance behind a `DICTIONARY`, a cluster name, a bucket, and so on.
+
+Enable `CH_MIGRATIONS_SUBSTITUTE_ENV=true` and `${VAR}` placeholders in migration files are replaced with the matching environment variables when the migration is applied. It's off by default, so existing migrations behave exactly as before.
+
+Rules:
+
+- `${NAME}` is replaced with the value of the `NAME` environment variable.
+- If `NAME` is unset, the migration fails - the placeholder is never left behind in the SQL.
+- To keep a literal `${NAME}` (for example in a comment or a string), escape it as `$${NAME}`.
+- A malformed (`${PG-HOST}`, `${}`) or unterminated (`${PG_HOST`) placeholder also fails, so nothing is ever silently passed through.
+
+Substitution is textual (like `envsubst`): it doesn't parse SQL and doesn't escape values, so anything placed inside quotes must be quoting-safe.
+
+A typical use is an idempotent dictionary pointing at a per-environment source:
+
+```sql
+CREATE OR REPLACE DICTIONARY my_db.dict_offers
+(
+    `id` UUID,
+    `name` String DEFAULT ''
+)
+PRIMARY KEY id
+SOURCE(POSTGRESQL(HOST '${PG_HOST}' PORT ${PG_PORT} USER '${PG_USER}' PASSWORD '${PG_PASSWORD}' DB '${PG_DB}' TABLE 'offers'))
+LIFETIME(MIN 0 MAX 300)
+LAYOUT(COMPLEX_KEY_HASHED());
+```
+
+_Please note:_ When substitution is enabled, the checksum is taken from the **raw file**, before substitution. This keeps a migration stable across environments and keeps secrets out of `_migrations`. Two consequences follow: an already-applied migration is not re-run when a variable changes (ship a new migration instead), and the substituted SQL still reaches ClickHouse, so it may appear in `system.query_log` or server logs.

package/lib/cli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-var migrate_1 = require("./migrate");
+const migrate_1 = require("./migrate");
 (0, migrate_1.migrate)();

package/lib/migrate.js

@@ -1,74 +1,28 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
-    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
-    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
-        if (ar || !(i in from)) {
-            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
-            ar[i] = from[i];
-        }
-    }
-    return to.concat(ar || Array.prototype.slice.call(from));
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.migration = exports.migrate = void 0;
-var client_1 = require("@clickhouse/client");
-var commander_1 = require("commander");
-var fs_1 = __importDefault(require("fs"));
-var crypto_1 = __importDefault(require("crypto"));
-var sql_parse_1 = require("./sql-parse");
-var version_1 = require("./version");
-var log = function (type, message, error) {
-    if (type === void 0) { type = 'info'; }
+const client_1 = require("@clickhouse/client");
+const commander_1 = require("commander");
+const fs_1 = __importDefault(require("fs"));
+const crypto_1 = __importDefault(require("crypto"));
+const sql_parse_1 = require("./sql-parse");
+const version_1 = require("./version");
+const log = (type = 'info', message, error) => {
     if (type === 'info') {
-        console.log('\x1b[36m', "clickhouse-migrations :", '\x1b[0m', message);
+        console.log('\x1b[36m', `clickhouse-migrations :`, '\x1b[0m', message);
     }
     else {
-        console.error('\x1b[36m', "clickhouse-migrations :", '\x1b[31m', "Error: ".concat(message), error ? "\n\n ".concat(error) : '');
+        console.error('\x1b[36m', `clickhouse-migrations :`, '\x1b[31m', `Error: ${message}`, error ? `\n\n ${error}` : '');
     }
 };
-var connect = function (url, username, password, db_name, timeout, ca_cert, cert, key) {
-    var db_params = {
-        url: url,
-        username: username,
-        password: password,
+const connect = (url, username, password, db_name, timeout, ca_cert, cert, key) => {
+    const db_params = {
+        url,
+        username,
+        password,
         application: 'clickhouse-migrations',
     };
     if (db_name) {
@@ -99,301 +53,213 @@ var connect = function (url, username, password, db_name, timeout, ca_cert, cert
     }
     return (0, client_1.createClient)(db_params);
 };
-var create_db = function (host, username, password, db_name, db_engine, timeout, ca_cert, cert, key, skip_db_creation) { return __awaiter(void 0, void 0, void 0, function () {
-    var client, e_1, exists, resultSet, rows, e_2, q, e_3;
-    return __generator(this, function (_a) {
-        switch (_a.label) {
-            case 0:
-                client = connect(host, username, password, undefined, timeout, ca_cert, cert, key);
-                _a.label = 1;
-            case 1:
-                _a.trys.push([1, 3, , 4]);
-                return [4 /*yield*/, client.ping()];
-            case 2:
-                _a.sent();
-                return [3 /*break*/, 4];
-            case 3:
-                e_1 = _a.sent();
-                log('error', "Failed to connect to ClickHouse", e_1.message);
-                process.exit(1);
-                return [3 /*break*/, 4];
-            case 4:
-                if (!skip_db_creation) return [3 /*break*/, 14];
-                exists = false;
-                _a.label = 5;
-            case 5:
-                _a.trys.push([5, 8, , 10]);
-                return [4 /*yield*/, client.query({
-                        query: "SELECT 1 AS ok FROM system.databases WHERE name = {db_name:String}",
-                        query_params: { db_name: db_name },
-                        format: 'JSONEachRow',
-                    })];
-            case 6:
-                resultSet = _a.sent();
-                return [4 /*yield*/, resultSet.json()];
-            case 7:
-                rows = (_a.sent());
-                exists = rows.length > 0;
-                return [3 /*break*/, 10];
-            case 8:
-                e_2 = _a.sent();
-                log('error', "can't verify that the database ".concat(db_name, " exists."), e_2.message);
-                return [4 /*yield*/, client.close()];
-            case 9:
-                _a.sent();
-                process.exit(1);
-                return [3 /*break*/, 10];
-            case 10:
-                if (!!exists) return [3 /*break*/, 12];
-                log('error', "the database ".concat(db_name, " doesn't exist and --skip-db-creation is set. Please create the database manually or remove --skip-db-creation."));
-                return [4 /*yield*/, client.close()];
-            case 11:
-                _a.sent();
-                process.exit(1);
-                _a.label = 12;
-            case 12: return [4 /*yield*/, client.close()];
-            case 13:
-                _a.sent();
-                return [2 /*return*/];
-            case 14:
-                q = db_engine
-                    ? "CREATE DATABASE IF NOT EXISTS \"".concat(db_name, "\" ").concat(db_engine)
-                    : "CREATE DATABASE IF NOT EXISTS \"".concat(db_name, "\"");
-                _a.label = 15;
-            case 15:
-                _a.trys.push([15, 17, , 18]);
-                return [4 /*yield*/, client.command({
-                        query: q,
-                        clickhouse_settings: {
-                            wait_end_of_query: 1,
-                        },
-                    })];
-            case 16:
-                _a.sent();
-                return [3 /*break*/, 18];
-            case 17:
-                e_3 = _a.sent();
-                log('error', "can't create the database ".concat(db_name, "."), e_3.message);
-                process.exit(1);
-                return [3 /*break*/, 18];
-            case 18: return [4 /*yield*/, client.close()];
-            case 19:
-                _a.sent();
-                return [2 /*return*/];
+const create_db = async (host, username, password, db_name, db_engine, timeout, ca_cert, cert, key, skip_db_creation) => {
+    // Don't specify database name when creating it - connect to default database
+    const client = connect(host, username, password, undefined, timeout, ca_cert, cert, key);
+    try {
+        await client.ping();
+    }
+    catch (e) {
+        log('error', `Failed to connect to ClickHouse`, e.message);
+        process.exit(1);
+    }
+    if (skip_db_creation) {
+        // Database creation is disabled - verify the database already exists.
+        let exists = false;
+        try {
+            const resultSet = await client.query({
+                query: `SELECT 1 AS ok FROM system.databases WHERE name = {db_name:String}`,
+                query_params: { db_name },
+                format: 'JSONEachRow',
+            });
+            const rows = (await resultSet.json());
+            exists = rows.length > 0;
         }
-    });
-}); };
-var init_migration_table = function (client_2) {
-    var args_1 = [];
-    for (var _i = 1; _i < arguments.length; _i++) {
-        args_1[_i - 1] = arguments[_i];
+        catch (e) {
+            log('error', `can't verify that the database ${db_name} exists.`, e.message);
+            await client.close();
+            process.exit(1);
+        }
+        if (!exists) {
+            log('error', `the database ${db_name} doesn't exist and --skip-db-creation is set. Please create the database manually or remove --skip-db-creation.`);
+            await client.close();
+            process.exit(1);
+        }
+        await client.close();
+        return;
     }
-    return __awaiter(void 0, __spreadArray([client_2], args_1, true), void 0, function (client, table_engine) {
-        var q, e_4;
-        if (table_engine === void 0) { table_engine = 'MergeTree'; }
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0:
-                    q = "CREATE TABLE IF NOT EXISTS _migrations (\n      uid UUID DEFAULT generateUUIDv4(),\n      version UInt32,\n      checksum String,\n      migration_name String,\n      applied_at DateTime DEFAULT now()\n    )\n    ENGINE = ".concat(table_engine, "\n    ORDER BY tuple(applied_at)");
-                    _a.label = 1;
-                case 1:
-                    _a.trys.push([1, 3, , 4]);
-                    return [4 /*yield*/, client.command({
-                            query: q,
-                            clickhouse_settings: {
-                                wait_end_of_query: 1,
-                            },
-                        })];
-                case 2:
-                    _a.sent();
-                    return [3 /*break*/, 4];
-                case 3:
-                    e_4 = _a.sent();
-                    log('error', "can't create the _migrations table.", e_4.message);
-                    process.exit(1);
-                    return [3 /*break*/, 4];
-                case 4: return [2 /*return*/];
-            }
+    // In open source ClickHouse - default DB engine is "Atomic", for Cloud - "Shared". If not set, appropriate default is used.
+    const q = db_engine
+        ? `CREATE DATABASE IF NOT EXISTS "${db_name}" ${db_engine}`
+        : `CREATE DATABASE IF NOT EXISTS "${db_name}"`;
+    try {
+        await client.command({
+            query: q,
+            clickhouse_settings: {
+                wait_end_of_query: 1,
+            },
         });
-    });
+    }
+    catch (e) {
+        log('error', `can't create the database ${db_name}.`, e.message);
+        process.exit(1);
+    }
+    await client.close();
+};
+const init_migration_table = async (client, table_engine = 'MergeTree') => {
+    const q = `CREATE TABLE IF NOT EXISTS _migrations (
+      uid UUID DEFAULT generateUUIDv4(),
+      version UInt32,
+      checksum String,
+      migration_name String,
+      applied_at DateTime DEFAULT now()
+    )
+    ENGINE = ${table_engine}
+    ORDER BY tuple(applied_at)`;
+    try {
+        await client.command({
+            query: q,
+            clickhouse_settings: {
+                wait_end_of_query: 1,
+            },
+        });
+    }
+    catch (e) {
+        log('error', `can't create the _migrations table.`, e.message);
+        process.exit(1);
+    }
 };
-var get_migrations = function (migrations_home) {
-    var files;
+const get_migrations = (migrations_home) => {
+    let files;
     try {
         files = fs_1.default.readdirSync(migrations_home);
     }
     catch (e) {
-        log('error', "no migration directory ".concat(migrations_home, ". Please create it."));
+        log('error', `no migration directory ${migrations_home}. Please create it.`);
         process.exit(1);
     }
-    var migrations = [];
-    files.forEach(function (file) {
+    const migrations = [];
+    files.forEach((file) => {
         // Manage only .sql files.
         if (!file.endsWith('.sql')) {
             return;
         }
-        var version = Number(file.split('_')[0]);
+        const version = Number(file.split('_')[0]);
         if (!version) {
-            log('error', "a migration name should start from number, example: 1_init.sql. Please check, if the migration ".concat(file, " is named correctly"));
+            log('error', `a migration name should start from number, example: 1_init.sql. Please check, if the migration ${file} is named correctly`);
             process.exit(1);
         }
         migrations.push({
-            version: version,
-            file: file,
+            version,
+            file,
         });
     });
     if (!migrations) {
-        log('error', "no migrations in the ".concat(migrations_home, " migrations directory"));
+        log('error', `no migrations in the ${migrations_home} migrations directory`);
     }
     // Order by version.
-    migrations.sort(function (m1, m2) { return m1.version - m2.version; });
+    migrations.sort((m1, m2) => m1.version - m2.version);
     return migrations;
 };
-var apply_migrations = function (client, migrations, migrations_home) { return __awaiter(void 0, void 0, void 0, function () {
-    var migration_query_result, resultSet, e_5, migrations_applied, applied_migrations, _i, migrations_1, migration_1, content, checksum, queries, sets, _a, queries_1, query, e_6, e_7;
-    return __generator(this, function (_b) {
-        switch (_b.label) {
-            case 0:
-                migration_query_result = [];
-                _b.label = 1;
-            case 1:
-                _b.trys.push([1, 4, , 5]);
-                return [4 /*yield*/, client.query({
-                        query: "SELECT version, checksum, migration_name FROM _migrations ORDER BY version",
-                        format: 'JSONEachRow',
-                    })];
-            case 2:
-                resultSet = _b.sent();
-                return [4 /*yield*/, resultSet.json()];
-            case 3:
-                migration_query_result = _b.sent();
-                return [3 /*break*/, 5];
-            case 4:
-                e_5 = _b.sent();
-                log('error', "can't select data from the _migrations table.", e_5.message);
+const apply_migrations = async (client, migrations, migrations_home) => {
+    let migration_query_result = [];
+    try {
+        const resultSet = await client.query({
+            query: `SELECT version, checksum, migration_name FROM _migrations ORDER BY version`,
+            format: 'JSONEachRow',
+        });
+        migration_query_result = await resultSet.json();
+    }
+    catch (e) {
+        log('error', `can't select data from the _migrations table.`, e.message);
+        process.exit(1);
+    }
+    const migrations_applied = [];
+    migration_query_result.forEach((row) => {
+        migrations_applied[row.version] = {
+            version: row.version,
+            checksum: row.checksum,
+            migration_name: row.migration_name,
+        };
+        // Check if migration file was not removed after apply.
+        const migration_exist = migrations.find(({ version }) => version === row.version);
+        if (!migration_exist) {
+            log('error', `a migration file shouldn't be removed after apply. Please, restore the migration ${row.migration_name}.`);
+            process.exit(1);
+        }
+    });
+    let applied_migrations = '';
+    for (const migration of migrations) {
+        const content = fs_1.default.readFileSync(migrations_home + '/' + migration.file).toString();
+        const checksum = crypto_1.default.createHash('md5').update(content).digest('hex');
+        if (migrations_applied[migration.version]) {
+            // Check if migration file was not changed after apply.
+            if (migrations_applied[migration.version].checksum !== checksum) {
+                log('error', `a migration file should't be changed after apply. Please, restore content of the ${migrations_applied[migration.version].migration_name} migrations.`);
                 process.exit(1);
-                return [3 /*break*/, 5];
-            case 5:
-                migrations_applied = [];
-                migration_query_result.forEach(function (row) {
-                    migrations_applied[row.version] = {
-                        version: row.version,
-                        checksum: row.checksum,
-                        migration_name: row.migration_name,
-                    };
-                    // Check if migration file was not removed after apply.
-                    var migration_exist = migrations.find(function (_a) {
-                        var version = _a.version;
-                        return version === row.version;
-                    });
-                    if (!migration_exist) {
-                        log('error', "a migration file shouldn't be removed after apply. Please, restore the migration ".concat(row.migration_name, "."));
-                        process.exit(1);
-                    }
+            }
+            // Skip if a migration is already applied.
+            continue;
+        }
+        // Substitute ${VAR} from the environment (opt-in) at apply time only. The
+        // checksum above is over the raw file, so it stays stable across environments.
+        let sql;
+        try {
+            sql = (0, sql_parse_1.substitute_env)(content);
+        }
+        catch (e) {
+            log('error', `the migration ${migration.file} has an error.`, e instanceof Error ? e.message : String(e));
+            process.exit(1);
+        }
+        // Extract sql from the migration.
+        const queries = (0, sql_parse_1.sql_queries)(sql);
+        const sets = (0, sql_parse_1.sql_sets)(sql);
+        for (const query of queries) {
+            try {
+                await client.command({
+                    query: query,
+                    clickhouse_settings: sets,
                 });
-                applied_migrations = '';
-                _i = 0, migrations_1 = migrations;
-                _b.label = 6;
-            case 6:
-                if (!(_i < migrations_1.length)) return [3 /*break*/, 17];
-                migration_1 = migrations_1[_i];
-                content = fs_1.default.readFileSync(migrations_home + '/' + migration_1.file).toString();
-                checksum = crypto_1.default.createHash('md5').update(content).digest('hex');
-                if (migrations_applied[migration_1.version]) {
-                    // Check if migration file was not changed after apply.
-                    if (migrations_applied[migration_1.version].checksum !== checksum) {
-                        log('error', "a migration file should't be changed after apply. Please, restore content of the ".concat(migrations_applied[migration_1.version].migration_name, " migrations."));
-                        process.exit(1);
-                    }
-                    // Skip if a migration is already applied.
-                    return [3 /*break*/, 16];
-                }
-                queries = (0, sql_parse_1.sql_queries)(content);
-                sets = (0, sql_parse_1.sql_sets)(content);
-                _a = 0, queries_1 = queries;
-                _b.label = 7;
-            case 7:
-                if (!(_a < queries_1.length)) return [3 /*break*/, 12];
-                query = queries_1[_a];
-                _b.label = 8;
-            case 8:
-                _b.trys.push([8, 10, , 11]);
-                return [4 /*yield*/, client.command({
-                        query: query,
-                        clickhouse_settings: sets,
-                    })];
-            case 9:
-                _b.sent();
-                return [3 /*break*/, 11];
-            case 10:
-                e_6 = _b.sent();
+            }
+            catch (e) {
                 if (applied_migrations) {
-                    log('info', "The migration(s) ".concat(applied_migrations, " was successfully applied!"));
+                    log('info', `The migration(s) ${applied_migrations} was successfully applied!`);
                 }
-                log('error', "the migrations ".concat(migration_1.file, " has an error. Please, fix it (be sure that already executed parts of the migration would not be run second time) and re-run migration script."), e_6.message);
-                process.exit(1);
-                return [3 /*break*/, 11];
-            case 11:
-                _a++;
-                return [3 /*break*/, 7];
-            case 12:
-                _b.trys.push([12, 14, , 15]);
-                return [4 /*yield*/, client.insert({
-                        table: '_migrations',
-                        values: [{ version: migration_1.version, checksum: checksum, migration_name: migration_1.file }],
-                        format: 'JSONEachRow',
-                    })];
-            case 13:
-                _b.sent();
-                return [3 /*break*/, 15];
-            case 14:
-                e_7 = _b.sent();
-                log('error', "can't insert a data into the table _migrations.", e_7.message);
+                log('error', `the migrations ${migration.file} has an error. Please, fix it (be sure that already executed parts of the migration would not be run second time) and re-run migration script.`, e.message);
                 process.exit(1);
-                return [3 /*break*/, 15];
-            case 15:
-                applied_migrations = applied_migrations ? applied_migrations + ', ' + migration_1.file : migration_1.file;
-                _b.label = 16;
-            case 16:
-                _i++;
-                return [3 /*break*/, 6];
-            case 17:
-                if (applied_migrations) {
-                    log('info', "The migration(s) ".concat(applied_migrations, " was successfully applied!"));
-                }
-                else {
-                    log('info', "No migrations to apply.");
-                }
-                return [2 /*return*/];
+            }
         }
-    });
-}); };
-var migration = function (migrations_home, host, username, password, db_name, db_engine, table_engine, timeout, ca_cert, cert, key, skip_db_creation) { return __awaiter(void 0, void 0, void 0, function () {
-    var migrations, client;
-    return __generator(this, function (_a) {
-        switch (_a.label) {
-            case 0:
-                migrations = get_migrations(migrations_home);
-                return [4 /*yield*/, create_db(host, username, password, db_name, db_engine, timeout, ca_cert, cert, key, skip_db_creation)];
-            case 1:
-                _a.sent();
-                client = connect(host, username, password, db_name, timeout, ca_cert, cert, key);
-                return [4 /*yield*/, init_migration_table(client, table_engine)];
-            case 2:
-                _a.sent();
-                return [4 /*yield*/, apply_migrations(client, migrations, migrations_home)];
-            case 3:
-                _a.sent();
-                return [4 /*yield*/, client.close()];
-            case 4:
-                _a.sent();
-                return [2 /*return*/];
+        try {
+            await client.insert({
+                table: '_migrations',
+                values: [{ version: migration.version, checksum: checksum, migration_name: migration.file }],
+                format: 'JSONEachRow',
+            });
         }
-    });
-}); };
+        catch (e) {
+            log('error', `can't insert a data into the table _migrations.`, e.message);
+            process.exit(1);
+        }
+        applied_migrations = applied_migrations ? applied_migrations + ', ' + migration.file : migration.file;
+    }
+    if (applied_migrations) {
+        log('info', `The migration(s) ${applied_migrations} was successfully applied!`);
+    }
+    else {
+        log('info', `No migrations to apply.`);
+    }
+};
+const migration = async (migrations_home, host, username, password, db_name, db_engine, table_engine, timeout, ca_cert, cert, key, skip_db_creation) => {
+    const migrations = get_migrations(migrations_home);
+    await create_db(host, username, password, db_name, db_engine, timeout, ca_cert, cert, key, skip_db_creation);
+    const client = connect(host, username, password, db_name, timeout, ca_cert, cert, key);
+    await init_migration_table(client, table_engine);
+    await apply_migrations(client, migrations, migrations_home);
+    await client.close();
+};
 exports.migration = migration;
-var migrate = function () {
-    var program = new commander_1.Command();
+const migrate = () => {
+    const program = new commander_1.Command();
     program.name('clickhouse-migrations').description('ClickHouse migrations.').version(version_1.VERSION);
     program
         .command('migrate')
@@ -410,16 +276,9 @@ var migrate = function () {
         .option('--cert <path>', 'Client certificate file path', process.env.CH_MIGRATIONS_CERT)
         .option('--key <path>', 'Client key file path', process.env.CH_MIGRATIONS_KEY)
         .option('--skip-db-creation', 'Skip database creation', process.env.CH_MIGRATIONS_SKIP_DB_CREATION === 'true')
-        .action(function (options) { return __awaiter(void 0, void 0, void 0, function () {
-        return __generator(this, function (_a) {
-            switch (_a.label) {
-                case 0: return [4 /*yield*/, migration(options.migrationsHome, options.host, options.user, options.password, options.db, options.dbEngine, options.tableEngine, options.timeout, options.caCert, options.cert, options.key, options.skipDbCreation)];
-                case 1:
-                    _a.sent();
-                    return [2 /*return*/];
-            }
-        });
-    }); });
+        .action(async (options) => {
+        await migration(options.migrationsHome, options.host, options.user, options.password, options.db, options.dbEngine, options.tableEngine, options.timeout, options.caCert, options.cert, options.key, options.skipDbCreation);
+    });
     program.parse();
 };
 exports.migrate = migrate;

package/lib/sql-parse.d.ts

@@ -1,5 +1,6 @@
+declare const substitute_env: (content: string) => string;
 declare const sql_queries: (content: string) => string[];
 declare const sql_sets: (content: string) => {
     [key: string]: string;
 };
-export { sql_queries, sql_sets };
+export { sql_queries, sql_sets, substitute_env };

package/lib/sql-parse.js

@@ -1,31 +1,81 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sql_sets = exports.sql_queries = void 0;
+exports.substitute_env = exports.sql_sets = exports.sql_queries = void 0;
+// Replace ${NAME} in migration content with env vars. Opt-in via
+// CH_MIGRATIONS_SUBSTITUTE_ENV=true; otherwise returns content unchanged.
+// Use $${NAME} to keep a literal ${NAME}. Throws on an unset variable or a
+// malformed/unterminated placeholder, so nothing is ever silently left as-is.
+// Note: the caller checksums the raw file, so substitution only affects the
+// SQL sent to ClickHouse, not what's stored in _migrations.
+const substitute_env = (content) => {
+    if (process.env.CH_MIGRATIONS_SUBSTITUTE_ENV !== 'true') {
+        return content;
+    }
+    const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+    let result = '';
+    let position = 0;
+    while (position < content.length) {
+        // Escaped placeholder: $${NAME} -> ${NAME} (kept literal, not substituted).
+        if (content.startsWith('$${', position)) {
+            const end = content.indexOf('}', position + 3);
+            if (end === -1) {
+                throw new Error(`unterminated escaped placeholder at position ${position}`);
+            }
+            // Drop only the leading "$" and keep the rest as a literal.
+            result += content.slice(position + 1, end + 1);
+            position = end + 1;
+            continue;
+        }
+        // Environment placeholder: ${NAME}.
+        if (content.startsWith('${', position)) {
+            const end = content.indexOf('}', position + 2);
+            if (end === -1) {
+                throw new Error(`unterminated environment placeholder at position ${position}`);
+            }
+            const placeholder = content.slice(position, end + 1);
+            const name = content.slice(position + 2, end);
+            if (!ENV_NAME.test(name)) {
+                throw new Error(`invalid environment placeholder "${placeholder}"; expected \${NAME} or escape it as $${placeholder}`);
+            }
+            const value = process.env[name];
+            if (value === undefined) {
+                throw new Error(`migration references ${placeholder}, but environment variable ${name} is not set`);
+            }
+            result += value;
+            position = end + 1;
+            continue;
+        }
+        result += content[position];
+        position += 1;
+    }
+    return result;
+};
+exports.substitute_env = substitute_env;
 // Extract sql queries from migrations.
-var sql_queries = function (content) {
-    var queries = content
+const sql_queries = (content) => {
+    const queries = content
         .replace(/(--|#!|#\s).*(\n|\r\n|\r|$)/gm, '\n')
         .replace(/^\s*(SET\s).*(\n|\r\n|\r|$)/gm, '')
         .replace(/(\n|\r\n|\r)/gm, ' ')
         .replace(/\s+/g, ' ')
         .split(';')
-        .map(function (el) { return el.trim(); })
-        .filter(function (el) { return el.length != 0; });
+        .map((el) => el.trim())
+        .filter((el) => el.length != 0);
     return queries;
 };
 exports.sql_queries = sql_queries;
 // Extract query settings from migrations.
-var sql_sets = function (content) {
-    var sets = {};
-    var sets_arr = content
+const sql_sets = (content) => {
+    const sets = {};
+    const sets_arr = content
         .replace(/(--|#!|#\s).*(\n|\r\n|\r|$)/gm, '\n')
         .replace(/^\s*(?!SET\s).*(\n|\r\n|\r|$)/gm, '')
         .replace(/^\s*(SET\s)/gm, '')
         .replace(/(\n|\r\n|\r)/gm, ' ')
         .replace(/\s+/g, '')
         .split(';');
-    sets_arr.forEach(function (set_full) {
-        var set = set_full.split('=');
+    sets_arr.forEach((set_full) => {
+        const set = set_full.split('=');
         if (set[0]) {
             sets[set[0]] = set[1];
         }

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "1.3.1";
+export declare const VERSION = "1.4.0";

package/lib/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.3.1'; // x-release-please-version
+exports.VERSION = '1.4.0'; // x-release-please-version

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clickhouse-migrations",
-  "version": "1.3.1",
+  "version": "1.4.0",
   "description": "ClickHouse Migrations",
   "bin": {
     "clickhouse-migrations": "lib/cli.js"
@@ -15,7 +15,7 @@
   "author": "VVVi",
   "repository": "VVVi/clickhouse-migrations",
   "scripts": {
-    "build": "rm lib/*; tsc",
+    "build": "rm -rf lib/*; tsc",
     "test": "jest --config jestconfig.json",
     "test:unit": "jest --config jestconfig.json --testPathPatterns=\"\\.(unit|integration)\\.test\\.ts$\"",
     "test:e2e": "jest --config jestconfig.json --testPathPatterns=\"\\.e2e\\.test\\.ts$\"",
@@ -28,23 +28,29 @@
     "postversion": "git push && git push --tags"
   },
   "dependencies": {
-    "@clickhouse/client": "^1.20.0",
+    "@clickhouse/client": "^1.23.1",
     "commander": "^14.0.3"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
     "@types/jest": "^30.0.0",
-    "@types/node": "^20.19.37",
-    "@typescript-eslint/eslint-plugin": "^8.57.2",
-    "@typescript-eslint/parser": "^8.57.2",
-    "eslint": "^10.1.0",
+    "@types/node": "^20.19.43",
+    "@typescript-eslint/eslint-plugin": "^8.61.1",
+    "@typescript-eslint/parser": "^8.61.1",
+    "eslint": "^10.5.0",
     "eslint-config-prettier": "^10.1.8",
-    "eslint-plugin-prettier": "^5.5.5",
-    "jest": "^30.3.0",
-    "prettier": "^3.8.1",
-    "ts-jest": "^29.4.6",
-    "typescript": "^5.9.3",
-    "typescript-eslint": "^8.57.2"
+    "eslint-plugin-prettier": "^5.5.6",
+    "jest": "^30.4.2",
+    "prettier": "^3.8.4",
+    "ts-jest": "^29.4.11",
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.61.1"
+  },
+  "overrides": {
+    "js-yaml": "^4.2.0",
+    "@babel/core": "^7.29.7",
+    "test-exclude": "^8.0.0",
+    "glob": "^13.0.6"
   },
   "files": [
     "lib/**/*"