clickhouse-migrations 1.3.2 → 1.4.0

package/README.md

@@ -63,6 +63,10 @@ For TLS/HTTPS connections, you can provide a custom CA certificate and optional
                                 (optional) Skip database creation,
                                   set to 'true' to enable
                                   (--skip-db-creation)
+      CH_MIGRATIONS_SUBSTITUTE_ENV
+                                (optional) Substitute ${VAR} in migration
+                                  files from the environment at apply time,
+                                  set to 'true' to enable
 
 
   CLI executions examples
@@ -99,7 +103,7 @@ For TLS/HTTPS connections, you can provide a custom CA certificate and optional
 Migration file example:
 (e.g., located at /app/clickhouse/migrations/1_init.sql)
 
-```
+```sql
 -- an example of migration file 1_init.sql
 
 SET allow_experimental_json_type = 1;
@@ -115,3 +119,34 @@ SAMPLE BY session_id
 ORDER BY (session_id)
 SETTINGS index_granularity = 8192;
 ```
+
+### Environment variable substitution (optional)
+
+Some migrations need a value that differs per environment, for example: the host of a Postgres instance behind a `DICTIONARY`, a cluster name, a bucket, and so on.
+
+Enable `CH_MIGRATIONS_SUBSTITUTE_ENV=true` and `${VAR}` placeholders in migration files are replaced with the matching environment variables when the migration is applied. It's off by default, so existing migrations behave exactly as before.
+
+Rules:
+
+- `${NAME}` is replaced with the value of the `NAME` environment variable.
+- If `NAME` is unset, the migration fails - the placeholder is never left behind in the SQL.
+- To keep a literal `${NAME}` (for example in a comment or a string), escape it as `$${NAME}`.
+- A malformed (`${PG-HOST}`, `${}`) or unterminated (`${PG_HOST`) placeholder also fails, so nothing is ever silently passed through.
+
+Substitution is textual (like `envsubst`): it doesn't parse SQL and doesn't escape values, so anything placed inside quotes must be quoting-safe.
+
+A typical use is an idempotent dictionary pointing at a per-environment source:
+
+```sql
+CREATE OR REPLACE DICTIONARY my_db.dict_offers
+(
+    `id` UUID,
+    `name` String DEFAULT ''
+)
+PRIMARY KEY id
+SOURCE(POSTGRESQL(HOST '${PG_HOST}' PORT ${PG_PORT} USER '${PG_USER}' PASSWORD '${PG_PASSWORD}' DB '${PG_DB}' TABLE 'offers'))
+LIFETIME(MIN 0 MAX 300)
+LAYOUT(COMPLEX_KEY_HASHED());
+```
+
+_Please note:_ When substitution is enabled, the checksum is taken from the **raw file**, before substitution. This keeps a migration stable across environments and keeps secrets out of `_migrations`. Two consequences follow: an already-applied migration is not re-run when a variable changes (ship a new migration instead), and the substituted SQL still reaches ClickHouse, so it may appear in `system.query_log` or server logs.

package/lib/migrate.js

@@ -201,9 +201,19 @@ const apply_migrations = async (client, migrations, migrations_home) => {
             // Skip if a migration is already applied.
             continue;
         }
+        // Substitute ${VAR} from the environment (opt-in) at apply time only. The
+        // checksum above is over the raw file, so it stays stable across environments.
+        let sql;
+        try {
+            sql = (0, sql_parse_1.substitute_env)(content);
+        }
+        catch (e) {
+            log('error', `the migration ${migration.file} has an error.`, e instanceof Error ? e.message : String(e));
+            process.exit(1);
+        }
         // Extract sql from the migration.
-        const queries = (0, sql_parse_1.sql_queries)(content);
-        const sets = (0, sql_parse_1.sql_sets)(content);
+        const queries = (0, sql_parse_1.sql_queries)(sql);
+        const sets = (0, sql_parse_1.sql_sets)(sql);
         for (const query of queries) {
             try {
                 await client.command({

package/lib/sql-parse.d.ts

@@ -1,5 +1,6 @@
+declare const substitute_env: (content: string) => string;
 declare const sql_queries: (content: string) => string[];
 declare const sql_sets: (content: string) => {
     [key: string]: string;
 };
-export { sql_queries, sql_sets };
+export { sql_queries, sql_sets, substitute_env };

package/lib/sql-parse.js

@@ -1,6 +1,56 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sql_sets = exports.sql_queries = void 0;
+exports.substitute_env = exports.sql_sets = exports.sql_queries = void 0;
+// Replace ${NAME} in migration content with env vars. Opt-in via
+// CH_MIGRATIONS_SUBSTITUTE_ENV=true; otherwise returns content unchanged.
+// Use $${NAME} to keep a literal ${NAME}. Throws on an unset variable or a
+// malformed/unterminated placeholder, so nothing is ever silently left as-is.
+// Note: the caller checksums the raw file, so substitution only affects the
+// SQL sent to ClickHouse, not what's stored in _migrations.
+const substitute_env = (content) => {
+    if (process.env.CH_MIGRATIONS_SUBSTITUTE_ENV !== 'true') {
+        return content;
+    }
+    const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+    let result = '';
+    let position = 0;
+    while (position < content.length) {
+        // Escaped placeholder: $${NAME} -> ${NAME} (kept literal, not substituted).
+        if (content.startsWith('$${', position)) {
+            const end = content.indexOf('}', position + 3);
+            if (end === -1) {
+                throw new Error(`unterminated escaped placeholder at position ${position}`);
+            }
+            // Drop only the leading "$" and keep the rest as a literal.
+            result += content.slice(position + 1, end + 1);
+            position = end + 1;
+            continue;
+        }
+        // Environment placeholder: ${NAME}.
+        if (content.startsWith('${', position)) {
+            const end = content.indexOf('}', position + 2);
+            if (end === -1) {
+                throw new Error(`unterminated environment placeholder at position ${position}`);
+            }
+            const placeholder = content.slice(position, end + 1);
+            const name = content.slice(position + 2, end);
+            if (!ENV_NAME.test(name)) {
+                throw new Error(`invalid environment placeholder "${placeholder}"; expected \${NAME} or escape it as $${placeholder}`);
+            }
+            const value = process.env[name];
+            if (value === undefined) {
+                throw new Error(`migration references ${placeholder}, but environment variable ${name} is not set`);
+            }
+            result += value;
+            position = end + 1;
+            continue;
+        }
+        result += content[position];
+        position += 1;
+    }
+    return result;
+};
+exports.substitute_env = substitute_env;
 // Extract sql queries from migrations.
 const sql_queries = (content) => {
     const queries = content

package/lib/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "1.3.2";
+export declare const VERSION = "1.4.0";

package/lib/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = '1.3.2'; // x-release-please-version
+exports.VERSION = '1.4.0'; // x-release-please-version

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clickhouse-migrations",
-  "version": "1.3.2",
+  "version": "1.4.0",
   "description": "ClickHouse Migrations",
   "bin": {
     "clickhouse-migrations": "lib/cli.js"
@@ -28,7 +28,7 @@
     "postversion": "git push && git push --tags"
   },
   "dependencies": {
-    "@clickhouse/client": "^1.20.0",
+    "@clickhouse/client": "^1.23.1",
     "commander": "^14.0.3"
   },
   "devDependencies": {