@mlagie/sql-connector 3.0.3 → 3.1.0

package/src/migration/MigrationStore.js

@@ -0,0 +1,137 @@
+const fs = require("fs/promises");
+const path = require("path");
+const { encrypt, decrypt } = require("./crypto");
+const { parseDuration } = require("./utils");
+const { MigrationChecksumError } = require("./errors");
+
+class MigrationStore {
+    constructor(options = {}) {
+        this.directory = path.resolve(options.directory || "migrations");
+        this.backupDirectory = path.resolve(options.backupDirectory || path.join(this.directory, "backups"));
+        this.encryptionKey = options.encryptionKey || process.env.SQL_CONNECTOR_MIGRATION_KEY;
+        this.retention = parseDuration(options.retention ?? null);
+    }
+
+    async ensureDirectories() {
+        await fs.mkdir(this.directory, { recursive: true, mode: 0o700 });
+        await fs.mkdir(this.backupDirectory, { recursive: true, mode: 0o700 });
+    }
+
+    async writeMigration(plan) {
+        await this.ensureDirectories();
+        if (!this.encryptionKey) {
+            throw new Error("An encryption key is required to write migration files.");
+        }
+        const outputPath = path.join(this.directory, `${plan.id}.sql.enc`);
+        const payload = encrypt(plan.sql, this.encryptionKey);
+        await fs.writeFile(outputPath, payload, { encoding: "utf8", mode: 0o600 });
+
+        const metadataPath = path.join(this.directory, `${plan.id}.meta.json.enc`);
+        const metadata = {
+            version: 1,
+            id: plan.id,
+            dialect: plan.dialect,
+            checksum: plan.checksum,
+            operations: plan.operations,
+            previousSnapshot: plan.previousSnapshot,
+            snapshot: plan.snapshot
+        };
+        await fs.writeFile(metadataPath, encrypt(JSON.stringify(metadata, null, 2), this.encryptionKey), {
+            encoding: "utf8",
+            mode: 0o600
+        });
+        return outputPath;
+    }
+
+    async readMigration(id) {
+        await this.ensureDirectories();
+        if (!this.encryptionKey) throw new Error("An encryption key is required to read migration files.");
+        const file = path.join(this.directory, `${id}.meta.json.enc`);
+        try {
+            return JSON.parse(decrypt(await fs.readFile(file, "utf8"), this.encryptionKey));
+        } catch (error) {
+            if (error.code === "ENOENT") return null;
+            throw error;
+        }
+    }
+
+    async readBackup(id) {
+        await this.ensureDirectories();
+        if (!this.encryptionKey) throw new Error("An encryption key is required to read backups.");
+        const file = path.join(this.backupDirectory, `${id}.json.enc`);
+        try {
+            return JSON.parse(decrypt(await fs.readFile(file, "utf8"), this.encryptionKey));
+        } catch (error) {
+            if (error.code === "ENOENT") return null;
+            throw error;
+        }
+    }
+
+    async writeBackup(name, content) {
+        await this.ensureDirectories();
+        if (!this.retention) return null;
+        if (!this.encryptionKey) {
+            throw new Error("An encryption key is required to write backups.");
+        }
+        const outputPath = path.join(this.backupDirectory, `${name}.json.enc`);
+        await fs.writeFile(outputPath, encrypt(JSON.stringify(content, null, 2), this.encryptionKey), {
+            encoding: "utf8",
+            mode: 0o600
+        });
+        return outputPath;
+    }
+
+    async cleanup(now = Date.now()) {
+        if (!this.retention) return 0;
+        await this.ensureDirectories();
+
+        const files = await fs.readdir(this.backupDirectory);
+        let removed = 0;
+        for (const file of files) {
+            const fullPath = path.join(this.backupDirectory, file);
+            const stat = await fs.stat(fullPath);
+            if (now - stat.mtimeMs >= this.retention) {
+                await fs.rm(fullPath, { force: true });
+                removed++;
+            }
+        }
+        return removed;
+    }
+
+    async writeSnapshot(snapshot) {
+        await this.ensureDirectories();
+        const file = path.join(this.directory, "schema.snapshot.json");
+        await fs.writeFile(file, JSON.stringify(snapshot, null, 2) + "\n", {
+            encoding: "utf8",
+            mode: 0o600
+        });
+        return file;
+    }
+
+    async readSnapshot() {
+        try {
+            const file = path.join(this.directory, "schema.snapshot.json");
+            return JSON.parse(await fs.readFile(file, "utf8"));
+        } catch (error) {
+            if (error.code === "ENOENT") return null;
+            throw error;
+        }
+    }
+
+    async listMigrationFiles() {
+        await this.ensureDirectories();
+        return (await fs.readdir(this.directory))
+            .filter(file => file.endsWith(".sql.enc"))
+            .sort();
+    }
+
+    validateChecksum(actual, expected) {
+        if (actual !== expected) {
+            throw new MigrationChecksumError(
+                `Migration checksum mismatch. Expected ${expected}, got ${actual}.`
+            );
+        }
+    }
+}
+
+module.exports = { MigrationStore };

package/src/migration/SchemaDiffer.js

@@ -0,0 +1,261 @@
+const { getSafe } = require("../utils/security/safe");
+const { DestructiveMigrationError } = require("./errors");
+
+function normalizeType(type) {
+    const value = String(type ?? "").toLowerCase();
+
+    const aliases = {
+        string: "varchar",
+        varchar: "varchar",
+        character_varying: "varchar",
+        "character varying": "varchar",
+
+        integer: "integer",
+        int: "integer",
+        int4: "integer",
+
+        bigint: "bigint",
+        int8: "bigint",
+
+        boolean: "boolean",
+        bool: "boolean",
+
+        float: "float",
+        double: "float",
+        double_precision: "float",
+
+        date: "date",
+        datetime: "datetime",
+        timestamp: "timestamp",
+
+        text: "text"
+    };
+
+    return getSafe(aliases, value) || value;
+}
+
+function normalizeField(field) {
+    return {
+        type: normalizeType(field.type),
+        length: field.length ?? null,
+        required: Boolean(field.required),
+        default: field.default ?? null,
+        unique: Boolean(field.unique),
+        auto_increment: Boolean(field.auto_increment),
+        primary_key: Boolean(field.primary_key)
+    };
+}
+
+function equivalentField(a, b) {
+    return JSON.stringify(normalizeField(a)) === JSON.stringify(normalizeField(b));
+}
+
+class SchemaDiffer {
+    constructor(options = {}) {
+        this.allowDestructiveChanges = Boolean(
+            options.allowDestructiveChanges
+        );
+
+        this.allowDropTables = Boolean(
+            options.allowDropTables
+        );
+    }
+
+    diff(desired, current) {
+        const operations = [];
+
+        const desiredTables = desired.tables || {};
+        const currentTables = current.tables || {};
+
+        for (const [tableName, desiredTable] of Object.entries(desiredTables)) {
+            const desiredColumns = desiredTable.columns || {};
+
+            // Table does not exist yet.
+            if (!getSafe(currentTables, tableName)) {
+                operations.push({
+                    type: "createTable",
+                    table: tableName,
+                    schema: desiredColumns
+                });
+
+                continue;
+            }
+
+            const currentTable = getSafe(currentTables, tableName);
+            const currentColumns = getSafe(currentTable, "columns") || {};
+
+            /*
+             * First pass:
+             * detect explicit renames using `oldname`.
+             *
+             * We deliberately do not guess renames.
+             */
+            const renameOperations = [];
+            const renamedFrom = new Set();
+            const renamedTo = new Set();
+
+            for (const [newName, field] of Object.entries(desiredColumns)) {
+                const oldName = field?.oldname;
+
+                if (!oldName) {
+                    continue;
+                }
+
+                if (oldName === newName) {
+                    throw new Error(
+                        `Column '${newName}' cannot have oldname equal to itself.`
+                    );
+                }
+
+                if (!getSafe(currentColumns, oldName)) {
+                    throw new Error(
+                        `Column '${oldName}' specified by oldname for '${tableName}.${newName}' does not exist.`
+                    );
+                }
+
+                if (getSafe(currentColumns, newName)) {
+                    throw new Error(
+                        `Cannot rename '${tableName}.${oldName}' to '${newName}' because '${newName}' already exists.`
+                    );
+                }
+
+                if (renamedFrom.has(oldName)) {
+                    throw new Error(
+                        `Column '${tableName}.${oldName}' is used by multiple oldname declarations.`
+                    );
+                }
+                const operation = {
+                    type: "renameColumn",
+                    table: tableName,
+                    oldName,
+                    newName
+                };
+
+                renameOperations.push(operation);
+                renamedFrom.add(oldName);
+                renamedTo.add(newName);
+            }
+
+            /*
+             * Add rename operations first.
+             */
+            operations.push(...renameOperations);
+
+            /*
+             * Build a virtual representation of the current schema
+             * after applying the renames.
+             *
+             * This is important because:
+             *
+             * email -> email_address
+             *
+             * followed by:
+             *
+             * email_address VARCHAR(255) -> VARCHAR(512)
+             *
+             * must generate:
+             *
+             * RENAME COLUMN
+             * ALTER COLUMN
+             */
+            const virtualCurrentColumns = {
+                ...currentColumns
+            };
+
+            for (const operation of renameOperations) {
+                virtualCurrentColumns[operation.newName] =
+                    virtualCurrentColumns[operation.oldName];
+
+                delete virtualCurrentColumns[operation.oldName];
+            }
+
+            /*
+             * Second pass:
+             * detect additions and modifications.
+             */
+            for (const [columnName, field] of Object.entries(desiredColumns)) {
+                const currentField = getSafe(virtualCurrentColumns, columnName);
+
+                if (!currentField) {
+                    operations.push({
+                        type: "addColumn",
+                        table: tableName,
+                        column: columnName,
+                        field
+                    });
+
+                    continue;
+                }
+
+                if (!equivalentField(field, currentField)) {
+                    operations.push({
+                        type: "alterColumn",
+                        table: tableName,
+                        column: columnName,
+                        field,
+                        previous: currentField
+                    });
+                }
+            }
+
+            /*
+             * Third pass:
+             * detect removed columns.
+             *
+             * Renamed columns are excluded because their old name
+             * intentionally disappears from the desired schema.
+             */
+            for (const currentName of Object.keys(currentColumns)) {
+                if (renamedFrom.has(currentName)) {
+                    continue;
+                }
+
+                if (!getSafe(desiredColumns, currentName)) {
+                    const operation = {
+                        type: "dropColumn",
+                        table: tableName,
+                        column: currentName,
+                        destructive: true
+                    };
+
+                    if (!this.allowDestructiveChanges) {
+                        throw new DestructiveMigrationError(
+                            `Dropping column '${tableName}.${currentName}' requires allowDestructiveChanges=true.`,
+                            operation
+                        );
+                    }
+
+                    operations.push(operation);
+                }
+            }
+        }
+
+        /*
+         * Detect removed tables.
+         */
+        for (const tableName of Object.keys(currentTables)) {
+            if (!getSafe(desiredTables, tableName)) {
+                const operation = {
+                    type: "dropTable",
+                    table: tableName,
+                    destructive: true
+                };
+
+                if (!this.allowDropTables) {
+                    throw new DestructiveMigrationError(
+                        `Dropping table '${tableName}' requires allowDropTables=true.`,
+                        operation
+                    );
+                }
+
+                operations.push(operation);
+            }
+        }
+
+        return operations;
+    }
+}
+
+module.exports = {
+    SchemaDiffer
+};

package/src/migration/SchemaInspector.js

@@ -0,0 +1,44 @@
+const { setSafe } = require("../utils/security/safe");
+
+class SchemaInspector {
+    constructor(connection, dialect) {
+        this.connection = connection;
+        this.dialect = dialect;
+    }
+
+    async query(sql, values = []) {
+        return this.dialect.execute(this.connection, sql, values);
+    }
+
+    async inspect() {
+        const queries = this.dialect.introspectionQueries();
+        const tableRows = await this.query(queries.tables);
+
+        const tables = {};
+
+        for (const row of tableRows || []) {
+            const tableName = row.table_name ?? row.TABLE_NAME;
+            if (!tableName || tableName === "sql_connector_migrations") continue;
+
+            const columnRows = await this.query(
+                queries.columns,
+                this.dialect.name === "postgres" ? [tableName] : [tableName]
+            );
+            setSafe(tables, tableName, {
+                columns: Object.fromEntries(
+                    (columnRows || []).map(row => {
+                        const name = row.column_name ?? row.COLUMN_NAME;
+                        return [name, this.dialect.normalizeDatabaseColumn(row)];
+                    })
+                )
+            });
+        }
+
+        return {
+            version: 1,
+            tables
+        };
+    }
+}
+
+module.exports = { SchemaInspector };

package/src/migration/SchemaSnapshot.js

@@ -0,0 +1,53 @@
+const { clone, normalizeType, normalizeDefault, stableStringify, sha256 } = require("./utils");
+
+function fieldToSnapshot(field) {
+    return {
+        type: normalizeType(field),
+        length: field.length ?? null,
+        required: Boolean(field.required),
+        default: normalizeDefault(field.default),
+        unique: Boolean(field.unique),
+        auto_increment: Boolean(field.auto_increment),
+        foreignKey: field.foreignKey ?? null,
+        enum: Array.isArray(field.enum) ? [...field.enum] : null,
+        primary_key: Boolean(field.primary_key),
+        customize: field.customize ?? null,
+        oldname: field.oldname ?? null
+    };
+}
+
+function modelsToSnapshot(models) {
+    const tables = {};
+
+    for (const model of models) {
+        if (!model || typeof model.name !== "string" || !model.schema?.schemaDict) {
+            throw new Error("Migration received an invalid Model.");
+        }
+
+        tables[model.name] = {
+            columns: Object.fromEntries(
+                Object.entries(model.schema.schemaDict).map(([name, field]) => [name, fieldToSnapshot(field)])
+            )
+        };
+    }
+
+    return {
+        version: 1,
+        tables
+    };
+}
+
+function snapshotChecksum(snapshot) {
+    return sha256(stableStringify(snapshot));
+}
+
+function cloneSnapshot(snapshot) {
+    return clone(snapshot);
+}
+
+module.exports = {
+    fieldToSnapshot,
+    modelsToSnapshot,
+    snapshotChecksum,
+    cloneSnapshot
+};

package/src/migration/crypto.js

@@ -0,0 +1,79 @@
+const crypto = require("crypto");
+const { Buffer } = require("node:buffer");
+const fs = require("fs/promises");
+
+const FORMAT = "sql-connector-migration-v1";
+const ALGORITHM = "aes-256-gcm";
+const KEY_LENGTH = 32;
+const IV_LENGTH = 12;
+const SALT_LENGTH = 16;
+const TAG_LENGTH = 16;
+
+function deriveKey(secret, salt) {
+    if (!secret || typeof secret !== "string") {
+        throw new Error(
+            "Migration encryption key is missing. Provide encryption.key or SQL_CONNECTOR_MIGRATION_KEY."
+        );
+    }
+
+    return crypto.scryptSync(secret, salt, KEY_LENGTH);
+}
+
+function encrypt(text, secret) {
+    const salt = crypto.randomBytes(SALT_LENGTH);
+    const iv = crypto.randomBytes(IV_LENGTH);
+    const key = deriveKey(secret, salt);
+    const cipher = crypto.createCipheriv(ALGORITHM, key, iv);
+    const ciphertext = Buffer.concat([cipher.update(String(text), "utf8"), cipher.final()]);
+    const tag = cipher.getAuthTag();
+
+    return [
+        FORMAT,
+        `salt=${salt.toString("base64url")}`,
+        `iv=${iv.toString("base64url")}`,
+        `tag=${tag.toString("base64url")}`,
+        `data=${ciphertext.toString("base64url")}`
+    ].join("\n") + "\n";
+}
+
+function decrypt(payload, secret) {
+    const lines = String(payload).trim().split("\n");
+    if (lines[0] !== FORMAT) throw new Error("Unsupported encrypted migration format.");
+
+    const values = Object.fromEntries(lines.slice(1).map(line => {
+        const index = line.indexOf("=");
+        return [line.slice(0, index), line.slice(index + 1)];
+    }));
+
+    const salt = Buffer.from(values.salt, "base64url");
+    const iv = Buffer.from(values.iv, "base64url");
+    const tag = Buffer.from(values.tag, "base64url");
+    const ciphertext = Buffer.from(values.data, "base64url");
+
+    if (salt.length !== SALT_LENGTH || iv.length !== IV_LENGTH || tag.length !== TAG_LENGTH) {
+        throw new Error("Invalid encrypted migration payload.");
+    }
+
+    const key = deriveKey(secret, salt);
+    const decipher = crypto.createDecipheriv(ALGORITHM, key, iv);
+    decipher.setAuthTag(tag);
+
+    return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString("utf8");
+}
+
+async function encryptFile(inputPath, outputPath, secret) {
+    const content = await fs.readFile(inputPath, "utf8");
+    await fs.writeFile(outputPath, encrypt(content, secret), { mode: 0o600 });
+}
+
+async function decryptFile(inputPath, secret) {
+    return decrypt(await fs.readFile(inputPath, "utf8"), secret);
+}
+
+module.exports = {
+    FORMAT,
+    encrypt,
+    decrypt,
+    encryptFile,
+    decryptFile
+};

package/src/migration/dialects/Dialect.js

@@ -0,0 +1,112 @@
+const { quoteIdentifier } = require("../utils");
+
+class Dialect {
+    constructor(name, options = {}) {
+        this.name = name;
+        this.quote = options.quote;
+        this.capabilities = {
+            transactionalDDL: false,
+            renameColumn: true,
+            alterColumn: true,
+            ...options.capabilities
+        };
+    }
+
+    escape(name) {
+        return quoteIdentifier(name, this.quote);
+    }
+
+    placeholder() {
+        return "?";
+    }
+
+    placeholders(count) {
+        return Array.from({ length: count }, (_, i) => this.placeholder(i)).join(", ");
+    }
+
+    execute() {
+        throw new Error(`${this.name} dialect must implement execute().`);
+    }
+
+    async begin() {}
+    async commit() {}
+    async rollback() {}
+
+    mapType() {
+        throw new Error(`${this.name} dialect must implement mapType().`);
+    }
+
+    columnDefinition() {
+        throw new Error(`${this.name} dialect must implement columnDefinition().`);
+    }
+
+    createTable() {
+        throw new Error(`${this.name} dialect must implement createTable().`);
+    }
+
+    addColumn(table, name, field) {
+        return `ALTER TABLE ${this.escape(table)} ADD COLUMN ${this.columnDefinition(name, field)};`;
+    }
+
+    renameColumn(table, oldName, newName) {
+        return `ALTER TABLE ${this.escape(table)} RENAME COLUMN ${this.escape(oldName)} TO ${this.escape(newName)};`;
+    }
+
+    alterColumn() {
+        throw new Error(`${this.name} does not implement alterColumn().`);
+    }
+
+    dropColumn(table, name) {
+        return `ALTER TABLE ${this.escape(table)} DROP COLUMN ${this.escape(name)};`;
+    }
+
+    dropTable(table) {
+        return `DROP TABLE ${this.escape(table)};`;
+    }
+
+    introspectionQueries() {
+        throw new Error(`${this.name} dialect must implement introspectionQueries().`);
+    }
+
+    normalizeDatabaseColumn(row) {
+        return row;
+    }
+
+    normalizeDatabaseTable(row) {
+        return row;
+    }
+
+    createMigrationTable() {
+        throw new Error(`${this.name} dialect must implement createMigrationTable().`);
+    }
+
+    insertMigration() {
+        throw new Error(`${this.name} dialect must implement insertMigration().`);
+    }
+
+    selectMigrations() {
+        throw new Error(`${this.name} dialect must implement selectMigrations().`);
+    }
+
+    deleteMigration() {
+        throw new Error(`${this.name} dialect must implement deleteMigration().`);
+    }
+
+    async transaction(connection, callback) {
+        if (!this.capabilities.transactionalDDL) return callback();
+
+        await this.begin(connection);
+        try {
+            const result = await callback();
+            await this.commit(connection);
+            return result;
+        } catch (error) {
+            try {
+                await this.rollback(connection);
+            } catch {}
+            throw error;
+        }
+    }
+}
+
+module.exports = { Dialect };