@mlagie/sql-connector 3.0.3 → 3.2.0

package/src/migration/SchemaDiffer.js

@@ -0,0 +1,261 @@
+const { getSafe } = require("../utils/security/safe");
+const { DestructiveMigrationError } = require("./errors");
+
+function normalizeType(type) {
+    const value = String(type ?? "").toLowerCase();
+
+    const aliases = {
+        string: "varchar",
+        varchar: "varchar",
+        character_varying: "varchar",
+        "character varying": "varchar",
+
+        integer: "integer",
+        int: "integer",
+        int4: "integer",
+
+        bigint: "bigint",
+        int8: "bigint",
+
+        boolean: "boolean",
+        bool: "boolean",
+
+        float: "float",
+        double: "float",
+        double_precision: "float",
+
+        date: "date",
+        datetime: "datetime",
+        timestamp: "timestamp",
+
+        text: "text"
+    };
+
+    return getSafe(aliases, value) || value;
+}
+
+function normalizeField(field) {
+    return {
+        type: normalizeType(field.type),
+        length: field.length ?? null,
+        required: Boolean(field.required),
+        default: field.default ?? null,
+        unique: Boolean(field.unique),
+        auto_increment: Boolean(field.auto_increment),
+        primary_key: Boolean(field.primary_key)
+    };
+}
+
+function equivalentField(a, b) {
+    return JSON.stringify(normalizeField(a)) === JSON.stringify(normalizeField(b));
+}
+
+class SchemaDiffer {
+    constructor(options = {}) {
+        this.allowDestructiveChanges = Boolean(
+            options.allowDestructiveChanges
+        );
+
+        this.allowDropTables = Boolean(
+            options.allowDropTables
+        );
+    }
+
+    diff(desired, current) {
+        const operations = [];
+
+        const desiredTables = desired.tables || {};
+        const currentTables = current.tables || {};
+
+        for (const [tableName, desiredTable] of Object.entries(desiredTables)) {
+            const desiredColumns = desiredTable.columns || {};
+
+            // Table does not exist yet.
+            if (!getSafe(currentTables, tableName)) {
+                operations.push({
+                    type: "createTable",
+                    table: tableName,
+                    schema: desiredColumns
+                });
+
+                continue;
+            }
+
+            const currentTable = getSafe(currentTables, tableName);
+            const currentColumns = getSafe(currentTable, "columns") || {};
+
+            /*
+             * First pass:
+             * detect explicit renames using `oldname`.
+             *
+             * We deliberately do not guess renames.
+             */
+            const renameOperations = [];
+            const renamedFrom = new Set();
+            const renamedTo = new Set();
+
+            for (const [newName, field] of Object.entries(desiredColumns)) {
+                const oldName = field?.oldname;
+
+                if (!oldName) {
+                    continue;
+                }
+
+                if (oldName === newName) {
+                    throw new Error(
+                        `Column '${newName}' cannot have oldname equal to itself.`
+                    );
+                }
+
+                if (!getSafe(currentColumns, oldName)) {
+                    throw new Error(
+                        `Column '${oldName}' specified by oldname for '${tableName}.${newName}' does not exist.`
+                    );
+                }
+
+                if (getSafe(currentColumns, newName)) {
+                    throw new Error(
+                        `Cannot rename '${tableName}.${oldName}' to '${newName}' because '${newName}' already exists.`
+                    );
+                }
+
+                if (renamedFrom.has(oldName)) {
+                    throw new Error(
+                        `Column '${tableName}.${oldName}' is used by multiple oldname declarations.`
+                    );
+                }
+                const operation = {
+                    type: "renameColumn",
+                    table: tableName,
+                    oldName,
+                    newName
+                };
+
+                renameOperations.push(operation);
+                renamedFrom.add(oldName);
+                renamedTo.add(newName);
+            }
+
+            /*
+             * Add rename operations first.
+             */
+            operations.push(...renameOperations);
+
+            /*
+             * Build a virtual representation of the current schema
+             * after applying the renames.
+             *
+             * This is important because:
+             *
+             * email -> email_address
+             *
+             * followed by:
+             *
+             * email_address VARCHAR(255) -> VARCHAR(512)
+             *
+             * must generate:
+             *
+             * RENAME COLUMN
+             * ALTER COLUMN
+             */
+            const virtualCurrentColumns = {
+                ...currentColumns
+            };
+
+            for (const operation of renameOperations) {
+                virtualCurrentColumns[operation.newName] =
+                    virtualCurrentColumns[operation.oldName];
+
+                delete virtualCurrentColumns[operation.oldName];
+            }
+
+            /*
+             * Second pass:
+             * detect additions and modifications.
+             */
+            for (const [columnName, field] of Object.entries(desiredColumns)) {
+                const currentField = getSafe(virtualCurrentColumns, columnName);
+
+                if (!currentField) {
+                    operations.push({
+                        type: "addColumn",
+                        table: tableName,
+                        column: columnName,
+                        field
+                    });
+
+                    continue;
+                }
+
+                if (!equivalentField(field, currentField)) {
+                    operations.push({
+                        type: "alterColumn",
+                        table: tableName,
+                        column: columnName,
+                        field,
+                        previous: currentField
+                    });
+                }
+            }
+
+            /*
+             * Third pass:
+             * detect removed columns.
+             *
+             * Renamed columns are excluded because their old name
+             * intentionally disappears from the desired schema.
+             */
+            for (const currentName of Object.keys(currentColumns)) {
+                if (renamedFrom.has(currentName)) {
+                    continue;
+                }
+
+                if (!getSafe(desiredColumns, currentName)) {
+                    const operation = {
+                        type: "dropColumn",
+                        table: tableName,
+                        column: currentName,
+                        destructive: true
+                    };
+
+                    if (!this.allowDestructiveChanges) {
+                        throw new DestructiveMigrationError(
+                            `Dropping column '${tableName}.${currentName}' requires allowDestructiveChanges=true.`,
+                            operation
+                        );
+                    }
+
+                    operations.push(operation);
+                }
+            }
+        }
+
+        /*
+         * Detect removed tables.
+         */
+        for (const tableName of Object.keys(currentTables)) {
+            if (!getSafe(desiredTables, tableName)) {
+                const operation = {
+                    type: "dropTable",
+                    table: tableName,
+                    destructive: true
+                };
+
+                if (!this.allowDropTables) {
+                    throw new DestructiveMigrationError(
+                        `Dropping table '${tableName}' requires allowDropTables=true.`,
+                        operation
+                    );
+                }
+
+                operations.push(operation);
+            }
+        }
+
+        return operations;
+    }
+}
+
+module.exports = {
+    SchemaDiffer
+};

package/src/migration/SchemaInspector.js

@@ -0,0 +1,44 @@
+const { setSafe } = require("../utils/security/safe");
+
+class SchemaInspector {
+    constructor(connection, dialect) {
+        this.connection = connection;
+        this.dialect = dialect;
+    }
+
+    async query(sql, values = []) {
+        return this.dialect.execute(this.connection, sql, values);
+    }
+
+    async inspect() {
+        const queries = this.dialect.introspectionQueries();
+        const tableRows = await this.query(queries.tables);
+
+        const tables = {};
+
+        for (const row of tableRows || []) {
+            const tableName = row.table_name ?? row.TABLE_NAME;
+            if (!tableName || tableName === "sql_connector_migrations") continue;
+
+            const columnRows = await this.query(
+                queries.columns,
+                [tableName]
+            );
+            setSafe(tables, tableName, {
+                columns: Object.fromEntries(
+                    (columnRows || []).map(row => {
+                        const name = row.column_name ?? row.COLUMN_NAME;
+                        return [name, this.dialect.normalizeDatabaseColumn(row)];
+                    })
+                )
+            });
+        }
+
+        return {
+            version: 1,
+            tables
+        };
+    }
+}
+
+module.exports = { SchemaInspector };

package/src/migration/SchemaSnapshot.js

@@ -0,0 +1,53 @@
+const { clone, normalizeType, normalizeDefault, stableStringify, sha256 } = require("./utils");
+
+function fieldToSnapshot(field) {
+    return {
+        type: normalizeType(field),
+        length: field.length ?? null,
+        required: Boolean(field.required),
+        default: normalizeDefault(field.default),
+        unique: Boolean(field.unique),
+        auto_increment: Boolean(field.auto_increment),
+        foreignKey: field.foreignKey ?? null,
+        enum: Array.isArray(field.enum) ? [...field.enum] : null,
+        primary_key: Boolean(field.primary_key),
+        customize: field.customize ?? null,
+        oldname: field.oldname ?? null
+    };
+}
+
+function modelsToSnapshot(models) {
+    const tables = {};
+
+    for (const model of models) {
+        if (!model || typeof model.name !== "string" || !model.schema?.schemaDict) {
+            throw new Error("Migration received an invalid Model.");
+        }
+
+        tables[model.name] = {
+            columns: Object.fromEntries(
+                Object.entries(model.schema.schemaDict).map(([name, field]) => [name, fieldToSnapshot(field)])
+            )
+        };
+    }
+
+    return {
+        version: 1,
+        tables
+    };
+}
+
+function snapshotChecksum(snapshot) {
+    return sha256(stableStringify(snapshot));
+}
+
+function cloneSnapshot(snapshot) {
+    return clone(snapshot);
+}
+
+module.exports = {
+    fieldToSnapshot,
+    modelsToSnapshot,
+    snapshotChecksum,
+    cloneSnapshot
+};

package/src/migration/crypto.js

@@ -0,0 +1,79 @@
+const crypto = require("crypto");
+const { Buffer } = require("node:buffer");
+const fs = require("fs/promises");
+
+const FORMAT = "sql-connector-migration-v1";
+const ALGORITHM = "aes-256-gcm";
+const KEY_LENGTH = 32;
+const IV_LENGTH = 12;
+const SALT_LENGTH = 16;
+const TAG_LENGTH = 16;
+
+function deriveKey(secret, salt) {
+    if (!secret || typeof secret !== "string") {
+        throw new Error(
+            "Migration encryption key is missing. Provide encryption.key or SQL_CONNECTOR_MIGRATION_KEY."
+        );
+    }
+
+    return crypto.scryptSync(secret, salt, KEY_LENGTH);
+}
+
+function encrypt(text, secret) {
+    const salt = crypto.randomBytes(SALT_LENGTH);
+    const iv = crypto.randomBytes(IV_LENGTH);
+    const key = deriveKey(secret, salt);
+    const cipher = crypto.createCipheriv(ALGORITHM, key, iv);
+    const ciphertext = Buffer.concat([cipher.update(String(text), "utf8"), cipher.final()]);
+    const tag = cipher.getAuthTag();
+
+    return [
+        FORMAT,
+        `salt=${salt.toString("base64url")}`,
+        `iv=${iv.toString("base64url")}`,
+        `tag=${tag.toString("base64url")}`,
+        `data=${ciphertext.toString("base64url")}`
+    ].join("\n") + "\n";
+}
+
+function decrypt(payload, secret) {
+    const lines = String(payload).trim().split("\n");
+    if (lines[0] !== FORMAT) throw new Error("Unsupported encrypted migration format.");
+
+    const values = Object.fromEntries(lines.slice(1).map(line => {
+        const index = line.indexOf("=");
+        return [line.slice(0, index), line.slice(index + 1)];
+    }));
+
+    const salt = Buffer.from(values.salt, "base64url");
+    const iv = Buffer.from(values.iv, "base64url");
+    const tag = Buffer.from(values.tag, "base64url");
+    const ciphertext = Buffer.from(values.data, "base64url");
+
+    if (salt.length !== SALT_LENGTH || iv.length !== IV_LENGTH || tag.length !== TAG_LENGTH) {
+        throw new Error("Invalid encrypted migration payload.");
+    }
+
+    const key = deriveKey(secret, salt);
+    const decipher = crypto.createDecipheriv(ALGORITHM, key, iv);
+    decipher.setAuthTag(tag);
+
+    return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString("utf8");
+}
+
+async function encryptFile(inputPath, outputPath, secret) {
+    const content = await fs.readFile(inputPath, "utf8");
+    await fs.writeFile(outputPath, encrypt(content, secret), { mode: 0o600 });
+}
+
+async function decryptFile(inputPath, secret) {
+    return decrypt(await fs.readFile(inputPath, "utf8"), secret);
+}
+
+module.exports = {
+    FORMAT,
+    encrypt,
+    decrypt,
+    encryptFile,
+    decryptFile
+};

package/src/migration/errors.js

@@ -0,0 +1,28 @@
+class MigrationError extends Error {
+    constructor(message, code = "MIGRATION_ERROR", details = undefined) {
+        super(message);
+        this.name = "MigrationError";
+        this.code = code;
+        this.details = details;
+    }
+}
+
+class DestructiveMigrationError extends MigrationError {
+    constructor(message, details) {
+        super(message, "DESTRUCTIVE_MIGRATION", details);
+        this.name = "DestructiveMigrationError";
+    }
+}
+
+class MigrationChecksumError extends MigrationError {
+    constructor(message, details) {
+        super(message, "MIGRATION_CHECKSUM_MISMATCH", details);
+        this.name = "MigrationChecksumError";
+    }
+}
+
+module.exports = {
+    MigrationError,
+    DestructiveMigrationError,
+    MigrationChecksumError
+};

package/src/migration/index.js

@@ -0,0 +1,15 @@
+const { Migration } = require("./Migration");
+const { MigrationPlan } = require("./MigrationPlan");
+const { MigrationStore } = require("./MigrationStore");
+const { MigrationRunner } = require("./MigrationRunner");
+const { SchemaDiffer } = require("./SchemaDiffer");
+const { SchemaInspector } = require("./SchemaInspector");
+
+module.exports = {
+    Migration,
+    MigrationPlan,
+    MigrationStore,
+    MigrationRunner,
+    SchemaDiffer,
+    SchemaInspector,
+};

package/src/migration/utils.js

@@ -0,0 +1,89 @@
+const crypto = require("crypto");
+const { getSafe } = require("../utils/security/safe");
+
+function assertIdentifier(value, label = "identifier") {
+    if (typeof value !== "string" || !/^[A-Za-z_][A-Za-z0-9_$]*$/.test(value)) {
+        throw new Error(`Invalid ${label}: ${value}`);
+    }
+    return value;
+}
+
+function quoteIdentifier(name, quote) {
+    assertIdentifier(name);
+    return `${quote}${name}${quote}`;
+}
+
+function normalizeType(field) {
+    const type = field && field.type;
+    if (typeof type === "function") return type.name;
+    if (type && typeof type === "object" && typeof type.name === "string") return type.name;
+    if (typeof type === "string") return type;
+    return undefined;
+}
+
+function normalizeDefault(value) {
+    if (value === undefined) return undefined;
+    if (value === null) return null;
+    if (typeof value === "string") return value;
+    if (typeof value === "number" || typeof value === "boolean") return value;
+    return String(value);
+}
+
+function stableStringify(value) {
+    if (value === null || typeof value !== "object") return JSON.stringify(value);
+    if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
+    return `{${Object.keys(value).sort().map(k => `${JSON.stringify(k)}:${getSafe(value, k)}`)}.join(",")}}`;
+}
+
+function sha256(value) {
+    return crypto.createHash("sha256").update(value).digest("hex");
+}
+
+function migrationId(name = "migration") {
+    const now = new Date();
+    const stamp = now.toISOString()
+        .replace(/[-:TZ.]/g, "")
+        .slice(0, 14);
+    const safeName = String(name)
+        .trim()
+        .replace(/[^A-Za-z0-9_-]+/g, "_")
+        .replace(/^_+|_+$/g, "")
+        .slice(0, 80) || "migration";
+    return `${stamp}_${safeName}`;
+}
+
+function parseDuration(value) {
+    if (value === null || value === false || value === undefined) return null;
+    if (typeof value === "number" && Number.isFinite(value) && value >= 0) return value;
+
+    const match = /^(\d+(?:\.\d+)?)(ms|s|m|h|d|w)$/i.exec(String(value).trim());
+    if (!match) throw new Error(`Invalid retention duration: ${value}`);
+
+    const amount = Number(match[1]);
+    const unit = match[2].toLowerCase();
+    const factors = {
+        ms: 1,
+        s: 1000,
+        m: 60_000,
+        h: 3_600_000,
+        d: 86_400_000,
+        w: 604_800_000
+    };
+    return amount * getSafe(factors, unit);
+}
+
+function clone(value) {
+    return JSON.parse(JSON.stringify(value));
+}
+
+module.exports = {
+    assertIdentifier,
+    quoteIdentifier,
+    normalizeType,
+    normalizeDefault,
+    stableStringify,
+    sha256,
+    migrationId,
+    parseDuration,
+    clone
+};

package/src/models/Model.js

@@ -5,6 +5,7 @@ const { ModelInstance } = require("./ModelInstance");
 const { buildSelect, buildQueryParts } = require("../utils/buildQuery/buildQuery");
 const { getSafe, setSafe } = require("../utils/security/safe");
 const { getDialect } = require("../db/dialects");
+const LENGTH_TYPES = new Set(["VARCHAR", "CHAR", "BINARY", "VARBINARY"]);
 
 function getFieldType(field) {
     if (typeof field === "object") {
@@ -25,16 +26,20 @@ function getColumnDefinition(fieldName, field) {
     }
 
     const fieldType = getFieldType(field);
-    const type = getSafe(sqlTypeMap, fieldType);
+    const type = getDialect().mapType(fieldType);
     if (!type) throw new Error(`Field ${fieldName} has unsupported type ${fieldType}.`);
 
     let colDef;
     if (Array.isArray(field.enum) && field.enum.length > 0) {
         const enumValues = field.enum.map(v => `'${v.replace(/'/g, "''")}'`).join(", ");
         colDef = `ENUM(${enumValues})`;
+    } else if (type === "DECIMAL") {
+        const precision = field.precision > 0 ? field.precision : 10;
+        const scale = field.scale >= 0 ? field.scale : 0;
+        colDef = `DECIMAL(${precision}, ${scale})`;
     } else {
-        const hasLength = type === "VARCHAR" || (type === "INT" && getDialect().name === "mysql");
-        colDef = `${type}${hasLength ? `(${field.length > 0 ? field.length : 255})` : ""}`;
+        const hasLength = LENGTH_TYPES.has(type) || type === "INT";
+        colDef = getDialect().columnType(type, hasLength ? (field.length > 0 ? field.length : 255) : undefined);
     }
 
     if (field.required) colDef += ' NOT NULL';
@@ -43,7 +48,7 @@ function getColumnDefinition(fieldName, field) {
 
     if (defaultDefinition !== null) colDef += ` ${defaultDefinition}`;
     if (field.unique) colDef += ' UNIQUE';
-    if (field.auto_increment) colDef += ' AUTO_INCREMENT';
+    if (field.auto_increment) colDef += ` ${getDialect().autoIncrement(fieldType)}`;
     if (field.primary_key) colDef += ' PRIMARY KEY'
     if (typeof field.customize === 'string' && field.customize.length != 0) colDef += ` ${field.customize}`;
     return `${getDialect().escape(fieldName)} ${colDef}`;
@@ -160,10 +165,11 @@ class Model {
                 return `${getDialect().escape(fieldName)} ENUM(${enumValues})`;
             }
 
-            const type = getSafe(sqlTypeMap, fieldType);
+            const type = getDialect().mapType(fieldType);
 
             if (!type) throw new Error(`Field ${fieldName} has unsupported type ${field}`);
-            if (type == "VARCHAR") return `${getDialect().escape(fieldName)} ${type}(${lengthDefault})`;
+            if (LENGTH_TYPES.has(type)) return `${getDialect().escape(fieldName)} ${type}(${lengthDefault})`;
+            if (type === "DECIMAL") return `${getDialect().escape(fieldName)} ${type}(10, 0)`;
             return `${getDialect().escape(fieldName)} ${type}`;
         });
         return `CREATE TABLE IF NOT EXISTS ${getDialect().escape(this.name)} (${columns.join(', ')}${foreignKey.length > 0 ? ", " + foreignKey.join(', ') : ""}) ${getDialect().tableSuffix};`;
@@ -177,7 +183,7 @@ class Model {
      */
     async save(data) {
         const keys = Object.keys(data);
-        const sql_request = `INSERT INTO ${getDialect().escape(this.name)} (${getDialect().escapeIdentifierList(keys)}) VALUES (${keys.map(() => "?").join(", ")})`;
+        const sql_request = `INSERT INTO ${getDialect().escape(this.name)} (${getDialect().escapeIdentifierList(keys)}) VALUES (${keys.map((_, index) => getDialect().getPlaceholder(index)).join(", ")})`;
 
         try {
             const result = await getDialect().execute(getConnexion(), sql_request, Object.values(data));

package/src/models/Schema.js

@@ -1,22 +1,22 @@
-/**
- * Represents a database schema.
- * 
- * @example
- * const transferSchema = new Schema({
- *     token: {
- *         type: String,
- *         length: 50
- *     },
- *     mdp: {
- *         type: String,
- *         length: 15
- *     }
- * });
- */
-class Schema {
-    constructor(schemaDict) {
-        this.schemaDict = schemaDict;
+    /**
+     * Represents a database schema.
+     * 
+     * @example
+     * const transferSchema = new Schema({
+     *     token: {
+     *         type: String,
+     *         length: 50
+     *     },
+     *     mdp: {
+     *         type: String,
+     *         length: 15
+     *     }
+     * });
+     */
+    class Schema {
+        constructor(schemaDict) {
+            this.schemaDict = schemaDict;
+        }
     }
-}
 
-module.exports = { Schema }
+    module.exports = { Schema }