mysqlsyncr 1.0.0

package/src/create.js

@@ -0,0 +1,15 @@
+export const generateCreateTableSQL = (tableName, fields) => {
+    const columns = fields
+        .map(col => {
+            const nullClause = col.Null === 'NO' ? 'NOT NULL' : 'NULL';
+            const defaultClause = col.Default === 'CURRENT_TIMESTAMP' ? 'DEFAULT CURRENT_TIMESTAMP' : (col.Default !== null ? `DEFAULT '${col.Default}'` : '');
+            const extraClause = col.Extra || '';
+            return `\`${col.Field}\` ${col.Type} ${nullClause} ${defaultClause} ${extraClause}`.trim();
+        })
+        .join(', ');
+
+    const primaryKey = fields.find(field => field.Key === 'PRI');
+    const primaryKeyClause = primaryKey ? `, PRIMARY KEY (\`${primaryKey.Field}\`)` : '';
+
+    return `CREATE TABLE \`${tableName}\` (${columns}${primaryKeyClause});`;
+}

package/src/differences.js

@@ -0,0 +1,178 @@
+import { logger } from './loggers.js'; // Import the logger
+import { normalizeSQLDefinition } from './normalizers.js';
+export const findDifferences = (expected, current) => {
+    const differences = [];
+
+    // Detect extra tables
+    if (!current.tables) throw new Error('No tables found in current');
+    for (const tableName of Object.keys(current.tables)) {
+        if (!expected.tables[tableName]) {
+            differences.push({ type: 'extra_table', tableName });
+        }
+    }
+
+    // Compare tables and fields
+    if (!expected.tables) throw new Error('No tables found in expected');
+    for (const [tableName, contents] of Object.entries(expected.tables)) {
+        //check if craete table matches from json, if so, we dont need to compare indexes, triggers, or fields
+        
+        if (!current.tables[tableName]) {
+            console.log('missing_table', contents);
+            differences.push({ type: 'missing_table', tableName, createSQL: contents.createSQL });
+            continue;
+        }
+
+        const createTableSql = current.tables[tableName].createSQL;
+        if (createTableSql === contents.createSQL) {
+            logger(`Table ${tableName} createSQL matches from json, skipping indexes, triggers, and fields`);
+            continue;
+        } else {
+            logger(`Table ${tableName} createSQL does not match from json, comparing indexes, triggers, and fields`);
+            logger(createTableSql);
+            logger(contents.createSQL);
+        }
+        const { columns, indexes, triggers, createSQL, name, engine, charset, collate } = contents;
+
+
+        const currentFieldNames = current.tables[tableName].columns.map(f => f.Field);
+        const expectedFieldNames = columns.map(f => f.Field);
+
+        logger('Detecting missing fields for table', tableName);
+        // Detect missing fields
+        for (const field of columns) {
+            if (!currentFieldNames.includes(field.Field)) {
+                differences.push({ type: 'missing_field', tableName, field });
+            }
+        }
+
+        logger('Detecting extra fields for table', tableName);
+        // Detect extra fields
+        for (const field of current.tables[tableName].columns) {
+            if (!expectedFieldNames.includes(field.Field)) {
+                differences.push({ type: 'extra_field', tableName, field });
+            }
+        }
+
+
+        // verify Type Null and Default
+        logger('Verifying Type Null and Default for table', tableName);
+        for (const field of columns) {
+            const currentField = current.tables[tableName].columns.find(f => f.Field === field.Field);
+            if (field.Type !== currentField.Type) {
+                differences.push({ type: 'mismatched_field', tableName, field });
+            }
+            if (field.Null !== currentField.Null) {
+                differences.push({ type: 'mismatched_field', tableName, field });
+            }
+            if (field.Default !== currentField.Default) {
+                differences.push({ type: 'mismatched_field', tableName, field });
+            }
+        }
+
+        // verify indexes
+        logger('Verifying indexes for table', tableName);
+        for (const index of indexes) {
+            // Check if the current index matches the expected index structure
+            const currentIndex = current.tables[tableName].indexes.find(i =>
+                i.ColumnName.sort().join(', ') === index.ColumnName.sort().join(', ')
+            );
+            if (!currentIndex) {
+                logger('Missing index', index);
+                differences.push({ type: 'missing_index', tableName, indexName: index.Name, index });
+            } else {
+                // If needed, you can add further checks for uniqueness or other properties here
+                if (currentIndex.Name == index.Name) {
+                    if (currentIndex.Unique !== index.Unique) {
+                        differences.push({ type: 'mismatched_index', from: 'unique', tableName, indexName: index.Name, index });
+                    } else if (currentIndex.Primary !== index.Primary) {
+                        differences.push({ type: 'mismatched_index', from: 'primary', tableName, indexName: index.Name, index });
+                    }
+                }
+            }
+        }
+
+        for (const index of current.tables[tableName].indexes) {
+            logger('Checking index', index);
+            let indexFound = false
+
+            // Sort the ColumnName array for proper comparison
+            const sortedCurrentIndexColumns = index.ColumnName.sort().join(', ');
+            if (!indexes.some(i => i.ColumnName.sort().join(', ') === sortedCurrentIndexColumns)) {
+                differences.push({ type: 'extra_index', tableName, indexName: index.Name, index });
+            }
+        }
+
+        //verify triggers
+        logger('Verifying triggers for table', tableName);
+        for (const trigger of triggers) {
+            const currentTrigger = current.tables[tableName].triggers.find(t => t.Name === trigger.Name);
+            if (!currentTrigger) {
+                logger('Missing trigger', trigger);
+                differences.push({ type: 'missing_trigger', tableName, trigger });
+            } else {
+                // Compare additional properties if needed
+                if (currentTrigger.Event !== trigger.Event || currentTrigger.Statement !== trigger.Statement) {
+                    logger('Mismatched trigger', { expected: trigger, current: currentTrigger });
+                    differences.push({ type: 'mismatched_trigger', tableName, trigger });
+                }
+            }
+        }
+        for (const trigger of current.tables[tableName].triggers) {
+            if (!triggers.some(t => t.Name === trigger.Name)) {
+                logger('Extra trigger', trigger);
+                differences.push({ type: 'extra_trigger', tableName, trigger });
+            }
+        }
+    }
+
+
+
+
+    // Compare stored procedures
+    if (!expected.procedures) throw new Error('No procedures found in expected');
+    for (const [procName, contents] of Object.entries(expected.procedures)) {
+        if (!current.procedures[procName]) {
+            console.log('missing_procedure', contents);
+            differences.push({ type: 'missing_procedure', ...contents });
+            continue;
+        }
+
+        const currentProc = current.procedures[procName];
+
+        // Check if the procedure definition matches
+        if (currentProc.Definition !== contents.Definition) {
+            const normalizedCurrentDef = normalizeSQLDefinition(currentProc.Definition);
+            const normalizedNewDef = normalizeSQLDefinition(contents.Definition);
+
+            if (normalizedCurrentDef !== normalizedNewDef) {
+                differences.push({ type: 'mismatched_procedure', ...contents });
+            }
+        }
+
+
+    }
+
+    // Compare views
+    if (!expected.views) throw new Error('No views found in expected');
+    for (const [viewName, expectedDefinition] of Object.entries(expected.views)) {
+        if (!current.views[viewName]) {
+            differences.push({ type: 'missing_view', viewName, definition: expectedDefinition });
+            continue;
+        }
+
+        const currentDefinition = current.views[viewName];
+        if (currentDefinition !== expectedDefinition) {
+            differences.push({ type: 'mismatched_view', viewName, definition: expectedDefinition, current: currentDefinition });
+        }
+    }
+
+    // Detect extra views
+    if (!current.views) throw new Error('No views found in current');
+    for (const viewName of Object.keys(current.views)) {
+        if (!expected.views[viewName]) {
+            differences.push({ type: 'extra_view', viewName });
+        }
+    }
+
+    return differences;
+}

package/src/dump.js

@@ -0,0 +1,12 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { getDatabases, getDatabaseStructure } from './getters.js';
+
+export const dumpAllDatabases = async (connection, command, options) => {
+    const databases = await getDatabases(connection);
+    for (const database of databases) {
+        let dbStructure = await getDatabaseStructure(connection, database, command, options);
+        await fs.writeFile(path.join(options.output, database, `${database}.json`), JSON.stringify(dbStructure, null, 4), 'utf-8');
+    }
+}
+

package/src/getters.js

@@ -0,0 +1,140 @@
+import { parseCreateTableSQL } from './parsers.js';
+import path from 'path';
+import fs from 'fs/promises';
+import { logger } from './loggers.js';
+
+export const getDatabases = async (connection) => {
+    const databases = await connection.query('SHOW DATABASES');
+    return databases
+        .map(db => db.Database)
+        .filter(db => !['information_schema', 'mysql', 'performance_schema', 'sys'].includes(db));
+}
+
+
+export const getDatabaseStructure = async (connection, database, command, options) => {
+    if (connection.state === 'disconnected') {
+        await connection.reconnect(); // Ensure the connection is open
+    }
+    await connection.query(`USE \`${database}\``); // Switch to the target database
+    const tables = {};
+    const procedures = [];
+    const views = {};
+    const dbDir = path.join(options.output, database); // Use argv.output for the base path
+
+    // Create the output directory for the database if it doesn't exist
+    if (command === 'dump') {
+        const tableDir = path.join(dbDir, 'tables'); // Subfolder for table SQL files
+        await fs.mkdir(tableDir, { recursive: true }); // Create the tables directory
+    }
+
+    // Get tables and their structures
+    const tableRows = await connection.query('SHOW FULL TABLES WHERE Table_Type != "VIEW"');
+
+
+    for (const row of tableRows) {
+        const tableName = Object.values(row)[0]; // Get the first value in the row object
+        if (!tableName) {
+            logger(`Undefined table name found in database ${database}`, row);
+            continue; // Skip undefined table names
+        }
+
+        // Get the CREATE TABLE statement
+        const createTableRow = await connection.query(`SHOW CREATE TABLE \`${tableName}\``);
+        const createTableSQL = createTableRow[0]['Create Table'];
+
+        const parsedTable = parseCreateTableSQL(createTableSQL);
+
+        // Structure the table object
+        tables[tableName] = {
+            columns: parsedTable.columns, // Array of columns
+            indexes: parsedTable.indexes,   // Array of indexes
+            constraints: parsedTable.constraints, // Array of constraints
+            triggers: [],                   // Initialize triggers array
+            createSQL: createTableSQL       // Store the CREATE TABLE SQL if needed
+        };
+
+        // Save the CREATE TABLE statement to an individual SQL file for each table
+        if (command === 'dump') {
+            const tableDir = path.join(dbDir, 'tables'); // Subfolder for table SQL files
+            const filePath = path.join(tableDir, `${tableName}.sql`); // Use the table directory for file saving
+            await fs.mkdir(path.dirname(filePath), { recursive: true }); // Create the tables directory if it doesn't exist
+            await fs.writeFile(filePath, `${createTableSQL}\n`, 'utf-8');
+            logger(`Table structure saved to: ${filePath}`);
+        }
+    }
+
+    // Get stored procedures
+    const routineRows = await connection.query(`SHOW PROCEDURE STATUS WHERE Db = ?`, [database]);
+
+    for (const row of routineRows) {
+        let createSQLrow = await connection.query(`SHOW CREATE PROCEDURE \`${row.Name}\``);
+        let createSQL = createSQLrow[0]['Create Procedure'];
+        procedures.push({
+            Name: row.Name,
+            Definition: createSQL
+        }); // Add procedure names to the array
+        if (command === 'dump') {
+            const filePath = path.join(dbDir, 'procedures', `${row.Name}.sql`); // Use the table directory for file saving
+            await fs.mkdir(path.dirname(filePath), { recursive: true }); // Create the procedures directory if it doesn't exist
+            await fs.writeFile(filePath, `${createSQL}\n`, 'utf-8');
+            logger(`Procedure definition saved to: ${filePath}`);
+        }
+    }
+
+    // Get triggers
+    const triggerRows = await connection.query(`
+        SELECT TRIGGER_NAME, EVENT_OBJECT_TABLE
+        FROM INFORMATION_SCHEMA.TRIGGERS
+        WHERE TRIGGER_SCHEMA = ?
+    `, [database]);
+
+    for (const row of triggerRows) {
+        const triggerName = row.TRIGGER_NAME;
+        const tableName = row.EVENT_OBJECT_TABLE;
+
+        // Ensure triggerDefinition is defined before using it
+        const triggerDefinitionRow = await connection.query(`SHOW CREATE TRIGGER \`${triggerName}\``);
+        const triggerDefinition = triggerDefinitionRow[0]['SQL Original Statement'];
+
+        if (triggerDefinition) {
+
+            tables[tableName].triggers.push({
+                Name: triggerName,
+                Definition: triggerDefinition
+            });
+            if (command === 'dump') {
+                const tableDir = path.join(dbDir, 'tables'); // Subfolder for table SQL files
+                const triggerFilePath = path.join(tableDir, 'triggers', `${triggerName}.sql`);
+                await fs.mkdir(path.dirname(triggerFilePath), { recursive: true }); // Create the triggers directory if it doesn't exist
+                await fs.writeFile(triggerFilePath, `${triggerDefinition}\n`, 'utf-8');
+                logger(`Trigger definition saved to: ${triggerFilePath}`);
+            }
+
+        } else {
+            console.warn(`Trigger ${triggerName} has no action statement.`);
+        }
+    }
+
+    // Get views
+    const viewRows = await connection.query(`SHOW FULL TABLES IN \`${database}\` WHERE TABLE_TYPE LIKE 'VIEW'`);
+    for (const row of viewRows) {
+        let viewName = row[`Tables_in_${database.toLowerCase()}`];
+        if (!viewName) viewName = row[`Tables_in_${database}`];
+        logger(`getting view Tables_in_${database}`, viewName);
+        try {
+            const viewDefinition = await connection.query(`SHOW CREATE VIEW \`${viewName}\``);
+            views[viewName] = viewDefinition[0]['Create View'];
+            if (command === 'dump') {
+                const filePath = path.join(dbDir, 'views', `${viewName}.sql`); // Use the table directory for file saving
+                await fs.mkdir(path.dirname(filePath), { recursive: true }); // Create the views directory if it doesn't exist
+                await fs.writeFile(filePath, `${viewDefinition[0]['Create View']}\n`, 'utf-8');
+                logger(`View definition saved to: ${filePath}`);
+            }
+        } catch (error) {
+            logger(`Error retrieving definition for view ${viewName}:`, error.message);
+            logger(row)
+        }
+    }
+
+    return { tables, procedures, views };
+}

package/src/index.js

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+
+//TODO 
+
+//1. Check constraints in tables
+//2. fix index recognition
+
+import mysql from 'mysql2';
+import util from 'util';
+
+import { cli } from './cli.js';
+import { dumpAllDatabases } from './dump.js';
+import { compareAllDatabases } from './compare.js';
+
+
+class DBStructureChecker {
+    constructor(config, targetDatabase = null) {
+        this.config = config;
+        this.targetDatabase = targetDatabase;
+        this.connection = mysql.createConnection(this.config);
+        this.connection.on('error', (err) => {
+            console.error('Database connection error:', err);
+        });
+        this.connection.query = util.promisify(this.connection.query); // Promisify for async/await
+    }
+
+    async reconnect() {
+        this.connection = mysql.createConnection(this.config);
+        this.connection.query = util.promisify(this.connection.query); // Promisify for async/await
+        await this.connection.connect(); // Connect to the database
+    }
+
+    async closeConnection() {
+        if (this.connection.state !== 'disconnected') {
+            await this.connection.end();
+        }
+    }
+
+
+}
+
+// CLI Setup
+(async () => {
+    const { command, config, argv } = cli();
+
+    const checker = new DBStructureChecker(config, argv.database);
+    checker.connection.on('connect', async () => {
+        console.log('Connected to the database');
+        checker.targetDatabase = argv.database;
+        checker.command = command;
+        checker.options = argv;
+        try {
+            if (command === 'dump') {
+                console.log('Dumping the database structure to files');
+                await dumpAllDatabases(checker.connection, command, argv);
+            } else if (command === 'compare') {
+                console.log('Comparing the database structure with JSON dumps (dry run)');
+                let diffs = await compareAllDatabases(checker.connection, argv.output, false, command, argv); // Dry run
+                let totalDiffs = Object.values(diffs).reduce((acc, arr) => acc + arr.length, 0);
+                console.log('Total differences count:', totalDiffs);
+            } else if (command === 'update') {
+                console.log('Updating the database structure based on JSON dumps');
+                let totalDiffs = Infinity; // Initialize with a large number
+                let previousDiffs = totalDiffs; // Store previous differences
+                let runCount = 0; // Counter for the number of runs
+                let maxRuns = 6;
+                while (runCount < maxRuns && totalDiffs > 0 && (totalDiffs < previousDiffs || previousDiffs === Infinity)) {
+                    console.log('Running update command. Run count:', runCount);
+                    let diffs = await compareAllDatabases(checker.connection, argv.output, true, command, argv); // Apply updates
+                    previousDiffs = totalDiffs; // Update previous differences
+                    totalDiffs = Object.values(diffs).reduce((acc, arr) => acc + arr.length, 0);
+                    console.log('Total differences count:', totalDiffs);
+                    runCount++;
+                    if(totalDiffs > 0 && runCount < maxRuns){
+                        console.log('Total differences count is still greater than 0. Waiting 1 second before next run. Run count:', runCount);
+                        await new Promise(resolve => setTimeout(resolve, 1000));
+                    }
+                }
+            }
+        } catch (err) {
+            console.error(err.message);
+        } finally {
+            await checker.closeConnection();
+        }
+    });
+
+})();

package/src/loggers.js

@@ -0,0 +1,140 @@
+let isVerbose = false; // Global variable to manage verbosity
+
+export const setVerbose = (verbose) => {
+    isVerbose = verbose; // Function to set the verbosity level
+};
+
+export const logger = (...args) => {
+    if (isVerbose) {
+        console.log(...args); // Log all arguments if verbose is enabled
+    }
+};
+
+export const logDifferences = (differences, database, dryRun) => {
+    const formattedDifferences = differences.map(diff => {
+        switch (diff.type) {
+            case 'missing_procedure':
+                return {
+                    Type: diff.type,
+                    Name: diff.Name,
+                    Details: `Procedure ${diff.Name} is missing`,
+                    Action: 'Procedure is missing',
+                };
+            case 'mismatched_procedure':
+
+                return {
+                    Type: diff.type,
+                    Name: diff.Name,
+                    Details: `Mismatched procedure ${diff.Name}`,
+                    Action: 'Mismatched procedure',
+                };
+            case 'missing_field':
+                return {
+                    Type: diff.type,
+                    Name: diff.tableName,
+                    Details: `Field ${diff.field.Field} is missing`,
+                    Action: 'Field is missing',
+                };
+            case 'missing_table':
+                return {
+                    Type: diff.type,
+                    Name: diff.tableName,
+                    Details: `Table ${diff.tableName} is missing`,
+                    Action: 'Table is missing',
+                };
+            case 'extra_field':
+                return {
+                    Type: diff.type,
+                    Name: diff.tableName,
+                    Details: `Field ${diff.field.Field} is extra`,
+                    Action: 'Field is extra',
+                };
+            case 'extra_table':
+                return {
+                    Type: diff.type,
+                    Name: diff.tableName,
+                    Details: `Table ${diff.tableName} is extra`,
+                    Action: 'Table is extra',
+                };
+            case 'missing_view':
+                return {
+                    Type: diff.type,
+                    Name: diff.viewName,
+                    Details: `View ${diff.viewName} is missing`,
+                    Action: 'View is missing',
+                };
+            case 'extra_index':
+                return {
+                    Type: diff.type,
+                    Name: diff.tableName,
+                    Details: `Index ${diff.index.Name} is extra`,
+                    Action: 'Index is extra',
+                };
+            case 'missing_index':
+                return {
+                    Type: diff.type,
+                    TableName: diff.tableName,
+                    Name: diff.index.Name,
+                    Details: `Index ${diff.index.Name} is missing`,
+                    Action: 'Index is missing',
+                };
+
+            case 'mismatched_index':
+                return {
+                    Type: diff.type,
+                    TableName: diff.tableName,
+                    Name: diff.index.Name,
+                    From: diff.from,
+                    Details: `Index ${diff.index.Name} is mismatched`,
+                    Action: 'Index is mismatched',
+                };
+
+            case 'extra_trigger':
+                return {
+                    Type: diff.type,
+                    Name: diff.trigger.Name,
+                    Details: `Trigger ${diff.trigger.Name} is extra`,
+                    Action: 'Trigger is extra',
+                };
+            case 'mismatched_trigger':
+                return {
+                    Type: diff.type,
+                    Name: diff.trigger.Name,
+                    Details: `Trigger ${diff.trigger.Name} is mismatched`,
+                    Action: 'Trigger is mismatched',
+                };  
+            case 'missing_trigger':
+                console.log('missing_trigger', diff)
+                return {
+                    Type: diff.type,
+                    Name: diff.trigger.Name,
+                    Details: `Trigger ${diff.trigger.Name} is missing`,
+                    Action: 'Trigger is missing',
+                };
+            default:
+                return {
+                    Type: diff.type,
+                    Details: `Unhandled difference type: ${diff.type}`,
+                };
+        }
+    });
+
+    // Filter out any undefined or null entries
+    const filteredDifferences = formattedDifferences.filter(diff => diff);
+
+    // Log the differences in a table format
+    if (filteredDifferences.length > 0) {
+        if (dryRun) {
+            logger(`Dry Run: Differences for database ${database}:`);
+        } else {
+            logger(`Differences applied for database ${database}:`);
+        }
+        console.table(filteredDifferences);
+    } else {
+        if (dryRun) {
+            logger(`Dry Run: No differences found. Database: ${database}`);
+        } else {
+            logger(`No differences found. Database: ${database}`);
+        }
+    }
+}

package/src/normalizers.js

@@ -0,0 +1,6 @@
+export const normalizeSQLDefinition = (definition) => {
+    return definition
+        .replace(/\s+/g, ' ') // Replace multiple whitespace with a single space
+        .replace(/'/g, "\\'") // Escape single quotes
+        .trim(); // Trim leading and trailing whitespace
+}