mysqlsyncr 1.1.12 → 1.2.1

package/src/apply.js

@@ -1,6 +1,167 @@
 import { logger } from './loggers.js';
+import { adaptSQLForServer, getServerVersion, portableCollation } from './versions.js';
 
+const quoteString = (value) => `'${String(value).replace(/\\/g, '\\\\').replace(/'/g, "''")}'`;
+
+// Types MySQL won't give a literal default
+const noLiteralDefaultRegex = /^(tiny|medium|long)?(text|blob)$|^json$|^geometry|^point$|^linestring$|^polygon$/i;
+
+const defaultClause = (field) => {
+    const value = field.Default;
+    if (value === undefined || value === null || value === 'NULL') {
+        // A NOT NULL column can't default to NULL; leave it without a default
+        return field.NotNull ? '' : 'DEFAULT NULL';
+    }
+    // Functions, expressions, and bit/hex literals such as b'0' are written as they were read
+    if (/^(CURRENT_TIMESTAMP|now)(\(\d*\))?$/i.test(value) || value.startsWith('(') || /^[bx]'[0-9a-f]*'$/i.test(value) || /^0x[0-9a-f]+$/i.test(value)) {
+        return `DEFAULT ${value}`;
+    }
+    if (noLiteralDefaultRegex.test(field.Type)) {
+        return '';
+    }
+    return `DEFAULT ${quoteString(value)}`;
+}
+
+// Template value for a column detail, or the live column's when the template doesn't record it
+const columnDetail = (field, currentField, key) => key in field ? field[key] : currentField?.[key];
+
+/**
+ * Full column definition for ADD/CHANGE, e.g.
+ *   timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+ * NULL/NOT NULL is always explicit: without it a timestamp is NOT NULL when
+ * explicit_defaults_for_timestamp is OFF, and DEFAULT NULL then fails.
+ * Charset, collation, comment, ON UPDATE and a generated column's expression
+ * come from the template when it has them and are otherwise kept from the live
+ * column, since CHANGE drops anything not restated.
+ */
+export const columnDefinition = (field, currentField) => {
+    const charset = columnDetail(field, currentField, 'Charset');
+    const collate = columnDetail(field, currentField, 'Collate');
+    const onUpdate = columnDetail(field, currentField, 'OnUpdate');
+    const comment = columnDetail(field, currentField, 'Comment');
+    const generated = columnDetail(field, currentField, 'Generated');
+    if (generated) {
+        // A generated column takes no DEFAULT, ON UPDATE or AUTO_INCREMENT
+        return [
+            field.Type,
+            charset ? `CHARACTER SET ${charset}` : '',
+            collate ? `COLLATE ${collate}` : '',
+            `GENERATED ALWAYS AS ${generated}`,
+            field.NotNull ? 'NOT NULL' : 'NULL',
+            comment ? `COMMENT ${quoteString(comment)}` : ''
+        ].filter(Boolean).join(' ');
+    }
+    return [
+        field.Type,
+        charset ? `CHARACTER SET ${charset}` : '',
+        collate ? `COLLATE ${collate}` : '',
+        field.NotNull ? 'NOT NULL' : 'NULL',
+        defaultClause(field),
+        onUpdate ? `ON UPDATE ${onUpdate}` : '',
+        field.AutoIncrement ? 'AUTO_INCREMENT' : '',
+        comment ? `COMMENT ${quoteString(comment)}` : ''
+    ].filter(Boolean).join(' ');
+}
+
+// `col`, or `col`(255) for a prefix index on a text column
+const formatIndexColumns = (columns) => columns.map(col => {
+    const match = col.match(/^`?([^`(]+)`?(?:\((\d+)\))?$/);
+    if (!match) return `\`${col}\``;
+    return `\`${match[1]}\`${match[2] ? `(${match[2]})` : ''}`;
+}).join(', ');
+
+const isPrimaryIndex = (index) => !!index.Primary || index.Name === 'PRIMARY';
+
+const addIndexClause = (index) => isPrimaryIndex(index)
+    ? `ADD PRIMARY KEY (${formatIndexColumns(index.ColumnName)})`
+    : `ADD ${index.Unique ? 'UNIQUE ' : ''}INDEX \`${index.Name}\` (${formatIndexColumns(index.ColumnName)})`;
+
+const dropIndexClause = (index) => isPrimaryIndex(index) ? 'DROP PRIMARY KEY' : `DROP INDEX \`${index.Name}\``;
+
+const describeDifference = (diff) => {
+    const name = diff.tableName || diff.viewName || diff.Name || diff.database || '';
+    const detail = diff.field?.Field || diff.indexName || diff.trigger?.Name || (diff.key ? JSON.stringify(diff.key) : '');
+    return `${diff.type} ${name}${detail ? '.' + detail : ''}`;
+}
+
+// Rewrites one collation in a CREATE TABLE, both as a table option (COLLATE=x) and as a
+// column attribute (COLLATE x). Passing null for `to` removes the clause entirely.
+const replaceCollation = (sql, from, to) => {
+    const escaped = from.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    return sql
+        .replace(new RegExp('COLLATE(\\s*)=(\\s*)' + escaped, 'gi'), to ? `COLLATE$1=$2${to}` : '')
+        .replace(new RegExp('COLLATE(\\s+)' + escaped, 'gi'), to ? `COLLATE$1${to}` : '');
+};
+
+// Rewrites that let DDL dumped from a newer server run on an older one. Each inspects the
+// error the server actually returned and returns the amended SQL plus what changed, or
+// null when it has nothing to offer - so a rewrite only ever fires for the failure it
+// was written for, and anything else still surfaces as a failure. These catch what
+// adaptSQLForServer can't anticipate, e.g. when the server's version couldn't be read.
+const compatRewrites = [
+    (sql, errorMessage) => {
+        const unknown = /Unknown collation: '([^']+)'/i.exec(errorMessage);
+        if (!unknown) return null;
+        const substitute = portableCollation(unknown[1]);
+        const rewritten = replaceCollation(sql, unknown[1], substitute);
+        if (rewritten === sql) return null;
+        return {
+            sql: rewritten,
+            message: `server does not support collation ${unknown[1]}, created with ${substitute || 'the charset default'} instead`,
+        };
+    },
+    (sql, errorMessage) => {
+        // Index visibility is MySQL 8.0.23+. Only strip the keyword where it is legal -
+        // directly after an index definition's closing parenthesis - so the word cannot
+        // be removed from a column comment or a column named `visible`.
+        if (!/error in your SQL syntax/i.test(errorMessage) || !/\b(?:IN)?VISIBLE\b/i.test(errorMessage)) return null;
+        const rewritten = sql.replace(/\)(\s+)(?:IN)?VISIBLE(?=\s*[,)\r\n])/g, ')');
+        if (rewritten === sql) return null;
+        return {
+            sql: rewritten,
+            message: 'server does not support index visibility, created with every index visible',
+        };
+    },
+];
+
+/**
+ * Apply the differences in order. A failing difference doesn't stop the rest.
+ * Returns { attempted, failures, warnings }: failures are the differences that
+ * couldn't be applied, warnings the ones applied with a change for this server
+ * (e.g. an 8.0 collation on 5.7), each as { type, tableName, description, message }.
+ */
 export const applyDifferences = async (connection, database, differences) => {
+    const failures = [];
+    const warnings = [];
+    let current = null; // the difference being applied, for warnings from query()
+    const warn = (diff, message) => {
+        console.warn(`Applied ${describeDifference(diff)} with a change: ${message}`);
+        warnings.push({ type: diff.type, tableName: diff.tableName, description: describeDifference(diff), message });
+    };
+
+    // Template SQL from MySQL 8.0 can name collations a 5.7 server doesn't have
+    const serverVersion = await getServerVersion(connection);
+    const query = async (sql, values) => {
+        const adapted = adaptSQLForServer(sql, serverVersion);
+        if (adapted !== sql) {
+            console.log(`Adapted for MySQL ${serverVersion}: ${adapted}`);
+            if (current) warn(current, `MySQL ${serverVersion} has no utf8mb4_0900 collations, so the closest one it has was used`);
+        }
+        return connection.query(adapted, values);
+    };
+
+    // A dump written by MySQL 8 assumes explicit_defaults_for_timestamp is on, where a
+    // `timestamp NOT NULL` column with no DEFAULT is just a column with no default. With
+    // the MySQL 5.7 default of off, the server instead assigns every TIMESTAMP NOT NULL
+    // column after the first an implicit '0000-00-00 00:00:00', which strict mode then
+    // rejects at CREATE TABLE time. Matching MySQL 8 for this session applies that DDL as
+    // written; DDL that states its defaults - everything SHOW CREATE TABLE emits, on
+    // either version - parses the same way with the setting on or off.
+    try {
+        await connection.query('SET SESSION explicit_defaults_for_timestamp = 1');
+    } catch (e) {
+        logger(`Could not set explicit_defaults_for_timestamp for this session: ${e.message}`);
+    }
    
     differences.sort((a, b) => {
         //Sort to make sure we apply the differences in a logical order.
@@ -21,7 +182,9 @@ export const applyDifferences = async (connection, database, differences) => {
             'extra_view': 6,
             'missing_trigger': 7,
             'mismatched_trigger': 7,
-            'extra_trigger': 7
+            'extra_trigger': 7,
+            // after the table and its columns exist
+            'missing_row': 8
         };
         
         return (typePriority[a.type] || 999) - (typePriority[b.type] || 999);
@@ -29,57 +192,71 @@ export const applyDifferences = async (connection, database, differences) => {
 
 
     for (const diff of differences) {
+        current = diff;
         try {
 
             if(diff.type == 'missing_database'){
-                const createDatabaseSQL = `CREATE DATABASE \`${differences[0].database}\`;`;
+                const createDatabaseSQL = `CREATE DATABASE \`${diff.database}\`;`;
                 console.log(`Executing: ${createDatabaseSQL}`);
-                await connection.query(createDatabaseSQL);
+                await query(createDatabaseSQL);
                 continue;
             }
 
 
-            await connection.query(`USE \`${database}\``);
+            await query(`USE \`${database}\``);
             if (diff.type === 'missing_table') {
                 const { tableName, createSQL } = diff;
                 logger(`Creating missing table ${tableName}`);
                 console.log(`Executing: ${createSQL}`);
-                await connection.query(createSQL);
+                try {
+                    await query(createSQL);
+                } catch (e) {
+                    // The dump can name things this server has never heard of - a MySQL 8
+                    // collation, index visibility. Amend the statement for whatever the
+                    // server objected to and retry, one objection at a time, since it only
+                    // reports the first. The table then exists, but not exactly as dumped,
+                    // so record every change rather than calling it a clean success.
+                    let sql = createSQL;
+                    let error = e;
+                    const changes = [];
+
+                    while (error && changes.length < compatRewrites.length) {
+                        const message = error.message;
+                        const rewrite = compatRewrites.reduce((found, fn) => found || fn(sql, message), null);
+                        if (!rewrite) break;
+
+                        sql = rewrite.sql;
+                        changes.push(rewrite.message);
+                        console.log(`Retrying: ${sql}`);
+                        try {
+                            await query(sql);
+                            error = null;
+                        } catch (retryError) {
+                            error = retryError;
+                        }
+                    }
+
+                    if (error) throw error;
+                    warn(diff, changes.join('; '));
+                }
 
             } else if (diff.type === 'extra_table') {
                 const { tableName } = diff;
                 logger(`Dropping extra table ${tableName}`);
                 const dropTableSQL = `DROP TABLE \`${tableName}\`;`;
                 console.log(`Executing: ${dropTableSQL}`);
-                await connection.query(dropTableSQL);
+                await query(dropTableSQL);
 
             } else if (diff.type === 'missing_field') {
-                // Step 1: Add column with DEFAULT NULL to handle existing records
-                const addFieldSQL = `ALTER TABLE \`${diff.tableName}\` ADD \`${diff.field.Field}\` ${diff.field.Type} DEFAULT NULL;`;
+                // Existing rows get the column's default, or the type's implicit default for NOT NULL without one
+                const addFieldSQL = `ALTER TABLE \`${diff.tableName}\` ADD COLUMN \`${diff.field.Field}\` ${columnDefinition(diff.field)};`;
                 console.log(`Executing: ${addFieldSQL}`);
-                await connection.query(addFieldSQL);
-
-                // Step 2: If NOT NULL is required and no default was specified, modify the column to remove DEFAULT NULL
-                if (diff.field.NotNull && !diff.field.Default) {
-                    const modifyFieldSQL = `ALTER TABLE \`${diff.tableName}\` MODIFY \`${diff.field.Field}\` ${diff.field.Type} NOT NULL;`;
-                    console.log(`Executing: ${modifyFieldSQL}`);
-                    await connection.query(modifyFieldSQL);
-                }
-                if(diff.field.Default){
-                    const defaultClause = diff.field.Default === 'CURRENT_TIMESTAMP' 
-                    ? 'DEFAULT CURRENT_TIMESTAMP'
-                    : diff.field.Default !== undefined && diff.field.Default !== null 
-                        ? `DEFAULT '${diff.field.Default}'` 
-                        : '';
-                    const modifyFieldSQL = `ALTER TABLE \`${diff.tableName}\` MODIFY \`${diff.field.Field}\` ${diff.field.Type} ${defaultClause};`;
-                    console.log(`Executing: ${modifyFieldSQL}`);
-                    await connection.query(modifyFieldSQL);
-                }
+                await query(addFieldSQL);
 
             } else if (diff.type === 'extra_field') {
                 const dropFieldSQL = `ALTER TABLE \`${diff.tableName}\` DROP COLUMN \`${diff.field.Field}\`;`;
                 console.log(`Executing: ${dropFieldSQL}`);
-                await connection.query(dropFieldSQL);
+                await query(dropFieldSQL);
 
             } else if (diff.type === 'mismatched_field') {
                 const { tableName, field, currentField } = diff;
@@ -132,7 +309,7 @@ export const applyDifferences = async (connection, database, differences) => {
                 //     // Step 1: Add new column
                 //     const addColumnSQL = `ALTER TABLE \`${tableName}\` ADD COLUMN \`${field.Field}_new\` ${field.Type} DEFAULT NULL;`;
                 //     console.log(`Adding new column for type conversion: ${addColumnSQL}`);
-                //     await connection.query(addColumnSQL);
+                //     await query(addColumnSQL);
 
                 //     // Step 2: Copy data in batches
                 //     let totalUpdated = 0;
@@ -147,7 +324,7 @@ export const applyDifferences = async (connection, database, differences) => {
                 //             AND \`${field.Field}\` IS NOT NULL
                 //             LIMIT ${batchSize};
                 //         `;
-                //         const result = await connection.query(updateSQL);
+                //         const result = await query(updateSQL);
                 //         const rowsAffected = result.affectedRows;
                         
                 //         if (rowsAffected === 0) break;
@@ -163,7 +340,7 @@ export const applyDifferences = async (connection, database, differences) => {
                 //         WHERE \`${field.Field}\` IS NOT NULL 
                 //         AND \`${field.Field}_new\` IS NULL;
                 //     `;
-                //     const [verifyResult] = await connection.query(verifySQL);
+                //     const [verifyResult] = await query(verifySQL);
                 //     if (verifyResult.mismatch > 0) {
                 //         console.log(`Warning: ${verifyResult.mismatch} rows may have failed conversion`);
                 //     }
@@ -175,189 +352,130 @@ export const applyDifferences = async (connection, database, differences) => {
                 //         RENAME COLUMN \`${field.Field}_new\` TO \`${field.Field}\`
                 //     `;
                 //     console.log(`Executing: ${swapColumnsSQL}`);
-                //     await connection.query(swapColumnsSQL);
+                //     await query(swapColumnsSQL);
                     
                 //     console.log(`Completed type conversion for ${field.Field}. Total rows converted: ${totalUpdated}`);
                 // } else {
                     // Original mismatched field handling for simple changes
-                    const isTimestamp = field.Type.toLowerCase().includes('timestamp');
-                    
-                    // Handle timestamp specific attributes
-                    const defaultClause = field.Default === 'CURRENT_TIMESTAMP' 
-                        ? 'DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'
-                        : field.Default === 'NULL'
-                            ? isTimestamp ? 'NULL DEFAULT NULL' : 'DEFAULT NULL'
-                            : field.Default !== undefined && field.Default !== null 
-                                ? `DEFAULT '${field.Default}'` 
-                                : isTimestamp ? 'NULL DEFAULT NULL' : 'DEFAULT NULL';
-                    
-                    // Remove ON UPDATE CURRENT_TIMESTAMP if we're changing away from CURRENT_TIMESTAMP
-                    const hasOnUpdate = currentField?.Default === 'CURRENT_TIMESTAMP' && 
-                        field.Default !== 'CURRENT_TIMESTAMP';
-                    
-                    const modifyFieldSQL = `ALTER TABLE \`${tableName}\` CHANGE \`${field.Field}\` \`${field.Field}\` ${field.Type} ${field.NotNull ? 'NOT NULL' : ''} ${defaultClause}${hasOnUpdate ? ' ON UPDATE CURRENT_TIMESTAMP' : ''};`;
+                    const modifyFieldSQL = `ALTER TABLE \`${tableName}\` CHANGE \`${field.Field}\` \`${field.Field}\` ${columnDefinition(field, currentField)};`;
                     console.log(`Executing: ${modifyFieldSQL}`);
-                    await connection.query(modifyFieldSQL);
+                    await query(modifyFieldSQL);
                 // }
 
             } else if (diff.type === 'missing_index') {
-                const existingColumns = await connection.query(`SHOW COLUMNS FROM \`${diff.tableName}\``);
+                const existingColumns = await query(`SHOW COLUMNS FROM \`${diff.tableName}\``);
                 const columnNames = existingColumns.map(col => col.Field);
 
                 // Check if all columns for the index exist
                 const missingColumns = Array.isArray(diff.index.ColumnName) ? diff.index.ColumnName.filter(col => !columnNames.includes(col) && !columnNames.includes(col.split("(")?.[0])) : [];
 
                 if (missingColumns.length > 0) {
-                    logger(`Cannot create index ${diff.index.Name} on ${diff.tableName}. Missing columns: ${missingColumns.join(', ')}`);
-                    continue; // Skip index creation if columns are missing
+                    throw new Error(`Cannot create index ${diff.index.Name} on ${diff.tableName}. Missing columns: ${missingColumns.join(', ')}`);
                 }
 
-                // Prepare the index columns with proper formatting
-                const indexColumns = diff.index.ColumnName.map(col => {
-                    if(col.includes("(")){                        
-                        
-                        const match = col.match(/^(?:`([^`]+)`|([A-Za-z_][A-Za-z0-9_\$]*))\((\d+)\)$/);
-
-                        if (match) {
-                            const name = match[1] || match[2]; // one of them will be filled
-                            const size = match[3];
-                            console.log(name, size);
-                            return `\`${name}\`(${size})`;
-                        }
-
-                        return `\`${col.split("(")[0]}\``
-                    } else {
-                        return `\`${col}\``; // Format column names
-                    }
-                });
-                // Handle PRIMARY index creation
-                if (diff.index.Name === 'PRIMARY') {
-                    if (indexColumns.length > 0) {
-                        const primaryKeySQL = `ALTER TABLE \`${diff.tableName}\` ADD PRIMARY KEY (${indexColumns.join(', ')});`;
-                        console.log(`Executing: ${primaryKeySQL}`);
-                        await connection.query(primaryKeySQL);
-                    } else {
-                        logger(`Cannot create PRIMARY index on ${diff.tableName}. ColumnName is undefined or empty.`, diff.index);
-                    }
-                } else {
-                    // Create the new index with UNIQUE if applicable
-                    const uniqueSQL = diff.index.Unique ? 'UNIQUE ' : ''; // Check if the index is unique
-                    if (indexColumns.length > 0) {
-                        const addIndexSQL = `ALTER TABLE \`${diff.tableName}\` ADD ${uniqueSQL}INDEX \`${diff.index.Name}\` (${indexColumns.join(', ')});`;
-                        console.log(`Executing: ${addIndexSQL}`);
-                        await connection.query(addIndexSQL);
-                    } else {
-                        logger(`Cannot create index ${diff.index.Name} on ${diff.tableName}. ColumnName is undefined or empty.`, diff.index);
-                    }
+                if (!Array.isArray(diff.index.ColumnName) || diff.index.ColumnName.length === 0) {
+                    throw new Error(`Cannot create index ${diff.index.Name} on ${diff.tableName}. ColumnName is undefined or empty.`);
                 }
+                const addIndexSQL = `ALTER TABLE \`${diff.tableName}\` ${addIndexClause(diff.index)};`;
+                console.log(`Executing: ${addIndexSQL}`);
+                await query(addIndexSQL);
 
             } else if (diff.type === 'extra_index') {
-                // Handle dropping of extra indexes
-                if (diff.index.Name === 'PRIMARY') {
-                    const dropPrimaryKeySQL = `ALTER TABLE \`${diff.tableName}\` DROP PRIMARY KEY;`;
-                    console.log(`Executing: ${dropPrimaryKeySQL}`);
-                    await connection.query(dropPrimaryKeySQL);
-                } else {
-                    const dropIndexSQL = `DROP INDEX \`${diff.index.Name}\` ON \`${diff.tableName}\`;`;
-                    console.log(`Executing: ${dropIndexSQL}`);
-                    await connection.query(dropIndexSQL);
-                }
+                const dropIndexSQL = `ALTER TABLE \`${diff.tableName}\` ${dropIndexClause(diff.index)};`;
+                console.log(`Executing: ${dropIndexSQL}`);
+                await query(dropIndexSQL);
 
             } else if (diff.type === 'mismatched_index') {
                 const { tableName, index } = diff;
-                // Drop the existing index
-                if (index.Name === 'PRIMARY') {
-                    const dropPrimaryKeySQL = `ALTER TABLE \`${tableName}\` DROP PRIMARY KEY;`;
-                    console.log(`Executing: ${dropPrimaryKeySQL}`);
-                    await connection.query(dropPrimaryKeySQL);
-                } else {
-                    const dropIndexSQL = `DROP INDEX \`${index.Name}\` ON \`${tableName}\`;`;
-                    console.log(`Executing: ${dropIndexSQL}`);
-                    await connection.query(dropIndexSQL);
-                }
-                console.log('index', index)
-
-                // Create the new index
-                const uniqueSQL = index.Unique ? 'UNIQUE ' : ''; // Check if the index is unique
-                if (index.Name === 'PRIMARY') {
-                    const addPrimaryKeySQL = `ALTER TABLE \`${tableName}\` ADD PRIMARY KEY (${index.ColumnName.join(', ')});`;
-                    console.log(`Executing: ${addPrimaryKeySQL}`);
-                    await connection.query(addPrimaryKeySQL);
-                } else {
-                    const addIndexSQL = `ALTER TABLE \`${tableName}\` ADD ${uniqueSQL}INDEX \`${index.Name}\` (${index.ColumnName.join(', ')});`; // Include UNIQUE if applicable
-                    console.log(`Executing: ${addIndexSQL}`);
-                    await connection.query(addIndexSQL);
-                }
+                // Drop and re-add in one statement: the index is never missing if the add fails,
+                // and a primary key on an AUTO_INCREMENT column can be replaced
+                const replaceIndexSQL = `ALTER TABLE \`${tableName}\` ${dropIndexClause(index)}, ${addIndexClause(index)};`;
+                console.log(`Executing: ${replaceIndexSQL}`);
+                await query(replaceIndexSQL);
 
             } else if (diff.type === 'missing_procedure' || diff.type === 'mismatched_procedure') {
 
                 // Drop the procedure if it exists
                 const dropCurrentProcSQL = `DROP PROCEDURE IF EXISTS \`${diff.Name}\`;`;
                 console.log(`Executing: ${dropCurrentProcSQL}`);
-                await connection.query(dropCurrentProcSQL);
+                await query(dropCurrentProcSQL);
 
                 // Create the procedure
                 const createProcSQL = diff.Definition;
                 console.log(`Executing: ${createProcSQL}`); // Log the SQL statement
-                await connection.query(createProcSQL);
+                await query(createProcSQL);
 
 
 
             } else if (diff.type === 'missing_view') {
                 const dropViewSQL = `DROP VIEW IF EXISTS \`${diff.viewName}\`;`;
                 console.log(`Executing: ${dropViewSQL}`);
-                await connection.query(dropViewSQL);
+                await query(dropViewSQL);
                 const createViewSQL = diff.definition; // Use the expected definition to create the view
                 console.log(`Executing: ${createViewSQL}`);
-                await connection.query(createViewSQL);
+                await query(createViewSQL);
 
 
             } else if (diff.type === 'mismatched_view') {
                 const dropViewSQL = `DROP VIEW IF EXISTS \`${diff.viewName}\`;`;
                 console.log(`Executing: ${dropViewSQL}`);
-                await connection.query(dropViewSQL);
+                await query(dropViewSQL);
                 const createViewSQL = diff.definition; // Use the expected definition to create the view
                 console.log(`Executing: ${createViewSQL}`);
-                await connection.query(createViewSQL);
+                await query(createViewSQL);
 
 
             } else if (diff.type === 'extra_view') {
                 const dropViewSQL = `DROP VIEW IF EXISTS \`${diff.viewName}\`;`;
                 console.log(`Executing: ${dropViewSQL}`);
-                await connection.query(dropViewSQL);
+                await query(dropViewSQL);
 
 
             } else if (diff.type === 'missing_trigger') {
                 const createTriggerSQL = diff.trigger.Definition;
                 logger('fixing trigger', diff.trigger.Name)
                 console.log(`Executing: ${createTriggerSQL}`);
-                await connection.query(createTriggerSQL);
+                await query(createTriggerSQL);
 
 
             } else if (diff.type === 'mismatched_trigger') {
                 const dropTriggerSQL = `DROP TRIGGER IF EXISTS \`${diff.trigger.Name}\`;`;
                 console.log(`Executing: ${dropTriggerSQL}`);
-                await connection.query(dropTriggerSQL);
+                await query(dropTriggerSQL);
                 const createTriggerSQL = diff.trigger.Definition;
                 console.log(`Executing: ${createTriggerSQL}`);
-                await connection.query(createTriggerSQL);
+                await query(createTriggerSQL);
 
 
             } else if (diff.type === 'extra_trigger') {
                 const dropTriggerSQL = `DROP TRIGGER IF EXISTS \`${diff.trigger.Name}\`;`;
                 console.log(`Executing: ${dropTriggerSQL}`);
-                await connection.query(dropTriggerSQL);
+                await query(dropTriggerSQL);
+
+            } else if (diff.type === 'missing_row') {
+                const columns = Object.keys(diff.row);
+                const insertRowSQL = `INSERT INTO \`${diff.tableName}\` (${columns.map(col => `\`${col}\``).join(', ')}) VALUES (${columns.map(() => '?').join(', ')});`;
+                console.log(`Executing: ${insertRowSQL}`, JSON.stringify(diff.row));
+                await query(insertRowSQL, columns.map(col => diff.row[col]));
 
             }else if(diff.type === 'missing_database'){
                 logger('fixing missing database', diff.database)
                 const createDatabaseSQL = `CREATE DATABASE \`${diff.database}\`;`;
                 console.log(`Executing: ${createDatabaseSQL}`);
-                await connection.query(createDatabaseSQL);
+                await query(createDatabaseSQL);
             } else {
-                logger(`Error applying difference of type:${diff.type}`, diff)
+                throw new Error(`Unknown difference type: ${diff.type}`);
             }
         } catch (e) {
-            logger(`Error applying difference of type:${diff.type}. Error: ${e.message}`, diff);
+            console.error(`Failed to apply ${describeDifference(diff)}: ${e.message}`);
+            logger(diff);
+            failures.push({ type: diff.type, tableName: diff.tableName, description: describeDifference(diff), message: e.message });
         }
     }
-}
+
+    return { attempted: differences.length, failures, warnings };
+}
+
+// One line summing up the failures from applyDifferences, for errors and logs
+export const describeFailures = ({ attempted, failures }) =>
+    `${failures.length} of ${attempted} difference(s) failed to apply: ${failures.map(f => `${f.description} (${f.message})`).join('; ')}`;

package/src/cli.js

@@ -1,35 +1,62 @@
-import yargs from 'yargs';
-import { hideBin } from 'yargs/helpers';
-import { setVerbose } from './loggers.js';
-
-export const cli = () => {
-    const argv = yargs(hideBin(process.argv))
-        .command('dump', 'Dump the database structure to files')
-        .command('compare', 'Compare the database structure with JSON dumps (dry run)')
-        .command('update', 'Update the database structure based on JSON dumps')
-        .command('ui', 'Open the UI')
-        .option('uiPort', { alias: 'U', describe: 'UI port', default: 3600 })
-        .option('openUI', { alias: 'O', describe: 'Open UI', default: true })
-        .option('host', { alias: 'h', describe: 'Database host', default: 'localhost' })
-        .option('port', { alias: 'P', describe: 'Database port', default: 3306 })
-        .option('user', { alias: 'u', describe: 'Database user', default: 'root' })
-        .option('password', { alias: 'p', describe: 'Database password', default: '' })
-        .option('database', { alias: 'd', describe: 'Target database', default: null })
-        .option('output', { alias: 'o', describe: 'Output directory for dumping. default is one level up from the current directory in a new folder called db-dump', default: '../db-dump' })
-        .option('verbose', { alias: 'v', describe: 'Enable verbose logging', type: 'boolean', default: false })
-        // .demandCommand(1)
-        .help()
-        .argv;
-
-    const command = argv._[0];
-    const config = {
-        host: argv.host,
-        port: argv.port,
-        user: argv.user,
-        password: argv.password,
-    };
-
-    setVerbose(argv.verbose);
-
-    return { command, config, argv };
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+import { setVerbose } from './loggers.js';
+import { promptForOptions, promptForPassword, DATABASE_COMMANDS } from './prompt.js';
+
+const COMMANDS = ['dump', 'compare', 'update', 'ui', 'rebuild'];
+
+/**
+ * Parse the command line. With no command, asks for the command and its options
+ * in a terminal, or prints the help otherwise. command is null when there's nothing to run.
+ */
+export const cli = async () => {
+    const parser = yargs(hideBin(process.argv))
+        .command('dump', 'Dump the database structure to files')
+        .command('compare', 'Compare the database structure with JSON dumps (dry run)')
+        .command('update', 'Update the database structure based on JSON dumps')
+        .command('ui', 'Open the UI')
+        .command('rebuild', "Rebuild the dump's JSON from each table's CREATE TABLE and list what changes (dry run unless --write)")
+        .option('uiPort', { alias: 'U', describe: 'UI port', default: 3600 })
+        .option('openUI', { alias: 'O', describe: 'Open UI', default: true })
+        .option('host', { alias: 'h', describe: 'Database host', default: 'localhost' })
+        .option('port', { alias: 'P', describe: 'Database port', default: 3306 })
+        .option('user', { alias: 'u', describe: 'Database user', default: 'root' })
+        // No default, so a missing password can be asked for; -p with no value means none
+        .option('password', { alias: 'p', describe: 'Database password (asked for in a terminal when not given)', type: 'string' })
+        .option('database', { alias: 'd', describe: 'Target database', default: null })
+        .option('output', { alias: 'o', describe: 'Output directory for dumping. default is one level up from the current directory in a new folder called db-dump', default: '../db-dump' })
+        .option('write', { describe: 'With rebuild, write the rebuilt files', type: 'boolean', default: false })
+        .option('verbose', { alias: 'v', describe: 'Enable verbose logging', type: 'boolean', default: false })
+        // .demandCommand(1)
+        .help();
+    const argv = parser.argv;
+
+    let command = argv._[0];
+    if (command === undefined) {
+        if (process.stdin.isTTY && process.stdout.isTTY) {
+            command = await promptForOptions(argv);
+        } else {
+            parser.showHelp();
+            command = null;
+        }
+    } else if (!COMMANDS.includes(command)) {
+        console.error(`Unknown command: ${command}. Use one of: ${COMMANDS.join(', ')}`);
+        command = null;
+    }
+
+    const interactive = process.stdin.isTTY && process.stdout.isTTY;
+    if (DATABASE_COMMANDS.includes(command) && argv.password === undefined) {
+        argv.password = interactive ? await promptForPassword(argv) : '';
+    }
+
+    const config = {
+        host: argv.host,
+        port: argv.port,
+        user: argv.user,
+        password: argv.password,
+    };
+
+    setVerbose(argv.verbose);
+
+    return { command, config, argv };
 };