@scandipwa/magento-scripts 2.0.0-alpha.0 → 2.0.0-alpha.3

package/lib/config/versions/magento-2.3.4-p1.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const { elasticsearch68 } = require('../services/elasticsearch/versions');
 const { defaultMagentoConfig } = require('../magento-config');
 const { magento23PHPExtensionList } = require('../magento/required-php-extensions');
 const { repo } = require('../php/base-repo');
@@ -27,9 +28,7 @@ module.exports = ({ templateDir } = {}) => ({
         mariadb: {
             version: '10.2'
         },
-        elasticsearch: {
-            version: '6.8.16'
-        },
+        elasticsearch: elasticsearch68(),
         composer: {
             version: '1'
         },

package/lib/config/versions/magento-2.3.4-p2.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const { elasticsearch68 } = require('../services/elasticsearch/versions');
 const { defaultMagentoConfig } = require('../magento-config');
 const { magento23PHPExtensionList } = require('../magento/required-php-extensions');
 const { repo } = require('../php/base-repo');
@@ -27,9 +28,7 @@ module.exports = ({ templateDir } = {}) => ({
         mariadb: {
             version: '10.2'
         },
-        elasticsearch: {
-            version: '6.8.16'
-        },
+        elasticsearch: elasticsearch68(),
         composer: {
             version: '1'
         },

package/lib/config/versions/magento-2.3.4.js

@@ -1,4 +1,5 @@
 const path = require('path');
+const { elasticsearch68 } = require('../services/elasticsearch/versions');
 const { defaultMagentoConfig } = require('../magento-config');
 const { magento23PHPExtensionList } = require('../magento/required-php-extensions');
 const { repo } = require('../php/base-repo');
@@ -27,9 +28,7 @@ module.exports = ({ templateDir } = {}) => ({
         mariadb: {
             version: '10.2'
         },
-        elasticsearch: {
-            version: '6.8.16'
-        },
+        elasticsearch: elasticsearch68(),
         composer: {
             version: '1'
         },

package/lib/tasks/cleanup.js

@@ -1,6 +1,6 @@
 const { removeCacheFolder } = require('./cache');
 const { stopServices } = require('./docker');
-const { removeVolumes } = require('./docker/volumes');
+const { removeVolumes } = require('./docker/volume/tasks');
 const {
     uninstallMagento,
     removeMagento

package/lib/tasks/database/connect-to-database.js

@@ -0,0 +1,117 @@
+const mysql2 = require('mysql2/promise');
+const UnknownError = require('../../errors/unknown-error');
+const { execAsyncSpawn } = require('../../util/exec-async-command');
+const sleep = require('../../util/sleep');
+const { createMagentoDatabase } = require('./create-magento-database');
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const waitForDatabaseInitialization = () => ({
+    title: 'Waiting for Database to initialize',
+    task: async (ctx, task) => {
+        const { mariadb } = ctx.config.docker.getContainers();
+
+        let databaseReadyForConnections = false;
+
+        while (!databaseReadyForConnections) {
+            const databaseOutput = await execAsyncSpawn(`docker logs ${mariadb.name}`);
+            if (databaseOutput.includes('ready for connections')) {
+                databaseReadyForConnections = true;
+                break;
+            } else if (databaseOutput.includes('Initializing database files')) {
+                task.output = `${mariadb._} is initializing database files!
+Please wait, this will take some time and do not restart the ${mariadb._} container until initialization is finished!`;
+
+                let databaseFinishedInitialization = false;
+                while (!databaseFinishedInitialization) {
+                    const databaseOutput2 = await execAsyncSpawn(`docker logs ${mariadb.name}`);
+                    if (databaseOutput2.includes('init process done.') && !databaseFinishedInitialization) {
+                        databaseFinishedInitialization = true;
+                        break;
+                    }
+                    await sleep(2000);
+                }
+            }
+
+            await sleep(2000);
+        }
+    },
+    options: {
+        bottomBar: 10
+    }
+});
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const gettingDatabaseConnection = () => ({
+    title: 'Getting Database connection',
+    task: async (ctx, task) => {
+        const { config: { docker }, ports } = ctx;
+        const { mariadb } = docker.getContainers(ctx.ports);
+        let tries = 0;
+        const maxTries = 20;
+        const errors = [];
+
+        while (tries < maxTries) {
+            tries++;
+            try {
+                const connection = await mysql2.createConnection({
+                    host: '127.0.0.1',
+                    port: ports.mariadb,
+                    user: mariadb.env.MARIADB_USER,
+                    password: mariadb.env.MARIADB_PASSWORD,
+                    database: mariadb.env.MARIADB_DATABASE
+                });
+
+                ctx.databaseConnection = connection;
+                break;
+            } catch (e) {
+                errors.push(e);
+            }
+            await sleep(1000);
+        }
+
+        if (tries === maxTries) {
+            throw new UnknownError(`Unable to connect to ${ mariadb._ } server. Check your server configuration!\n\n${ errors.join(' ') }`);
+        }
+
+        task.title = `${ mariadb._ } server connected!`;
+    }
+});
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const terminatingExistingConnection = () => ({
+    title: 'Terminating existing Database connection',
+    skip: (ctx) => !ctx.databaseConnection,
+    task: (ctx) => {
+        ctx.databaseConnection.destroy();
+    }
+});
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const connectToDatabase = () => ({
+    title: 'Connecting to Database server',
+    skip: (ctx) => ctx.skipSetup,
+    task: (ctx, task) => task.newListr([
+        waitForDatabaseInitialization(),
+        createMagentoDatabase(),
+        terminatingExistingConnection(),
+        gettingDatabaseConnection()
+    ], {
+        concurrent: false,
+        rendererOptions: {
+            collapse: true
+        }
+    }),
+    options: {
+        bottomBar: 10
+    }
+});
+
+module.exports = connectToDatabase;

package/lib/tasks/database/create-magento-database.js

@@ -0,0 +1,18 @@
+const { execAsyncSpawn } = require('../../util/exec-async-command');
+
+/**
+ * Will create database 'magento' in MariaDB if it does not exist for some reason
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const createMagentoDatabase = () => ({
+    title: 'Creating Magento database',
+    task: async (ctx, task) => {
+        const { mariadb } = ctx.config.docker.getContainers();
+        task.title = `Creating Magento database in ${ mariadb._ }`;
+        await execAsyncSpawn(`docker exec ${mariadb.name} mysql -umagento -pmagento -h 127.0.0.1 -e "CREATE DATABASE IF NOT EXISTS magento;"`);
+    }
+});
+
+module.exports = {
+    createMagentoDatabase
+};

package/lib/tasks/{mysql → database}/dump-theme-config.js

@@ -6,9 +6,9 @@ const { isTableExists } = require('../../util/database');
 const dumpThemeConfig = () => ({
     title: 'Dumping themes and theme configuration',
     task: async (ctx, task) => {
-        const { mysqlConnection } = ctx;
+        const { databaseConnection } = ctx;
         if (await isTableExists('magento', 'theme', ctx)) {
-            const [themes] = await mysqlConnection.query('select * from theme;');
+            const [themes] = await databaseConnection.query('select * from theme;');
             if (themes.length === 0) {
                 ctx.themeDump = {
                     themes: [],
@@ -17,7 +17,7 @@ const dumpThemeConfig = () => ({
             }
 
             if (await isTableExists('magento', 'core_config_data', ctx)) {
-                const [themeIdConfig] = await mysqlConnection.query('select * from core_config_data where path = \'design/theme/theme_id\';');
+                const [themeIdConfig] = await databaseConnection.query('select * from core_config_data where path = \'design/theme/theme_id\';');
                 if (themeIdConfig.length !== 0) {
                     ctx.themeDump = {
                         themes,

package/lib/tasks/{mysql → database}/fix-db.js

@@ -8,14 +8,14 @@ const varnishConfigSetup = require('../magento/setup-magento/varnish-config');
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
  */
 const enableForeignKeyCheck = () => ({
-    task: ({ mysqlConnection }) => mysqlConnection.query('SET FOREIGN_KEY_CHECKS = 0;')
+    task: ({ databaseConnection }) => databaseConnection.query('SET FOREIGN_KEY_CHECKS = 0;')
 });
 
 /**
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
  */
 const disableForeignKeyCheck = () => ({
-    task: ({ mysqlConnection }) => mysqlConnection.query('SET FOREIGN_KEY_CHECKS = 1;')
+    task: ({ databaseConnection }) => databaseConnection.query('SET FOREIGN_KEY_CHECKS = 1;')
 });
 
 /**

package/lib/tasks/{mysql/import-dump-to-mysql.js → database/import-dump-to-database.js}

@@ -4,7 +4,7 @@ const KnownError = require('../../errors/known-error');
 const UnknownError = require('../../errors/unknown-error');
 const { execAsyncSpawn, execCommandTask } = require('../../util/exec-async-command');
 const pathExists = require('../../util/path-exists');
-const connectToMySQL = require('./connect-to-mysql');
+const connectToDatabase = require('./connect-to-database');
 
 /**
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
@@ -13,10 +13,10 @@ const copyDatabaseDumpIntoContainer = () => ({
     title: 'Copying database dump into container',
     task: async (ctx, task) => {
         const { config: { docker }, ports } = ctx;
-        const { mysql } = docker.getContainers(ports);
+        const { mariadb } = docker.getContainers(ports);
 
         return task.newListr(
-            execCommandTask(`docker cp ${ctx.importDb} ${mysql.name}:/dump.sql`, {
+            execCommandTask(`docker cp ${ctx.importDb} ${mariadb.name}:/dump.sql`, {
                 logOutput: true
             })
         );
@@ -29,10 +29,10 @@ const copyDatabaseDumpIntoContainer = () => ({
 const runSetGlobalLogBinTrustFunctionCreatorsCommand = () => ({
     task: async (ctx, task) => {
         const { config: { docker }, ports } = ctx;
-        const { mysql } = docker.getContainers(ports);
+        const { mariadb } = docker.getContainers(ports);
 
         return task.newListr(
-            execCommandTask(`docker exec ${mysql.name} bash -c 'mysql -uroot -p${mysql.env.MYSQL_ROOT_PASSWORD} -e "SET GLOBAL log_bin_trust_function_creators = 1;"'`)
+            execCommandTask(`docker exec ${mariadb.name} bash -c 'mysql -uroot -p${mariadb.env.MARIADB_ROOT_PASSWORD} -e "SET GLOBAL log_bin_trust_function_creators = 1;"'`)
         );
     }
 });
@@ -63,8 +63,8 @@ Note that you will lose your existing database!`,
         });
 
         if (deleteDatabaseMagentoChoice === 'delete') {
-            await ctx.mysqlConnection.query('DROP DATABASE IF EXISTS magento;');
-            await ctx.mysqlConnection.query('CREATE DATABASE magento;');
+            await ctx.databaseConnection.query('DROP DATABASE IF EXISTS magento;');
+            await ctx.databaseConnection.query('CREATE DATABASE magento;');
             return;
         }
         task.skip();
@@ -77,28 +77,28 @@ Note that you will lose your existing database!`,
 const executeImportDumpSQL = () => ({
     task: async (ctx, task) => {
         const { config: { docker }, ports } = ctx;
-        const { mysql } = docker.getContainers(ports);
+        const { mariadb } = docker.getContainers(ports);
 
-        const userCredentialsForMySQLCLI = await task.prompt({
+        const userCredentialsForMariaDBCLI = await task.prompt({
             type: 'Select',
-            message: 'Which user do you want to use to import db in MySQL client?',
+            message: `Which user do you want to use to import db in ${ mariadb._ } client?`,
             choices: [
                 {
-                    name: `--user=root --password=${mysql.env.MYSQL_ROOT_PASSWORD}`,
+                    name: `--user=root --password=${mariadb.env.MARIADB_ROOT_PASSWORD}`,
                     message: `root (${logger.style.command('Probably safest option')})`
                 },
                 {
-                    name: `--user=${mysql.env.MYSQL_USER} --password=${mysql.env.MYSQL_PASSWORD}`,
-                    message: `${mysql.env.MYSQL_USER}`
+                    name: `--user=${mariadb.env.MARIADB_USER} --password=${mariadb.env.MARIADB_PASSWORD}`,
+                    message: `${mariadb.env.MARIADB_USER}`
                 }
             ]
         });
 
-        const importCommand = `docker exec ${mysql.name} bash -c "mysql ${userCredentialsForMySQLCLI} magento < ./dump.sql"`;
+        const importCommand = `docker exec ${mariadb.name} bash -c "mysql ${userCredentialsForMariaDBCLI} magento < ./dump.sql"`;
 
         const startImportTime = Date.now();
         const tickInterval = setInterval(() => {
-            task.title = `Importing Database Dump To MySQL, ${Math.floor((Date.now() - startImportTime) / 1000)}s in progress...`;
+            task.title = `Importing Database Dump To ${ mariadb._ }, ${Math.floor((Date.now() - startImportTime) / 1000)}s in progress...`;
         }, 1000);
 
         try {
@@ -135,7 +135,7 @@ ${logger.style.command('sed -i \'s/utf8mb4_0900_ai_ci/utf8mb4_general_ci/g\' dum
 
                 if (confirmFixingCollation === 'yes') {
                     task.output = 'Running fix command...';
-                    await execAsyncSpawn(`docker exec ${mysql.name} bash -c "sed -i 's/utf8mb4_0900_ai_ci/utf8mb4_general_ci/g' dump.sql"`);
+                    await execAsyncSpawn(`docker exec ${mariadb.name} bash -c "sed -i 's/utf8mb4_0900_ai_ci/utf8mb4_general_ci/g' dump.sql"`);
 
                     task.output = 'Trying to import dump again...';
                     try {
@@ -175,8 +175,8 @@ You can try replacing all occurrences of ${logger.style.misc('utf8mb4_0900_ai_ci
 /**
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
  */
-const importDumpToMySQL = () => ({
-    title: 'Importing Database Dump To MySQL',
+const importDumpToDatabase = () => ({
+    title: 'Importing Database Dump',
     task: async (ctx, task) => {
         if (!await pathExists(ctx.importDb)) {
             throw new KnownError(`Dump file at ${ctx.importDb} does not exist. Please provide correct relative path to the file`);
@@ -187,7 +187,7 @@ const importDumpToMySQL = () => ({
             deleteDatabaseBeforeImportingDumpPrompt(),
             runSetGlobalLogBinTrustFunctionCreatorsCommand(),
             executeImportDumpSQL(),
-            connectToMySQL(),
+            connectToDatabase(),
             {
                 task: () => {
                     task.title = 'Database imported!';
@@ -201,4 +201,4 @@ const importDumpToMySQL = () => ({
     }
 });
 
-module.exports = importDumpToMySQL;
+module.exports = importDumpToDatabase;

package/lib/tasks/{mysql/import-remote-db/ssh/mysqldump-command.js → database/import-remote-db/ssh/database-dump-command.js}

@@ -1,4 +1,4 @@
-const mysqlDumpCommandWithOptions = [
+const databaseDumpCommandWithOptions = [
     'mysqldump',
     'magento',
     '--skip-lock-tables',
@@ -9,4 +9,4 @@ const mysqlDumpCommandWithOptions = [
     '--no-tablespaces'
 ];
 
-module.exports = mysqlDumpCommandWithOptions;
+module.exports = databaseDumpCommandWithOptions;

package/lib/tasks/{mysql → database}/import-remote-db/ssh/readymage.js

@@ -1,7 +1,7 @@
 const mergeFiles = require('merge-files');
 const { orderTables, customerTables } = require('../../magento-tables');
 const { execAsyncSpawn } = require('../../../../util/exec-async-command');
-const mysqlDumpCommandWithOptions = require('./mysqldump-command');
+const databaseDumpCommandWithOptions = require('./database-dump-command');
 
 /**
  * @type {() => import('listr2').ListrTask<import('../../../../../typings/context').ListrContext & { ssh: import('node-ssh').NodeSSH }>}
@@ -25,7 +25,7 @@ const readymageSSH = () => ({
                  */
                 await ssh.execCommand(
                     [
-                        ...mysqlDumpCommandWithOptions,
+                        ...databaseDumpCommandWithOptions,
                         ...[...orderTables, ...customerTables].map((table) => `--ignore-table=magento.${table}`),
                         '--result-file=dump-0.sql'
                     ].join(' ')
@@ -33,7 +33,7 @@ const readymageSSH = () => ({
 
                 await ssh.execCommand(
                     [
-                        ...mysqlDumpCommandWithOptions,
+                        ...databaseDumpCommandWithOptions,
                         '--no-data',
                         '--result-file=dump-1.sql',
                         ...[...orderTables, ...customerTables]
@@ -41,7 +41,7 @@ const readymageSSH = () => ({
                 );
             } else {
                 task.output = 'Making remote database dump file with customers data...';
-                await ssh.execCommand([...mysqlDumpCommandWithOptions, '--result-file=dump.sql'].join(' '));
+                await ssh.execCommand([...databaseDumpCommandWithOptions, '--result-file=dump.sql'].join(' '));
             }
 
             if (!noCompress) {

package/lib/tasks/{mysql → database}/import-remote-db/ssh/regular-server.js

@@ -1,7 +1,7 @@
 const mergeFiles = require('merge-files');
 const { orderTables, customerTables } = require('../../magento-tables');
 const { execAsyncSpawn } = require('../../../../util/exec-async-command');
-const mysqlDumpCommandWithOptions = require('./mysqldump-command');
+const databaseDumpCommandWithOptions = require('./database-dump-command');
 const KnownError = require('../../../../errors/known-error');
 /**
  * @type {() => import('listr2').ListrTask<import('../../../../../typings/context').ListrContext & { ssh: import('node-ssh').NodeSSH }>}
@@ -27,7 +27,7 @@ Do not enter "--result-file" option, we need to control that part.
 
 (documentation reference available here: https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html)
 `,
-                initial: mysqlDumpCommandWithOptions.join(' ')
+                initial: databaseDumpCommandWithOptions.join(' ')
             });
 
             if (dumpCommand.includes('--result-file')) {

package/lib/tasks/{mysql → database}/index.js

@@ -1,6 +1,6 @@
 module.exports = {
-    connectToMySQL: require('./connect-to-mysql'),
-    importDumpToMySQL: require('./import-dump-to-mysql'),
+    connectToDatabase: require('./connect-to-database'),
+    importDumpToDatabase: require('./import-dump-to-database'),
     fixDB: require('./fix-db'),
     dumpThemeConfig: require('./dump-theme-config'),
     restoreThemeConfig: require('./restore-theme-config')

package/lib/tasks/{mysql → database}/restore-theme-config.js

@@ -6,7 +6,7 @@ const { updateTableValues } = require('../../util/database');
 const restoreThemeConfig = () => ({
     title: 'Restoring themes and theme configuration',
     task: async (ctx, task) => {
-        const { themeDump, mysqlConnection } = ctx;
+        const { themeDump, databaseConnection } = ctx;
 
         const { themeIdConfig, themes } = themeDump;
 
@@ -22,7 +22,7 @@ const restoreThemeConfig = () => ({
                     path: themeIdConfig.path,
                     value: themeIdConfig.value
                 }
-            ], { mysqlConnection, task });
+            ], { databaseConnection, task });
         }
 
         if (themes.length === 0) {
@@ -31,10 +31,10 @@ const restoreThemeConfig = () => ({
 
         // restore themes
         const themeKeys = Object.keys(themes[0]);
-        await mysqlConnection.query('delete from theme;');
+        await databaseConnection.query('delete from theme;');
 
         for (const theme of themes) {
-            await mysqlConnection.query(`
+            await databaseConnection.query(`
                 insert into theme (${themeKeys.join(', ')})
                 values (${themeKeys.map(() => '?').join(', ')})
             `, Object.values(theme));

package/lib/tasks/docker/convert-legacy-volumes.js

@@ -1,10 +1,11 @@
+/* eslint-disable no-param-reassign */
 /* eslint-disable max-len */
 const { stopServices } = require('./index');
 const { getBaseConfig, getConfigFromMagentoVersion } = require('../../config');
 const getDockerConfig = require('../../config/docker');
 const { execAsyncSpawn } = require('../../util/exec-async-command');
 const { folderName, legacyFolderName } = require('../../util/prefix');
-const { createVolume } = require('./volumes');
+const { volumeApi } = require('./volume');
 
 /**
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
@@ -14,20 +15,20 @@ const convertLegacyVolumes = () => ({
         const { config: { overridenConfiguration } } = ctx;
         const newDockerConfig = await getDockerConfig(ctx, overridenConfiguration, getBaseConfig(process.cwd(), folderName));
 
-        const newVolumeNames = Object.values(newDockerConfig.volumes).filter((v) => !v.opts).map(({ name }) => name);
+        const newVolumeNames = Object.values(newDockerConfig.volumes).filter((v) => !v.opt).map(({ name }) => name);
 
-        const existingVolumes = (await execAsyncSpawn('docker volume ls -q')).split('\n');
+        const existingVolumes = await volumeApi.ls({ formatToJSON: true });
 
         if (newVolumeNames.every((v) => existingVolumes.includes(v))) {
             return;
         }
 
         const legacyDockerConfig = await getDockerConfig(ctx, overridenConfiguration, getBaseConfig(process.cwd(), legacyFolderName));
-        const legacyVolumeNames = Object.values(legacyDockerConfig.volumes).filter((v) => !v.opts).map(({ name }) => name);
+        const legacyVolumeNames = Object.values(legacyDockerConfig.volumes).filter((v) => !v.opt).map(({ name }) => name);
 
         if (
-            newVolumeNames.every((name) => !existingVolumes.includes(name))
-            && legacyVolumeNames.every((name) => existingVolumes.includes(name))
+            newVolumeNames.every((name) => !existingVolumes.some((v) => v.Name === name))
+            && legacyVolumeNames.every((name) => existingVolumes.some((v) => v.Name === name))
         ) {
             ctx.config = await getConfigFromMagentoVersion(ctx, {
                 magentoVersion: ctx.magentoVersion,
@@ -46,7 +47,7 @@ const convertLegacyVolumes = () => ({
                             const legacyVolumeConfig = legacyDockerConfig.volumes[volumeName];
 
                             subTask.output = `Creating volume ${volumeConfig.name}...`;
-                            await createVolume(volumeConfig);
+                            await volumeApi.create(volumeConfig);
                             subTask.output = `Copying data from ${legacyVolumeConfig.name} to ${volumeConfig.name}...`;
                             await execAsyncSpawn(
                                 `docker run --rm -v ${legacyVolumeConfig.name}:/from:ro -v ${volumeConfig.name}:/to alpine ash -c "cd /from; cp -av . /to"`, {