@scandipwa/magento-scripts 2.0.0-alpha.0 → 2.0.0-alpha.3

package/lib/tasks/docker/convert-mysql-to-mariadb.js

@@ -0,0 +1,284 @@
+const path = require('path');
+const fs = require('fs');
+const logger = require('@scandipwa/scandipwa-dev-utils/logger');
+const { volumeApi, createVolumes } = require('./volume');
+const { execAsyncSpawn } = require('../../util/exec-async-command');
+const { containerApi, stopContainers, pullImages } = require('./containers');
+const { importDumpToDatabase, connectToDatabase } = require('../database');
+const { setPrefix } = require('../prefix');
+const getProjectConfiguration = require('../../config/get-project-configuration');
+const { getAvailablePorts, getCachedPorts } = require('../../config/get-port-config');
+const { saveConfiguration } = require('../../config/save-config');
+const { buildProjectImage, buildDebugProjectImage } = require('./project-image-builder');
+const checkPHPVersion = require('../requirements/php-version');
+const { getComposerVersionTask } = require('../composer');
+const { prepareFileSystem } = require('../file-system');
+const { installMagentoProject } = require('../magento');
+const enableMagentoComposerPlugins = require('../magento/enable-magento-composer-plugins');
+const { startServices } = require('./index');
+const dockerNetwork = require('./network');
+const KnownError = require('../../errors/known-error');
+const { createCacheFolder } = require('../cache');
+const { getSystemConfigTask } = require('../../config/system-config');
+const sleep = require('../../util/sleep');
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const convertMySQLDatabaseToMariaDB = () => ({
+    task: async (ctx, task) => {
+        const mysqlVolumeName = `${ ctx.config.baseConfig.prefix }_mysql-data`;
+        const volumes = await volumeApi.ls({ formatToJSON: true });
+        if (ctx.isArmMac) {
+            if (volumes.some(
+                (volume) => volume.Name === mysqlVolumeName
+            ) && !volumes.some(
+                (volume) => volume.Name === ctx.config.docker.volumes.mariadb.name
+            )
+            ) {
+                task.title = 'Converting MySQL database to MariaDB';
+                return task.newListr([
+                    stopContainers(),
+                    {
+                        title: 'Converting MySQL volume to MariaDB',
+                        task: async (subCtx, subTask) => {
+                            subTask.output = `Creating volume ${subCtx.config.docker.volumes.mariadb.name}...`;
+                            await volumeApi.create(subCtx.config.docker.volumes.mariadb);
+                            subTask.output = `Copying data from ${mysqlVolumeName} to ${subCtx.config.docker.volumes.mariadb.name}...`;
+                            await execAsyncSpawn(
+                            // eslint-disable-next-line max-len
+                                `docker run --rm -v ${mysqlVolumeName}:/from:ro -v ${subCtx.config.docker.volumes.mariadb.name}:/to alpine ash -c "cd /from; cp -av . /to"`, {
+                                    callback: (t) => {
+                                        subTask.output = t;
+                                    }
+                                }
+                            );
+                            subTask.output = 'Deleting old volume...';
+                            await execAsyncSpawn(`docker volume rm ${mysqlVolumeName}`);
+                        },
+                        options: {
+                            bottomBar: 10
+                        }
+                    }
+                ]);
+            }
+
+            return;
+        }
+
+        if (volumes.some(
+            (volume) => volume.Name === mysqlVolumeName
+        ) && !volumes.some(
+            (volume) => volume.Name === ctx.config.docker.volumes.mariadb.name
+        )
+        ) {
+            task.title = 'Converting MySQL database to MariaDB';
+            const confirmConvert = await task.prompt({
+                type: 'Select',
+                message: `We see that you have old mysql-data volume in the project!
+Since ${logger.style.code('magento-scripts@2.0.0')} ${logger.style.misc('MariaDB')} is used as database instead of MySQL.
+
+To use MariaDB with data from MySQL we need to convert your database.
+`,
+                choices: [
+                    {
+                        name: 'yes',
+                        message: 'Okay, let\'s do that!'
+                    },
+                    {
+                        name: 'no',
+                        message: 'ABORT ABORT ABORT'
+                    }
+                ]
+            });
+
+            if (confirmConvert === 'yes') {
+                const pathToMySQLDumpFile = path.join(process.cwd(), 'mysql-database.sql');
+                const containerName = 'mysql-database';
+                task.output = 'Creating dump file from MySQL database...';
+
+                const existingContainers = await containerApi.ls({ formatToJSON: true, all: true });
+
+                if (existingContainers.some((c) => c.Names === containerName)) {
+                    await execAsyncSpawn(`docker container stop ${containerName}`);
+                    await execAsyncSpawn(`docker container rm ${containerName}`);
+                }
+
+                task.output = 'Starting MySQL server...';
+
+                await containerApi.run(
+                    {
+                        mounts: [
+                            `source=${ mysqlVolumeName },target=/var/lib/mysql`
+                        ],
+                        env: {
+                            MYSQL_PORT: 3306,
+                            MYSQL_ROOT_PASSWORD: 'scandipwa',
+                            MYSQL_USER: 'magento',
+                            MYSQL_PASSWORD: 'magento',
+                            MYSQL_DATABASE: 'magento'
+                        },
+                        command: [
+                            '--log_bin_trust_function_creators=1',
+                            '--default-authentication-plugin=mysql_native_password',
+                            '--max_allowed_packet=1GB',
+                            '--bind-address=0.0.0.0'
+                        ]
+                            .join(' '),
+                        securityOptions: [
+                            'seccomp=unconfined'
+                        ],
+                        name: containerName,
+                        image: `mysql:${ ctx.config.overridenConfiguration.configuration.mysql.version }`
+                    }
+                );
+
+                task.output = 'Waiting for MySQL to initialize...';
+
+                let mysqlReadyForConnections = false;
+
+                while (!mysqlReadyForConnections) {
+                    const mysqlOutput = await execAsyncSpawn(`docker logs ${containerName}`);
+                    if (mysqlOutput.includes('ready for connections')) {
+                        mysqlReadyForConnections = true;
+                        task.output = 'MySQL is ready!';
+                        break;
+                    } else if (mysqlOutput.includes('Initializing database files')) {
+                        task.output = `MySQL is initializing database files!
+Please wait, this will take some time and do not restart the MySQL container until initialization is finished!`;
+
+                        let mysqlFinishedInitialization = false;
+                        while (!mysqlFinishedInitialization) {
+                            const mysqlOutput = await execAsyncSpawn(`docker logs ${containerName}`);
+                            if (mysqlOutput.includes('init process done.') && !mysqlFinishedInitialization) {
+                                mysqlFinishedInitialization = true;
+                                task.output = 'MySQL is initialized!';
+                                break;
+                            }
+                            await sleep(2000);
+                        }
+                    }
+
+                    await sleep(2000);
+                }
+
+                task.output = 'Dumping MySQL database to dump file...';
+
+                await containerApi.exec(
+                    [
+                        'mysqldump',
+                        '--user=root',
+                        '--password=scandipwa',
+                        'magento',
+                        `--result-file=${ path.parse(pathToMySQLDumpFile).base }`
+                    ].join(' '),
+                    containerName,
+                    {},
+                    {
+                        callback: (t) => {
+                            task.output = t;
+                        }
+                    }
+                );
+
+                task.output = 'Copying dump file from container to system...';
+
+                await execAsyncSpawn(
+                    `docker cp ${containerName}:/${ path.parse(pathToMySQLDumpFile).base } ${ pathToMySQLDumpFile }`
+                );
+
+                task.output = 'Removing migration container...';
+
+                await execAsyncSpawn(
+                    `docker container stop ${containerName}`
+                );
+                await execAsyncSpawn(
+                    `docker container rm ${containerName}`
+                );
+
+                ctx.importDb = pathToMySQLDumpFile;
+
+                return task.newListr([
+                    createCacheFolder(),
+                    getSystemConfigTask(),
+                    getCachedPorts(),
+                    stopContainers(),
+                    setPrefix(),
+                    getProjectConfiguration(),
+                    // get fresh ports
+                    getAvailablePorts(),
+                    saveConfiguration(),
+                    pullImages(),
+                    dockerNetwork.tasks.createNetwork(),
+                    createVolumes(),
+                    {
+                        task: (ctx, task) => task.newListr([
+                            buildProjectImage(),
+                            buildDebugProjectImage()
+                        ], {
+                            concurrent: true
+                        })
+                    },
+                    checkPHPVersion(),
+                    getComposerVersionTask(),
+                    prepareFileSystem(),
+                    installMagentoProject(),
+                    enableMagentoComposerPlugins(),
+                    startServices(),
+                    connectToDatabase(),
+                    importDumpToDatabase(),
+                    {
+                        task: async (subCtx, subTask) => {
+                            const confirmDeleteOldVolume = await subTask.prompt({
+                                type: 'Select',
+                                message: 'Okay, looks like conversion went well, do you want to delete old mysql-data volume?',
+                                choices: [
+                                    {
+                                        name: 'yes',
+                                        message: 'Sure'
+                                    },
+                                    {
+                                        name: 'no',
+                                        message: 'No, I want to keep it'
+                                    }
+                                ]
+                            });
+
+                            if (confirmDeleteOldVolume === 'yes') {
+                                await execAsyncSpawn(`docker volume rm ${ mysqlVolumeName }`);
+
+                                const confirmDeleteDump = await subTask.prompt({
+                                    type: 'Select',
+                                    message: `Last question for today.
+Do you want to keep database dump created during conversion process? (${logger.style.file(pathToMySQLDumpFile)})`,
+                                    choices: [
+                                        {
+                                            name: 'yes',
+                                            message: 'I don\'t need it, thanks'
+                                        },
+                                        {
+                                            name: 'no',
+                                            message: 'I would like to keep it, thanks'
+                                        }
+                                    ]
+                                });
+
+                                if (confirmDeleteDump === 'yes') {
+                                    await fs.promises.rm(pathToMySQLDumpFile);
+                                }
+                            }
+                        }
+                    }
+                ]);
+            }
+            throw new KnownError('ABORTING');
+        }
+
+        task.skip();
+    },
+    options: {
+        bottomBar: 10
+    }
+});
+
+module.exports = convertMySQLDatabaseToMariaDB;

package/lib/tasks/docker/network/network-api.d.ts

@@ -15,7 +15,7 @@ export interface NetworkLsResult {
     IPv6: string
     Internal: string
     Labels: string
-    Name: StreamPipeOptions
+    Name: string
     Scope: string
 }
 

package/lib/tasks/docker/project-image-builder.js

@@ -101,7 +101,7 @@ const buildDockerFileInstructions = async (ctx, { image, tag }) => {
     }
 
     dockerFileInstructions
-        .workDir('/var/www/html');
+        .workDir(ctx.config.baseConfig.containerMagentoDir);
 
     return dockerFileInstructions;
 };

package/lib/tasks/docker/volume/index.js

@@ -0,0 +1,8 @@
+const volumeApi = require('./volume-api');
+const { createVolumes, removeVolumes } = require('./tasks');
+
+module.exports = {
+    createVolumes,
+    removeVolumes,
+    volumeApi
+};

package/lib/tasks/docker/{volumes.js → volume/tasks.js}

@@ -1,21 +1,8 @@
-const { execAsyncSpawn } = require('../../util/exec-async-command');
-
-const create = ({
-    driver,
-    opts = {},
-    name
-}) => {
-    let command = `docker volume create ${ Object.entries(opts).map(([name, value]) => `--opt ${name}='${value}'`).join(' ') } `;
-
-    if (driver) {
-        command += `--driver ${ driver }`;
-    }
-
-    return execAsyncSpawn(`${ command } ${ name }`);
-};
+const { execAsyncSpawn } = require('../../../util/exec-async-command');
+const { create } = require('./volume-api');
 
 /**
- * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ * @type {() => import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
  */
 const createVolumes = () => ({
     title: 'Creating volumes',
@@ -36,7 +23,7 @@ const createVolumes = () => ({
 });
 
 /**
- * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ * @type {() => import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
  */
 const removeVolumes = () => ({
     title: 'Removing volumes',
@@ -58,6 +45,5 @@ const removeVolumes = () => ({
 
 module.exports = {
     createVolumes,
-    removeVolumes,
-    createVolume: create
+    removeVolumes
 };

package/lib/tasks/docker/volume/volume-api.d.ts

@@ -0,0 +1,39 @@
+import { ExecAsyncSpawnOptions } from '../../../util/exec-async-command';
+
+export interface VolumeLsOptions<T extends boolean = false> {
+    filter?: string | string[]
+    format?: string
+    formatToJSON?: T
+    quiet?: boolean
+}
+
+export interface VolumeLsResult {
+    Driver: string
+    Labels: string
+    Links: string
+    MountPoint: string
+    Name: string
+    Scope: string
+    Size: string
+}
+
+export function ls(
+    options?: VolumeLsOptions,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<string>
+export function ls(
+    options?: VolumeLsOptions<true>,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<VolumeLsResult[]>
+
+export interface VolumeCreateOptions {
+    driver?: string
+    label?: string
+    name: string
+    opt?: Record<string, string>
+}
+
+export function create(
+    options?: VolumeCreateOptions,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<string>

package/lib/tasks/docker/volume/volume-api.js

@@ -0,0 +1,66 @@
+const { execAsyncSpawn } = require('../../../util/exec-async-command');
+
+/**
+ * @param {import('./volume-api').VolumeCreateOptions} options
+ * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions} execOptions
+ */
+const create = async (options, execOptions = {}) => {
+    const {
+        driver,
+        label,
+        opt,
+        name
+    } = options;
+
+    const driverArg = driver && `--driver=${driver}`;
+    const labelArg = label && `--label=${label}`;
+    const optArg = opt && Object.entries(opt).map(([name, value]) => `--opt ${name}='${value}'`).join(' ');
+
+    const args = [
+        driverArg,
+        labelArg,
+        optArg,
+        name
+    ].filter(Boolean).join(' ');
+
+    return execAsyncSpawn(`docker volume create ${args}`, execOptions);
+};
+
+/**
+ * @param {import('./volume-api').VolumeLsOptions} options
+ * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions} execOptions
+ */
+const ls = async (options, execOptions = {}) => {
+    const {
+        filter,
+        format,
+        formatToJSON = false,
+        quiet
+    } = options;
+
+    const formatArg = !formatToJSON && format
+        ? `--format=${format}`
+        : formatToJSON && '--format=\'{{json .}}\'';
+    const filterArg = filter && typeof filter === 'string'
+        ? `--filter=${filter}`
+        : filter && Array.isArray(filter) && filter.every((f) => typeof f === 'string') && filter.map((f) => `--filter=${f}`).join(' ');
+    const quietArg = quiet && '--quiet';
+
+    const args = [
+        filterArg,
+        formatArg,
+        quietArg
+    ].filter(Boolean).join(' ');
+
+    if (formatToJSON) {
+        const result = await execAsyncSpawn(`docker volume ls ${args}`, execOptions);
+        return JSON.parse(`[${result.split('\n').join(', ')}]`);
+    }
+
+    return execAsyncSpawn(`docker volume ls ${args}`, execOptions);
+};
+
+module.exports = {
+    create,
+    ls
+};

package/lib/tasks/execute/index.js

@@ -1,4 +1,4 @@
-// const path = require('path');
+const os = require('os');
 const { spawn } = require('child_process');
 
 const executeInContainer = ({ containerName, commands }) => {
@@ -7,11 +7,14 @@ const executeInContainer = ({ containerName, commands }) => {
         process.exit(1);
     }
 
+    const userArg = os.platform() === 'linux' && `--user=${os.userInfo().uid}:${os.userInfo().gid}`;
+
     spawn('docker', [
         'exec',
         '-it',
+        userArg,
         containerName
-    ].concat(...commands.map((command) => command.split(' '))), {
+    ].filter(Boolean).concat(...commands.map((command) => command.split(' '))), {
         stdio: [0, 1, 2]
     });
 

package/lib/tasks/file-system/create-mariadb-config.js

@@ -0,0 +1,23 @@
+const path = require('path');
+const UnknownError = require('../../errors/unknown-error');
+const setConfigFile = require('../../util/set-config');
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const createMariaDBConfig = () => ({
+    title: 'Setting MariaDB config',
+    task: async (ctx) => {
+        try {
+            await setConfigFile({
+                configPathname: path.join(ctx.config.baseConfig.cacheDir, 'mariadb.cnf'),
+                template: path.join(ctx.config.baseConfig.templateDir, 'mariadb.template.cnf'),
+                overwrite: true
+            });
+        } catch (e) {
+            throw new UnknownError(`Unexpected error accrued during php-fpm config creation\n\n${e}`);
+        }
+    }
+});
+
+module.exports = createMariaDBConfig;

package/lib/tasks/file-system/create-phpstorm-config/workspace-config/php-workspace-project-configuration-config.js

@@ -31,7 +31,7 @@ const setupPHPWorkspaceProjectConfiguration = async (workspaceConfigs, ctx) => {
         hasChanges = true;
         workspaceConfigs.push({
             [nameKey]: PHP_WORKSPACE_PROJECT_CONFIGURATION_COMPONENT_NAME,
-            [currentInterpreterImage]: currentInterpreterImage,
+            [interpreterNameKey]: currentInterpreterImage,
             include_path: []
         });
     }

package/lib/tasks/file-system/index.js

@@ -1,3 +1,4 @@
+const createMariaDBConfig = require('./create-mariadb-config');
 const createNginxConfig = require('./create-nginx-config');
 const createPhpConfig = require('./create-php-config');
 const createPhpFpmConfig = require('./create-php-fpm-config');
@@ -18,7 +19,8 @@ const prepareFileSystem = () => ({
         createPhpConfig(),
         createPhpStormConfig(),
         createVSCodeConfig(),
-        createVarnishConfig()
+        createVarnishConfig(),
+        createMariaDBConfig()
     ], {
         concurrent: true,
         exitOnError: false

package/lib/tasks/import-dump.js

@@ -1,11 +1,11 @@
 const path = require('path');
 const { checkRequirements } = require('./requirements');
 const {
-    importDumpToMySQL,
+    importDumpToDatabase,
     fixDB,
     dumpThemeConfig,
     restoreThemeConfig
-} = require('./mysql');
+} = require('./database');
 const { setupMagento } = require('./magento');
 const indexProducts = require('./magento/setup-magento/index-products');
 const {
@@ -14,7 +14,7 @@ const {
     retrieveFreshProjectConfiguration,
     configureProject
 } = require('./start');
-const importRemoteDb = require('./mysql/import-remote-db');
+const importRemoteDb = require('./database/import-remote-db');
 const matchFilesystem = require('../util/match-filesystem');
 
 /**
@@ -34,8 +34,8 @@ const importDump = () => ({
             // skip setup if env.php and config.php are present in app/etc folder and db is not empty
             skip: async (ctx) => {
                 const isFsMatching = await matchFilesystem(path.join(process.cwd(), 'app', 'etc'), ['config.php', 'env.php']);
-                const { mysqlConnection } = ctx;
-                const [[{ tableCount }]] = await mysqlConnection.query(`
+                const { databaseConnection } = ctx;
+                const [[{ tableCount }]] = await databaseConnection.query(`
                     SELECT count(*) AS tableCount
                     FROM INFORMATION_SCHEMA.TABLES
                     WHERE TABLE_SCHEMA = 'magento';
@@ -48,7 +48,7 @@ const importDump = () => ({
             )
         },
         dumpThemeConfig(),
-        importDumpToMySQL(),
+        importDumpToDatabase(),
         restoreThemeConfig(),
         fixDB(),
         setupMagento(),

package/lib/tasks/link.js

@@ -5,20 +5,22 @@ const retrieveThemeData = require('./theme/retrieve-theme-data');
 const linkTheme = require('./theme/link-theme');
 const { startServices } = require('./docker');
 const checkConfigurationFile = require('../config/check-configuration-file');
-const { connectToMySQL } = require('./mysql');
+const { connectToDatabase } = require('./database');
+const { checkRequirements } = require('./requirements');
 
 /**
  * @type {(theme: string) => import('listr2').ListrTask<import('../../typings/context').ListrContext>}
  */
 const linkTask = (themePath) => ({
     task: (ctx, task) => task.newListr([
+        checkRequirements(),
         getMagentoVersionConfig(),
         checkConfigurationFile(),
         getProjectConfiguration(),
         getCachedPorts(),
         startServices(),
         // startPhpFpm(),
-        connectToMySQL(),
+        connectToDatabase(),
         retrieveThemeData(themePath),
         linkTheme()
     ])

package/lib/tasks/magento/install-magento-project.js

@@ -98,7 +98,7 @@ const createMagentoProject = async (ctx, task, {
     ];
 
     await runPHPContainerCommand(ctx, `composer ${installCommand.join(' ')} \n
-&& mv ${tempDir}/composer.json /var/www/html/composer.json`);
+&& mv ${tempDir}/composer.json ${ctx.config.baseConfig.containerMagentoDir}/composer.json`);
 };
 
 /**

package/lib/tasks/magento/setup-magento/configure-elasticsearch.js

@@ -7,7 +7,7 @@ module.exports = () => ({
     title: 'Configuring Elasticsearch',
     skip: async (ctx) => !(await isTableExists('magento', 'core_config_data', ctx)),
     task: async (ctx, task) => {
-        const { ports, mysqlConnection } = ctx;
+        const { ports, databaseConnection } = ctx;
         const isLinux = ctx.platform === 'linux';
         const isNativeLinux = isLinux && !ctx.isWsl;
         const hostMachine = isNativeLinux ? '127.0.0.1' : 'host.docker.internal';
@@ -15,6 +15,6 @@ module.exports = () => ({
             { path: 'catalog/search/engine', value: 'elasticsearch7' },
             { path: 'catalog/search/elasticsearch7_server_hostname', value: hostMachine },
             { path: 'catalog/search/elasticsearch7_server_port', value: `${ports.elasticsearch}` }
-        ], { mysqlConnection, task });
+        ], { databaseConnection, task });
     }
 });

package/lib/tasks/magento/setup-magento/create-admin.js

@@ -7,7 +7,7 @@ module.exports = () => ({
     title: 'Creating admin user',
     task: async (ctx, task) => {
         const { config: { magentoConfiguration } } = ctx;
-        const [[{ userCount }]] = await ctx.mysqlConnection.query(`
+        const [[{ userCount }]] = await ctx.databaseConnection.query(`
             SELECT count(*) AS userCount
             FROM admin_user
             WHERE username = ?;

package/lib/tasks/magento/setup-magento/delete-admin-users.js

@@ -7,8 +7,8 @@ const deleteAdminUsers = () => ({
     title: 'Deleting old admin users',
     skip: async (ctx) => !(await isTableExists('magento', 'admin_user', ctx)),
     task: async (ctx) => {
-        const { mysqlConnection } = ctx;
-        await mysqlConnection.query(`
+        const { databaseConnection } = ctx;
+        await databaseConnection.query(`
             TRUNCATE TABLE admin_user;
         `);
     }