@scandipwa/magento-scripts 2.0.0-alpha.0 → 2.0.0-alpha.11

package/lib/tasks/docker/containers/container-api.js

@@ -3,9 +3,9 @@ const { execAsyncSpawn } = require('../../../util/exec-async-command');
 
 /**
  * @param {import('./container-api').ContainerRunOptions} options
- * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions<false>} execOptions
+ * @returns {string[]}
  */
-const run = (options, execOptions = {}) => {
+const runCommand = (options) => {
     const {
         addHost,
         ports = [],
@@ -24,32 +24,35 @@ const run = (options, execOptions = {}) => {
         expose = [],
         detach = true,
         rm = false,
+        tty = false,
         user
     } = options;
 
     const detachArg = detach && '-d';
     const rmArg = rm && '--rm';
+    const ttyArg = tty && '-it';
     const exposeArg = expose && expose.map((e) => `--expose=${ e }`);
-    const restartArg = !rm && restart && `--restart ${ restart }`;
-    const networkArg = network && `--network ${ network }`;
-    const portsArgs = ports.map((port) => `-p ${ port }`).join(' ');
-    const mountsArgs = mounts.map((mount) => `--mount ${ mount }`).join(' ');
-    const mountVolumesArgs = mountVolumes.map((mount) => `-v ${mount}`).join(' ');
-    const envArgs = !env ? '' : Object.entries(env).map(([key, value]) => `--env ${ key }='${ value }'`).join(' ');
-    const nameArg = name && `--name ${name}`;
-    const entrypointArg = entrypoint && `--entrypoint "${entrypoint}"`;
-    const healthCheckArg = healthCheck && Object.entries(healthCheck).map(([key, value]) => `--health-${key} '${value}'`).join(' ');
-    const securityArg = securityOptions.length > 0 && securityOptions.map((opt) => `--security-opt ${opt}`).join(' ');
-    const tmpfsArg = tmpfs.length > 0 && tmpfs.map((t) => `--tmpfs ${t}`).join(' ');
+    const restartArg = !rm && restart && `--restart=${ restart }`;
+    const networkArg = network && `--network=${ network }`;
+    const portsArgs = ports && ports.length > 0 && ports.map((port) => `-p=${ port }`).join(' ');
+    const mountsArgs = mounts && mounts.map((mount) => `--mount=${ mount }`).join(' ');
+    const mountVolumesArgs = mountVolumes && mountVolumes.map((mount) => `-v=${mount}`).join(' ');
+    const envArgs = !env ? '' : Object.entries(env).map(([key, value]) => `--env=${ key }='${ value }'`).join(' ');
+    const nameArg = name && `--name=${name}`;
+    const entrypointArg = entrypoint && `--entrypoint="${entrypoint}"`;
+    const healthCheckArg = healthCheck && Object.entries(healthCheck).map(([key, value]) => `--health-${key}='${value}'`).join(' ');
+    const securityArg = securityOptions.length > 0 && securityOptions.map((opt) => `--security-opt=${opt}`).join(' ');
+    const tmpfsArg = tmpfs.length > 0 && tmpfs.map((t) => `--tmpfs=${t}`).join(' ');
     const userArg = user && `--user=${user}`;
     const addHostArg = addHost && `--add-host=${addHost}`;
 
     const dockerCommand = [
         'docker',
         'run',
+        nameArg,
+        ttyArg,
         detachArg,
         rmArg,
-        nameArg,
         networkArg,
         restartArg,
         portsArgs,
@@ -65,11 +68,17 @@ const run = (options, execOptions = {}) => {
         addHostArg,
         image,
         command
-    ].filter(Boolean).join(' ');
+    ].filter(Boolean).filter((arg) => typeof arg === 'string');
 
-    return execAsyncSpawn(dockerCommand, execOptions);
+    return dockerCommand;
 };
 
+/**
+ * @param {import('./container-api').ContainerRunOptions} options
+ * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions<false>} execOptions
+ */
+const run = (options, execOptions = {}) => execAsyncSpawn(runCommand(options).join(' '), execOptions);
+
 /**
  * @param {string} command
  * @param {string} container container id or name
@@ -140,14 +149,64 @@ const ls = async (options = {}, execOptions = {}) => {
 
     if (formatToJSON) {
         const result = await execAsyncSpawn(`docker container ls ${args}`, execOptions);
+        if (result.startsWith('[')) {
+            return JSON.parse(result);
+        }
+
         return JSON.parse(`[${result.split('\n').join(', ')}]`);
     }
 
     return execAsyncSpawn(`docker container ls ${args}`, execOptions);
 };
 
+/**
+ * @param {import('./container-api').ContainerLogsOptions} options
+ * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions<false>} execOptions
+ */
+const logs = async (options = {}, execOptions = {}) => {
+    const {
+        name,
+        details = false,
+        follow = false,
+        since = '',
+        tail = '',
+        timestamps = false,
+        until = '',
+        parser
+    } = options;
+    const detailsArg = details && '--details';
+    const followArg = follow && '--follow';
+    const sinceArg = since && `--since=${since}`;
+    const tailArg = tail && `--tail=${tail}`;
+    const timestampsArg = timestamps && '--timestamps';
+    const untilArg = until && `--until=${until}`;
+
+    const logsCommand = [
+        'docker',
+        'container',
+        'logs',
+        detailsArg,
+        followArg,
+        sinceArg,
+        tailArg,
+        timestampsArg,
+        untilArg,
+        name
+    ].filter(Boolean).join(' ');
+
+    if (parser) {
+        const result = await execAsyncSpawn(logsCommand, execOptions);
+
+        return result.split('\n').map((line) => parser(line));
+    }
+
+    return execAsyncSpawn(logsCommand, execOptions);
+};
+
 module.exports = {
     run,
+    runCommand,
     exec,
-    ls
+    ls,
+    logs
 };

package/lib/tasks/docker/containers/tasks.js

@@ -26,8 +26,39 @@ const remoteImageReducer = (acc, val) => {
  */
 const pullImages = () => ({
     title: 'Pulling container images',
-    task: async ({ config: { docker } }, task) => {
+    task: async ({ config: { docker }, pullImages }, task) => {
         const containers = Object.values(docker.getContainers());
+
+        if (pullImages) {
+            return task.newListr(
+                containers
+                    .reduce(remoteImageReducer, [])
+                    .map((image) => {
+                        const [repo, tag = 'latest'] = image.split(':');
+
+                        return { repo, tag };
+                    })
+                    .reduce(
+                        (acc, val) => acc.concat(
+                            acc.some(
+                                (c) => c.repo === val.repo
+                                && c.tag === val.tag
+                            )
+                                ? []
+                                : val
+                        ),
+                        []
+                    )
+                    .map(({ repo, tag }) => ({
+                        title: `Pulling ${ logger.style.file(`${repo}:${tag}`) } image`,
+                        task: () => pull(`${repo}:${tag}`)
+                    })), {
+                    concurrent: true,
+                    exitOnError: true
+                }
+            );
+        }
+
         const imagesFilter = containers
             .reduce(remoteImageReducer, [])
             .map((image) => `reference='${image}'`);

package/lib/tasks/docker/convert-legacy-volumes.js

@@ -1,10 +1,11 @@
+/* eslint-disable no-param-reassign */
 /* eslint-disable max-len */
 const { stopServices } = require('./index');
 const { getBaseConfig, getConfigFromMagentoVersion } = require('../../config');
 const getDockerConfig = require('../../config/docker');
 const { execAsyncSpawn } = require('../../util/exec-async-command');
 const { folderName, legacyFolderName } = require('../../util/prefix');
-const { createVolume } = require('./volumes');
+const { volumeApi } = require('./volume');
 
 /**
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
@@ -14,20 +15,20 @@ const convertLegacyVolumes = () => ({
         const { config: { overridenConfiguration } } = ctx;
         const newDockerConfig = await getDockerConfig(ctx, overridenConfiguration, getBaseConfig(process.cwd(), folderName));
 
-        const newVolumeNames = Object.values(newDockerConfig.volumes).filter((v) => !v.opts).map(({ name }) => name);
+        const newVolumeNames = Object.values(newDockerConfig.volumes).filter((v) => !v.opt).map(({ name }) => name);
 
-        const existingVolumes = (await execAsyncSpawn('docker volume ls -q')).split('\n');
+        const existingVolumes = await volumeApi.ls({ formatToJSON: true });
 
         if (newVolumeNames.every((v) => existingVolumes.includes(v))) {
             return;
         }
 
         const legacyDockerConfig = await getDockerConfig(ctx, overridenConfiguration, getBaseConfig(process.cwd(), legacyFolderName));
-        const legacyVolumeNames = Object.values(legacyDockerConfig.volumes).filter((v) => !v.opts).map(({ name }) => name);
+        const legacyVolumeNames = Object.values(legacyDockerConfig.volumes).filter((v) => !v.opt).map(({ name }) => name);
 
         if (
-            newVolumeNames.every((name) => !existingVolumes.includes(name))
-            && legacyVolumeNames.every((name) => existingVolumes.includes(name))
+            newVolumeNames.every((name) => !existingVolumes.some((v) => v.Name === name))
+            && legacyVolumeNames.every((name) => existingVolumes.some((v) => v.Name === name))
         ) {
             ctx.config = await getConfigFromMagentoVersion(ctx, {
                 magentoVersion: ctx.magentoVersion,
@@ -46,7 +47,7 @@ const convertLegacyVolumes = () => ({
                             const legacyVolumeConfig = legacyDockerConfig.volumes[volumeName];
 
                             subTask.output = `Creating volume ${volumeConfig.name}...`;
-                            await createVolume(volumeConfig);
+                            await volumeApi.create(volumeConfig);
                             subTask.output = `Copying data from ${legacyVolumeConfig.name} to ${volumeConfig.name}...`;
                             await execAsyncSpawn(
                                 `docker run --rm -v ${legacyVolumeConfig.name}:/from:ro -v ${volumeConfig.name}:/to alpine ash -c "cd /from; cp -av . /to"`, {

package/lib/tasks/docker/convert-mysql-to-mariadb.js

@@ -0,0 +1,284 @@
+const path = require('path');
+const fs = require('fs');
+const logger = require('@scandipwa/scandipwa-dev-utils/logger');
+const { volumeApi, createVolumes } = require('./volume');
+const { execAsyncSpawn } = require('../../util/exec-async-command');
+const { containerApi, stopContainers, pullImages } = require('./containers');
+const { importDumpToDatabase, connectToDatabase } = require('../database');
+const { setPrefix } = require('../prefix');
+const getProjectConfiguration = require('../../config/get-project-configuration');
+const { getAvailablePorts, getCachedPorts } = require('../../config/get-port-config');
+const { saveConfiguration } = require('../../config/save-config');
+const { buildProjectImage, buildDebugProjectImage } = require('./project-image-builder');
+const checkPHPVersion = require('../requirements/php-version');
+const { getComposerVersionTask } = require('../composer');
+const { prepareFileSystem } = require('../file-system');
+const { installMagentoProject } = require('../magento');
+const enableMagentoComposerPlugins = require('../magento/enable-magento-composer-plugins');
+const { startServices } = require('./index');
+const dockerNetwork = require('./network');
+const KnownError = require('../../errors/known-error');
+const { createCacheFolder } = require('../cache');
+const { getSystemConfigTask } = require('../../config/system-config');
+const sleep = require('../../util/sleep');
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const convertMySQLDatabaseToMariaDB = () => ({
+    task: async (ctx, task) => {
+        const mysqlVolumeName = `${ ctx.config.baseConfig.prefix }_mysql-data`;
+        const volumes = await volumeApi.ls({ formatToJSON: true });
+        if (ctx.isArmMac) {
+            if (volumes.some(
+                (volume) => volume.Name === mysqlVolumeName
+            ) && !volumes.some(
+                (volume) => volume.Name === ctx.config.docker.volumes.mariadb.name
+            )
+            ) {
+                task.title = 'Converting MySQL database to MariaDB';
+                return task.newListr([
+                    stopContainers(),
+                    {
+                        title: 'Converting MySQL volume to MariaDB',
+                        task: async (subCtx, subTask) => {
+                            subTask.output = `Creating volume ${subCtx.config.docker.volumes.mariadb.name}...`;
+                            await volumeApi.create(subCtx.config.docker.volumes.mariadb);
+                            subTask.output = `Copying data from ${mysqlVolumeName} to ${subCtx.config.docker.volumes.mariadb.name}...`;
+                            await execAsyncSpawn(
+                            // eslint-disable-next-line max-len
+                                `docker run --rm -v ${mysqlVolumeName}:/from:ro -v ${subCtx.config.docker.volumes.mariadb.name}:/to alpine ash -c "cd /from; cp -av . /to"`, {
+                                    callback: (t) => {
+                                        subTask.output = t;
+                                    }
+                                }
+                            );
+                            subTask.output = 'Deleting old volume...';
+                            await execAsyncSpawn(`docker volume rm ${mysqlVolumeName}`);
+                        },
+                        options: {
+                            bottomBar: 10
+                        }
+                    }
+                ]);
+            }
+
+            return;
+        }
+
+        if (volumes.some(
+            (volume) => volume.Name === mysqlVolumeName
+        ) && !volumes.some(
+            (volume) => volume.Name === ctx.config.docker.volumes.mariadb.name
+        )
+        ) {
+            task.title = 'Converting MySQL database to MariaDB';
+            const confirmConvert = await task.prompt({
+                type: 'Select',
+                message: `We see that you have old mysql-data volume in the project!
+Since ${logger.style.code('magento-scripts@2.0.0')} ${logger.style.misc('MariaDB')} is used as database instead of MySQL.
+
+To use MariaDB with data from MySQL we need to convert your database.
+`,
+                choices: [
+                    {
+                        name: 'yes',
+                        message: 'Okay, let\'s do that!'
+                    },
+                    {
+                        name: 'no',
+                        message: 'ABORT ABORT ABORT'
+                    }
+                ]
+            });
+
+            if (confirmConvert === 'yes') {
+                const pathToMySQLDumpFile = path.join(process.cwd(), 'mysql-database.sql');
+                const containerName = 'mysql-database';
+                task.output = 'Creating dump file from MySQL database...';
+
+                const existingContainers = await containerApi.ls({ formatToJSON: true, all: true });
+
+                if (existingContainers.some((c) => c.Names === containerName)) {
+                    await execAsyncSpawn(`docker container stop ${containerName}`);
+                    await execAsyncSpawn(`docker container rm ${containerName}`);
+                }
+
+                task.output = 'Starting MySQL server...';
+
+                await containerApi.run(
+                    {
+                        mounts: [
+                            `source=${ mysqlVolumeName },target=/var/lib/mysql`
+                        ],
+                        env: {
+                            MYSQL_PORT: 3306,
+                            MYSQL_ROOT_PASSWORD: 'scandipwa',
+                            MYSQL_USER: 'magento',
+                            MYSQL_PASSWORD: 'magento',
+                            MYSQL_DATABASE: 'magento'
+                        },
+                        command: [
+                            '--log_bin_trust_function_creators=1',
+                            '--default-authentication-plugin=mysql_native_password',
+                            '--max_allowed_packet=1GB',
+                            '--bind-address=0.0.0.0'
+                        ]
+                            .join(' '),
+                        securityOptions: [
+                            'seccomp=unconfined'
+                        ],
+                        name: containerName,
+                        image: `mysql:${ ctx.config.overridenConfiguration.configuration.mysql.version }`
+                    }
+                );
+
+                task.output = 'Waiting for MySQL to initialize...';
+
+                let mysqlReadyForConnections = false;
+
+                while (!mysqlReadyForConnections) {
+                    const mysqlOutput = await execAsyncSpawn(`docker logs ${containerName}`);
+                    if (mysqlOutput.includes('ready for connections')) {
+                        mysqlReadyForConnections = true;
+                        task.output = 'MySQL is ready!';
+                        break;
+                    } else if (mysqlOutput.includes('Initializing database files')) {
+                        task.output = `MySQL is initializing database files!
+Please wait, this will take some time and do not restart the MySQL container until initialization is finished!`;
+
+                        let mysqlFinishedInitialization = false;
+                        while (!mysqlFinishedInitialization) {
+                            const mysqlOutput = await execAsyncSpawn(`docker logs ${containerName}`);
+                            if (mysqlOutput.includes('init process done.') && !mysqlFinishedInitialization) {
+                                mysqlFinishedInitialization = true;
+                                task.output = 'MySQL is initialized!';
+                                break;
+                            }
+                            await sleep(2000);
+                        }
+                    }
+
+                    await sleep(2000);
+                }
+
+                task.output = 'Dumping MySQL database to dump file...';
+
+                await containerApi.exec(
+                    [
+                        'mysqldump',
+                        '--user=root',
+                        '--password=scandipwa',
+                        'magento',
+                        `--result-file=${ path.parse(pathToMySQLDumpFile).base }`
+                    ].join(' '),
+                    containerName,
+                    {},
+                    {
+                        callback: (t) => {
+                            task.output = t;
+                        }
+                    }
+                );
+
+                task.output = 'Copying dump file from container to system...';
+
+                await execAsyncSpawn(
+                    `docker cp ${containerName}:/${ path.parse(pathToMySQLDumpFile).base } ${ pathToMySQLDumpFile }`
+                );
+
+                task.output = 'Removing migration container...';
+
+                await execAsyncSpawn(
+                    `docker container stop ${containerName}`
+                );
+                await execAsyncSpawn(
+                    `docker container rm ${containerName}`
+                );
+
+                ctx.importDb = pathToMySQLDumpFile;
+
+                return task.newListr([
+                    createCacheFolder(),
+                    getSystemConfigTask(),
+                    getCachedPorts(),
+                    stopContainers(),
+                    setPrefix(),
+                    getProjectConfiguration(),
+                    // get fresh ports
+                    getAvailablePorts(),
+                    saveConfiguration(),
+                    pullImages(),
+                    dockerNetwork.tasks.createNetwork(),
+                    createVolumes(),
+                    {
+                        task: (ctx, task) => task.newListr([
+                            buildProjectImage(),
+                            buildDebugProjectImage()
+                        ], {
+                            concurrent: true
+                        })
+                    },
+                    checkPHPVersion(),
+                    getComposerVersionTask(),
+                    prepareFileSystem(),
+                    installMagentoProject(),
+                    enableMagentoComposerPlugins(),
+                    startServices(),
+                    connectToDatabase(),
+                    importDumpToDatabase(),
+                    {
+                        task: async (subCtx, subTask) => {
+                            const confirmDeleteOldVolume = await subTask.prompt({
+                                type: 'Select',
+                                message: 'Okay, looks like conversion went well, do you want to delete old mysql-data volume?',
+                                choices: [
+                                    {
+                                        name: 'yes',
+                                        message: 'Sure'
+                                    },
+                                    {
+                                        name: 'no',
+                                        message: 'No, I want to keep it'
+                                    }
+                                ]
+                            });
+
+                            if (confirmDeleteOldVolume === 'yes') {
+                                await execAsyncSpawn(`docker volume rm ${ mysqlVolumeName }`);
+
+                                const confirmDeleteDump = await subTask.prompt({
+                                    type: 'Select',
+                                    message: `Last question for today.
+Do you want to keep database dump created during conversion process? (${logger.style.file(pathToMySQLDumpFile)})`,
+                                    choices: [
+                                        {
+                                            name: 'yes',
+                                            message: 'I don\'t need it, thanks'
+                                        },
+                                        {
+                                            name: 'no',
+                                            message: 'I would like to keep it, thanks'
+                                        }
+                                    ]
+                                });
+
+                                if (confirmDeleteDump === 'yes') {
+                                    await fs.promises.rm(pathToMySQLDumpFile);
+                                }
+                            }
+                        }
+                    }
+                ]);
+            }
+            throw new KnownError('ABORTING');
+        }
+
+        task.skip();
+    },
+    options: {
+        bottomBar: 10
+    }
+});
+
+module.exports = convertMySQLDatabaseToMariaDB;

package/lib/tasks/docker/index.js

@@ -1,5 +1,7 @@
 const containers = require('./containers');
 const network = require('./network');
+const volume = require('./volume');
+const dockerApi = require('./api');
 
 /**
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
@@ -19,13 +21,16 @@ const startServices = () => ({
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
  */
 const stopServices = () => ({
+    title: 'Stopping Docker services',
     task: (ctx, task) => task.newListr([
         containers.stopContainers(),
+        volume.removeLocalVolumes(),
         network.tasks.removeNetwork()
     ])
 });
 
 module.exports = {
     startServices,
-    stopServices
+    stopServices,
+    dockerApi
 };

package/lib/tasks/docker/network/network-api.d.ts

@@ -15,7 +15,7 @@ export interface NetworkLsResult {
     IPv6: string
     Internal: string
     Labels: string
-    Name: StreamPipeOptions
+    Name: string
     Scope: string
 }
 

package/lib/tasks/docker/project-image-builder.js

@@ -27,13 +27,13 @@ const getEnabledExtensionsFromImage = async (imageWithTag) => {
         .reduce((acc, [name, version]) => ({ ...acc, [name]: version }), {});
 };
 
-const addExtensionToBuilder = (builder, ctx) => ([extensionName, extensionInstructions]) => {
+const addExtensionToBuilder = (builder, ctx) => async ([extensionName, extensionInstructions]) => {
     const { command, ...extensionInstructionsWithoutCommand } = extensionInstructions;
     let runCommand = '';
     if (typeof command === 'string') {
         runCommand += ` ${command}`;
-    } else if (typeof command === 'function') {
-        runCommand += ` ${command({ ...extensionInstructionsWithoutCommand, ctx })}`;
+    } else if (typeof command === 'function' || command instanceof Promise) {
+        runCommand += ` ${await Promise.resolve(command({ ...extensionInstructionsWithoutCommand, ctx }))}`;
     } else {
         runCommand += ` docker-php-ext-install ${extensionInstructionsWithoutCommand.name}`;
     }
@@ -68,10 +68,6 @@ const buildDockerFileInstructions = async (ctx, { image, tag }) => {
         .comment('project image')
         .from({ image, tag });
 
-    // install bash in image
-    dockerFileInstructions
-        .run('apk add --no-cache bash');
-
     if (missingExtensions.length > 0) {
         const allDependencies = missingExtensions.map(
             ([_extensionName, extensionInstructions]) => (extensionInstructions.dependencies || [])
@@ -79,7 +75,9 @@ const buildDockerFileInstructions = async (ctx, { image, tag }) => {
             .reduce((acc, val) => acc.concat(val.filter((ex) => !acc.includes(ex))), []);
 
         dockerFileInstructions.run(`apk add --no-cache ${allDependencies.join(' ')}`);
-        missingExtensions.forEach(addExtensionToBuilder(dockerFileInstructions, ctx));
+        for (const missingExtensionInstructions of missingExtensions) {
+            await addExtensionToBuilder(dockerFileInstructions, ctx)(missingExtensionInstructions);
+        }
     }
 
     const composerVersion = /^\d$/.test(composer.version)
@@ -101,7 +99,7 @@ const buildDockerFileInstructions = async (ctx, { image, tag }) => {
     }
 
     dockerFileInstructions
-        .workDir('/var/www/html');
+        .workDir(ctx.config.baseConfig.containerMagentoDir);
 
     return dockerFileInstructions;
 };

package/lib/tasks/docker/volume/index.js

@@ -0,0 +1,9 @@
+const volumeApi = require('./volume-api');
+const { createVolumes, removeVolumes, removeLocalVolumes } = require('./tasks');
+
+module.exports = {
+    createVolumes,
+    removeVolumes,
+    removeLocalVolumes,
+    volumeApi
+};