@scandipwa/magento-scripts 1.16.0 → 2.0.0-alpha.1

package/lib/tasks/docker/containers/index.js

@@ -0,0 +1,19 @@
+const {
+    startContainers,
+    stopContainers,
+    pullImages,
+    statusContainers,
+    checkContainersAreRunning,
+    getContainerStatus
+} = require('./tasks');
+const containerApi = require('./container-api');
+
+module.exports = {
+    startContainers,
+    stopContainers,
+    pullImages,
+    statusContainers,
+    checkContainersAreRunning,
+    getContainerStatus,
+    containerApi
+};

package/lib/tasks/docker/containers/tasks.js

@@ -0,0 +1,215 @@
+/* eslint-disable max-len */
+const sleep = require('../../../util/sleep');
+const logger = require('@scandipwa/scandipwa-dev-utils/logger');
+const KnownError = require('../../../errors/known-error');
+const containerApi = require('./container-api');
+const { imageApi } = require('../image');
+const { execAsyncSpawn } = require('../../../util/exec-async-command');
+
+const stop = async (containers) => {
+    await execAsyncSpawn(`docker container stop ${containers.join(' ')}`);
+    await execAsyncSpawn(`docker container rm ${containers.join(' ')}`);
+};
+
+const pull = async (image) => execAsyncSpawn(`docker pull ${image}`);
+
+const remoteImageReducer = (acc, val) => {
+    if (Array.isArray(val.remoteImages) && val.remoteImages.every((image) => typeof image === 'string')) {
+        return acc.concat(val.remoteImages);
+    }
+
+    return acc.concat([val.image]);
+};
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
+ */
+const pullImages = () => ({
+    title: 'Pulling container images',
+    task: async ({ config: { docker } }, task) => {
+        const containers = Object.values(docker.getContainers());
+        const imagesFilter = containers
+            .reduce(remoteImageReducer, [])
+            .map((image) => `reference='${image}'`);
+
+        const existingImages = await imageApi.ls({
+            formatToJSON: true,
+            filter: imagesFilter
+        });
+
+        const missingContainerImages = containers
+            .reduce(remoteImageReducer, [])
+            .map((image) => {
+                const [repo, tag = 'latest'] = image.split(':');
+
+                return { repo, tag };
+            })
+            .filter(
+                ({ repo, tag }) => !existingImages.some(
+                    (image) => image.Repository === repo
+                    && image.Tag === tag
+                )
+            )
+            .reduce(
+                (acc, val) => acc.concat(
+                    acc.some(
+                        (c) => c.repo === val.repo
+                        && c.tag === val.tag
+                    )
+                        ? []
+                        : val
+                ),
+                []
+            );
+
+        if (missingContainerImages.length === 0) {
+            task.skip();
+            return;
+        }
+
+        return task.newListr(
+            missingContainerImages.map(({ repo, tag }) => ({
+                title: `Pulling ${ logger.style.file(`${repo}:${tag}`) } image`,
+                task: () => pull(`${repo}:${tag}`)
+            })), {
+                concurrent: true,
+                exitOnError: true
+            }
+        );
+    }
+});
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const startContainers = () => ({
+    title: 'Starting containers',
+    task: async ({ ports, config: { docker }, debug }, task) => {
+        const containerList = (await execAsyncSpawn('docker container ls --all --format="{{.Names}}"')).split('\n');
+
+        const missingContainers = Object.values(docker.getContainers(ports)).filter(
+            ({ name }) => !containerList.includes(name)
+        );
+
+        if (missingContainers.length === 0) {
+            task.skip();
+            return;
+        }
+
+        if (debug) {
+            await Promise.all(
+                missingContainers
+                    .map((container) => {
+                        if (container.debugImage) {
+                            container.image = container.debugImage;
+                        }
+
+                        return container;
+                    }).map((container) => containerApi.run(container).then((out) => {
+                        task.output = `From ${container._}: ${out}`;
+                    }))
+            );
+
+            return;
+        }
+
+        // TODO: we might stop containers here ?
+        await Promise.all(missingContainers.map((container) => containerApi.run(container).then((out) => {
+            task.output = `From ${container._}: ${out}`;
+        })));
+    },
+    options: {
+        bottomBar: 10
+    }
+});
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const stopContainers = () => ({
+    title: 'Stopping Docker containers',
+    task: async ({ config: { baseConfig: { prefix } } }, task) => {
+        const containerList = (await execAsyncSpawn('docker container ls --all --format="{{.Names}}"')).split('\n');
+
+        const runningContainers = containerList.filter((containerName) => containerName.startsWith(prefix));
+
+        if (runningContainers.length === 0) {
+            task.skip();
+            return;
+        }
+
+        await stop(runningContainers);
+    }
+});
+
+const getContainerStatus = async (containerName) => {
+    try {
+        return JSON.parse(await execAsyncSpawn(`docker inspect --format='{{json .State}}' ${containerName}`));
+    } catch {
+        return null;
+    }
+};
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const checkContainersAreRunning = () => ({
+    title: 'Checking container statuses',
+    task: async (ctx, task) => {
+        const { config: { docker }, ports } = ctx;
+        const containers = Object.values(docker.getContainers(ports));
+        let tries = 0;
+        while (tries < 3) {
+            const containersWithStatus = await Promise.all(
+                containers.map(async (container) => ({
+                    ...container,
+                    status: await getContainerStatus(container.name)
+                }))
+            );
+
+            if (containersWithStatus.some((c) => c.status.Status !== 'running')) {
+                if (tries === 2) {
+                    throw new KnownError(`${containersWithStatus.filter((c) => c.status.Status !== 'running').map((c) => c._).join(', ')} containers are not running! Please check container logs for more details!`);
+                } else {
+                    task.output = `${containersWithStatus.filter((c) => c.status.Status !== 'running').map((c) => c._).join(', ')} are not running, waiting if something will change...`;
+                    await sleep(2000);
+                    tries++;
+                }
+            } else {
+                break;
+            }
+        }
+    },
+    options: {
+        bottomBar: 10
+    }
+});
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const statusContainers = () => ({
+    task: async (ctx) => {
+        const { config: { docker }, ports } = ctx;
+        const containers = Object.values(docker.getContainers(ports));
+
+        ctx.containers = await Promise.all(
+            containers.map(async (container) => ({
+                ...container,
+                status: await getContainerStatus(container.name)
+            }))
+        );
+    },
+    options: {
+        bottomBar: 10
+    }
+});
+
+module.exports = {
+    startContainers,
+    stopContainers,
+    pullImages,
+    statusContainers,
+    checkContainersAreRunning,
+    getContainerStatus
+};

package/lib/tasks/docker/convert-legacy-volumes.js

@@ -1,10 +1,11 @@
+/* eslint-disable no-param-reassign */
 /* eslint-disable max-len */
 const { stopServices } = require('./index');
 const { getBaseConfig, getConfigFromMagentoVersion } = require('../../config');
 const getDockerConfig = require('../../config/docker');
 const { execAsyncSpawn } = require('../../util/exec-async-command');
 const { folderName, legacyFolderName } = require('../../util/prefix');
-const { createVolume } = require('./volumes');
+const { volumeApi } = require('./volume');
 
 /**
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
@@ -12,24 +13,28 @@ const { createVolume } = require('./volumes');
 const convertLegacyVolumes = () => ({
     task: async (ctx, task) => {
         const { config: { overridenConfiguration } } = ctx;
-        const newDockerConfig = await getDockerConfig(overridenConfiguration, getBaseConfig(process.cwd(), folderName));
+        const newDockerConfig = await getDockerConfig(ctx, overridenConfiguration, getBaseConfig(process.cwd(), folderName));
 
-        const newVolumeNames = Object.values(newDockerConfig.volumes).filter((v) => !v.opts).map(({ name }) => name);
+        const newVolumeNames = Object.values(newDockerConfig.volumes).filter((v) => !v.opt).map(({ name }) => name);
 
-        const existingVolumes = (await execAsyncSpawn('docker volume ls -q')).split('\n');
+        const existingVolumes = await volumeApi.ls({ formatToJSON: true });
 
         if (newVolumeNames.every((v) => existingVolumes.includes(v))) {
             return;
         }
 
-        const legacyDockerConfig = await getDockerConfig(overridenConfiguration, getBaseConfig(process.cwd(), legacyFolderName));
-        const legacyVolumeNames = Object.values(legacyDockerConfig.volumes).filter((v) => !v.opts).map(({ name }) => name);
+        const legacyDockerConfig = await getDockerConfig(ctx, overridenConfiguration, getBaseConfig(process.cwd(), legacyFolderName));
+        const legacyVolumeNames = Object.values(legacyDockerConfig.volumes).filter((v) => !v.opt).map(({ name }) => name);
 
         if (
-            newVolumeNames.every((name) => !existingVolumes.includes(name))
-            && legacyVolumeNames.every((name) => existingVolumes.includes(name))
+            newVolumeNames.every((name) => !existingVolumes.some((v) => v.Name === name))
+            && legacyVolumeNames.every((name) => existingVolumes.some((v) => v.Name === name))
         ) {
-            ctx.config = await getConfigFromMagentoVersion(ctx.magentoVersion, process.cwd(), legacyFolderName);
+            ctx.config = await getConfigFromMagentoVersion(ctx, {
+                magentoVersion: ctx.magentoVersion,
+                projectPath: process.cwd(),
+                prefix: legacyFolderName
+            });
 
             return task.newListr([
                 stopServices(),
@@ -42,7 +47,7 @@ const convertLegacyVolumes = () => ({
                             const legacyVolumeConfig = legacyDockerConfig.volumes[volumeName];
 
                             subTask.output = `Creating volume ${volumeConfig.name}...`;
-                            await createVolume(volumeConfig);
+                            await volumeApi.create(volumeConfig);
                             subTask.output = `Copying data from ${legacyVolumeConfig.name} to ${volumeConfig.name}...`;
                             await execAsyncSpawn(
                                 `docker run --rm -v ${legacyVolumeConfig.name}:/from:ro -v ${volumeConfig.name}:/to alpine ash -c "cd /from; cp -av . /to"`, {

package/lib/tasks/docker/convert-mysql-to-mariadb.js

@@ -0,0 +1,284 @@
+const path = require('path');
+const fs = require('fs');
+const logger = require('@scandipwa/scandipwa-dev-utils/logger');
+const { volumeApi, createVolumes } = require('./volume');
+const { execAsyncSpawn } = require('../../util/exec-async-command');
+const { containerApi, stopContainers, pullImages } = require('./containers');
+const { importDumpToDatabase, connectToDatabase } = require('../database');
+const { setPrefix } = require('../prefix');
+const getProjectConfiguration = require('../../config/get-project-configuration');
+const { getAvailablePorts, getCachedPorts } = require('../../config/get-port-config');
+const { saveConfiguration } = require('../../config/save-config');
+const { buildProjectImage, buildDebugProjectImage } = require('./project-image-builder');
+const checkPHPVersion = require('../requirements/php-version');
+const { getComposerVersionTask } = require('../composer');
+const { prepareFileSystem } = require('../file-system');
+const { installMagentoProject } = require('../magento');
+const enableMagentoComposerPlugins = require('../magento/enable-magento-composer-plugins');
+const { startServices } = require('./index');
+const dockerNetwork = require('./network');
+const KnownError = require('../../errors/known-error');
+const { createCacheFolder } = require('../cache');
+const { getSystemConfigTask } = require('../../config/system-config');
+const sleep = require('../../util/sleep');
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const convertMySQLDatabaseToMariaDB = () => ({
+    task: async (ctx, task) => {
+        const mysqlVolumeName = `${ ctx.config.baseConfig.prefix }_mysql-data`;
+        const volumes = await volumeApi.ls({ formatToJSON: true });
+        if (ctx.isArmMac) {
+            if (volumes.some(
+                (volume) => volume.Name === mysqlVolumeName
+            ) && !volumes.some(
+                (volume) => volume.Name === ctx.config.docker.volumes.mariadb.name
+            )
+            ) {
+                task.title = 'Converting MySQL database to MariaDB';
+                return task.newListr([
+                    stopContainers(),
+                    {
+                        title: 'Converting MySQL volume to MariaDB',
+                        task: async (subCtx, subTask) => {
+                            subTask.output = `Creating volume ${subCtx.config.docker.volumes.mariadb.name}...`;
+                            await volumeApi.create(subCtx.config.docker.volumes.mariadb);
+                            subTask.output = `Copying data from ${mysqlVolumeName} to ${subCtx.config.docker.volumes.mariadb.name}...`;
+                            await execAsyncSpawn(
+                            // eslint-disable-next-line max-len
+                                `docker run --rm -v ${mysqlVolumeName}:/from:ro -v ${subCtx.config.docker.volumes.mariadb.name}:/to alpine ash -c "cd /from; cp -av . /to"`, {
+                                    callback: (t) => {
+                                        subTask.output = t;
+                                    }
+                                }
+                            );
+                            subTask.output = 'Deleting old volume...';
+                            await execAsyncSpawn(`docker volume rm ${mysqlVolumeName}`);
+                        },
+                        options: {
+                            bottomBar: 10
+                        }
+                    }
+                ]);
+            }
+
+            return;
+        }
+
+        if (volumes.some(
+            (volume) => volume.Name === mysqlVolumeName
+        ) && !volumes.some(
+            (volume) => volume.Name === ctx.config.docker.volumes.mariadb.name
+        )
+        ) {
+            task.title = 'Converting MySQL database to MariaDB';
+            const confirmConvert = await task.prompt({
+                type: 'Select',
+                message: `We see that you have old mysql-data volume in the project!
+Since ${logger.style.code('magento-scripts@2.0.0')} ${logger.style.misc('MariaDB')} is used as database instead of MySQL.
+
+To use MariaDB with data from MySQL we need to convert your database.
+`,
+                choices: [
+                    {
+                        name: 'yes',
+                        message: 'Okay, let\'s do that!'
+                    },
+                    {
+                        name: 'no',
+                        message: 'ABORT ABORT ABORT'
+                    }
+                ]
+            });
+
+            if (confirmConvert === 'yes') {
+                const pathToMySQLDumpFile = path.join(process.cwd(), 'mysql-database.sql');
+                const containerName = 'mysql-database';
+                task.output = 'Creating dump file from MySQL database...';
+
+                const existingContainers = await containerApi.ls({ formatToJSON: true, all: true });
+
+                if (existingContainers.some((c) => c.Names === containerName)) {
+                    await execAsyncSpawn(`docker container stop ${containerName}`);
+                    await execAsyncSpawn(`docker container rm ${containerName}`);
+                }
+
+                task.output = 'Starting MySQL server...';
+
+                await containerApi.run(
+                    {
+                        mounts: [
+                            `source=${ mysqlVolumeName },target=/var/lib/mysql`
+                        ],
+                        env: {
+                            MYSQL_PORT: 3306,
+                            MYSQL_ROOT_PASSWORD: 'scandipwa',
+                            MYSQL_USER: 'magento',
+                            MYSQL_PASSWORD: 'magento',
+                            MYSQL_DATABASE: 'magento'
+                        },
+                        command: [
+                            '--log_bin_trust_function_creators=1',
+                            '--default-authentication-plugin=mysql_native_password',
+                            '--max_allowed_packet=1GB',
+                            '--bind-address=0.0.0.0'
+                        ]
+                            .join(' '),
+                        securityOptions: [
+                            'seccomp=unconfined'
+                        ],
+                        name: containerName,
+                        image: `mysql:${ ctx.config.overridenConfiguration.configuration.mysql.version }`
+                    }
+                );
+
+                task.output = 'Waiting for MySQL to initialize...';
+
+                let mysqlReadyForConnections = false;
+
+                while (!mysqlReadyForConnections) {
+                    const mysqlOutput = await execAsyncSpawn(`docker logs ${containerName}`);
+                    if (mysqlOutput.includes('ready for connections')) {
+                        mysqlReadyForConnections = true;
+                        task.output = 'MySQL is ready!';
+                        break;
+                    } else if (mysqlOutput.includes('Initializing database files')) {
+                        task.output = `MySQL is initializing database files!
+Please wait, this will take some time and do not restart the MySQL container until initialization is finished!`;
+
+                        let mysqlFinishedInitialization = false;
+                        while (!mysqlFinishedInitialization) {
+                            const mysqlOutput = await execAsyncSpawn(`docker logs ${containerName}`);
+                            if (mysqlOutput.includes('init process done.') && !mysqlFinishedInitialization) {
+                                mysqlFinishedInitialization = true;
+                                task.output = 'MySQL is initialized!';
+                                break;
+                            }
+                            await sleep(2000);
+                        }
+                    }
+
+                    await sleep(2000);
+                }
+
+                task.output = 'Dumping MySQL database to dump file...';
+
+                await containerApi.exec(
+                    [
+                        'mysqldump',
+                        '--user=root',
+                        '--password=scandipwa',
+                        'magento',
+                        `--result-file=${ path.parse(pathToMySQLDumpFile).base }`
+                    ].join(' '),
+                    containerName,
+                    {},
+                    {
+                        callback: (t) => {
+                            task.output = t;
+                        }
+                    }
+                );
+
+                task.output = 'Copying dump file from container to system...';
+
+                await execAsyncSpawn(
+                    `docker cp ${containerName}:/${ path.parse(pathToMySQLDumpFile).base } ${ pathToMySQLDumpFile }`
+                );
+
+                task.output = 'Removing migration container...';
+
+                await execAsyncSpawn(
+                    `docker container stop ${containerName}`
+                );
+                await execAsyncSpawn(
+                    `docker container rm ${containerName}`
+                );
+
+                ctx.importDb = pathToMySQLDumpFile;
+
+                return task.newListr([
+                    createCacheFolder(),
+                    getSystemConfigTask(),
+                    getCachedPorts(),
+                    stopContainers(),
+                    setPrefix(),
+                    getProjectConfiguration(),
+                    // get fresh ports
+                    getAvailablePorts(),
+                    saveConfiguration(),
+                    pullImages(),
+                    dockerNetwork.tasks.createNetwork(),
+                    createVolumes(),
+                    {
+                        task: (ctx, task) => task.newListr([
+                            buildProjectImage(),
+                            buildDebugProjectImage()
+                        ], {
+                            concurrent: true
+                        })
+                    },
+                    checkPHPVersion(),
+                    getComposerVersionTask(),
+                    prepareFileSystem(),
+                    installMagentoProject(),
+                    enableMagentoComposerPlugins(),
+                    startServices(),
+                    connectToDatabase(),
+                    importDumpToDatabase(),
+                    {
+                        task: async (subCtx, subTask) => {
+                            const confirmDeleteOldVolume = await subTask.prompt({
+                                type: 'Select',
+                                message: 'Okay, looks like conversion went well, do you want to delete old mysql-data volume?',
+                                choices: [
+                                    {
+                                        name: 'yes',
+                                        message: 'Sure'
+                                    },
+                                    {
+                                        name: 'no',
+                                        message: 'No, I want to keep it'
+                                    }
+                                ]
+                            });
+
+                            if (confirmDeleteOldVolume === 'yes') {
+                                await execAsyncSpawn(`docker volume rm ${ mysqlVolumeName }`);
+
+                                const confirmDeleteDump = await subTask.prompt({
+                                    type: 'Select',
+                                    message: `Last question for today.
+Do you want to keep database dump created during conversion process? (${logger.style.file(pathToMySQLDumpFile)})`,
+                                    choices: [
+                                        {
+                                            name: 'yes',
+                                            message: 'I don\'t need it, thanks'
+                                        },
+                                        {
+                                            name: 'no',
+                                            message: 'I would like to keep it, thanks'
+                                        }
+                                    ]
+                                });
+
+                                if (confirmDeleteDump === 'yes') {
+                                    await fs.promises.rm(pathToMySQLDumpFile);
+                                }
+                            }
+                        }
+                    }
+                ]);
+            }
+            throw new KnownError('ABORTING');
+        }
+
+        task.skip();
+    },
+    options: {
+        bottomBar: 10
+    }
+});
+
+module.exports = convertMySQLDatabaseToMariaDB;

package/lib/tasks/docker/image/image-api.d.ts

@@ -0,0 +1,34 @@
+import { ExecAsyncSpawnOptions } from '../../../util/exec-async-command';
+
+export interface ImagesLsOptions<T extends boolean = false> {
+    all?: boolean
+    digests?: boolean
+    filter?: string | string[]
+    format?: string
+    quiet?: boolean
+    noTrunc?: boolean
+    formatToJSON?: T
+}
+
+export interface ImagesLsResult {
+    Containers: string
+    CreatedAt: string
+    CreatedSince: string
+    Digest: string
+    ID: string
+    Repository: string
+    SharedSize: string
+    Size: string
+    Tag: string
+    UniqueSize: string
+    VirtualSize: string
+}
+
+export function ls(
+    options?: ImagesLsOptions,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<string>
+export function ls(
+    options?: ImagesLsOptions<true>,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<ImagesLsResult[]>

package/lib/tasks/docker/image/image-api.js

@@ -0,0 +1,49 @@
+const { execAsyncSpawn } = require('../../../util/exec-async-command');
+
+/**
+ *
+ * @param {import('./image-api').ImagesLsOptions} options
+ * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions} execOptions
+ */
+const ls = async (options, execOptions = {}) => {
+    const {
+        all,
+        filter,
+        format,
+        formatToJSON = false,
+        noTrunc,
+        quiet,
+        digests
+    } = options;
+
+    const allArg = all && '--all';
+    const filterArg = filter && typeof filter === 'string'
+        ? `--filter=${filter}`
+        : filter && Array.isArray(filter) && filter.every((f) => typeof f === 'string') && filter.map((f) => `--filter=${f}`).join(' ');
+    const formatArg = !formatToJSON && format
+        ? `--format=${format}`
+        : formatToJSON && '--format=\'{{json .}}\'';
+    const digestsArg = digests && '--digests';
+    const noTruncArg = noTrunc && '--no-trunc';
+    const quietArg = quiet && '--quiet';
+
+    const args = [
+        allArg,
+        filterArg,
+        formatArg,
+        digestsArg,
+        noTruncArg,
+        quietArg
+    ].filter(Boolean).join(' ');
+
+    if (formatToJSON) {
+        const result = await execAsyncSpawn(`docker image ls ${args}`, execOptions);
+        return JSON.parse(`[${result.split('\n').join(', ')}]`);
+    }
+
+    return execAsyncSpawn(`docker image ls ${args}`, execOptions);
+};
+
+module.exports = {
+    ls
+};

package/lib/tasks/docker/image/index.js

@@ -0,0 +1,5 @@
+const imageApi = require('./image-api');
+
+module.exports = {
+    imageApi
+};