@scandipwa/magento-scripts 1.16.1 → 2.0.0-alpha.2

package/lib/tasks/docker/convert-mysql-to-mariadb.js

@@ -0,0 +1,284 @@
+const path = require('path');
+const fs = require('fs');
+const logger = require('@scandipwa/scandipwa-dev-utils/logger');
+const { volumeApi, createVolumes } = require('./volume');
+const { execAsyncSpawn } = require('../../util/exec-async-command');
+const { containerApi, stopContainers, pullImages } = require('./containers');
+const { importDumpToDatabase, connectToDatabase } = require('../database');
+const { setPrefix } = require('../prefix');
+const getProjectConfiguration = require('../../config/get-project-configuration');
+const { getAvailablePorts, getCachedPorts } = require('../../config/get-port-config');
+const { saveConfiguration } = require('../../config/save-config');
+const { buildProjectImage, buildDebugProjectImage } = require('./project-image-builder');
+const checkPHPVersion = require('../requirements/php-version');
+const { getComposerVersionTask } = require('../composer');
+const { prepareFileSystem } = require('../file-system');
+const { installMagentoProject } = require('../magento');
+const enableMagentoComposerPlugins = require('../magento/enable-magento-composer-plugins');
+const { startServices } = require('./index');
+const dockerNetwork = require('./network');
+const KnownError = require('../../errors/known-error');
+const { createCacheFolder } = require('../cache');
+const { getSystemConfigTask } = require('../../config/system-config');
+const sleep = require('../../util/sleep');
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const convertMySQLDatabaseToMariaDB = () => ({
+    task: async (ctx, task) => {
+        const mysqlVolumeName = `${ ctx.config.baseConfig.prefix }_mysql-data`;
+        const volumes = await volumeApi.ls({ formatToJSON: true });
+        if (ctx.isArmMac) {
+            if (volumes.some(
+                (volume) => volume.Name === mysqlVolumeName
+            ) && !volumes.some(
+                (volume) => volume.Name === ctx.config.docker.volumes.mariadb.name
+            )
+            ) {
+                task.title = 'Converting MySQL database to MariaDB';
+                return task.newListr([
+                    stopContainers(),
+                    {
+                        title: 'Converting MySQL volume to MariaDB',
+                        task: async (subCtx, subTask) => {
+                            subTask.output = `Creating volume ${subCtx.config.docker.volumes.mariadb.name}...`;
+                            await volumeApi.create(subCtx.config.docker.volumes.mariadb);
+                            subTask.output = `Copying data from ${mysqlVolumeName} to ${subCtx.config.docker.volumes.mariadb.name}...`;
+                            await execAsyncSpawn(
+                            // eslint-disable-next-line max-len
+                                `docker run --rm -v ${mysqlVolumeName}:/from:ro -v ${subCtx.config.docker.volumes.mariadb.name}:/to alpine ash -c "cd /from; cp -av . /to"`, {
+                                    callback: (t) => {
+                                        subTask.output = t;
+                                    }
+                                }
+                            );
+                            subTask.output = 'Deleting old volume...';
+                            await execAsyncSpawn(`docker volume rm ${mysqlVolumeName}`);
+                        },
+                        options: {
+                            bottomBar: 10
+                        }
+                    }
+                ]);
+            }
+
+            return;
+        }
+
+        if (volumes.some(
+            (volume) => volume.Name === mysqlVolumeName
+        ) && !volumes.some(
+            (volume) => volume.Name === ctx.config.docker.volumes.mariadb.name
+        )
+        ) {
+            task.title = 'Converting MySQL database to MariaDB';
+            const confirmConvert = await task.prompt({
+                type: 'Select',
+                message: `We see that you have old mysql-data volume in the project!
+Since ${logger.style.code('magento-scripts@2.0.0')} ${logger.style.misc('MariaDB')} is used as database instead of MySQL.
+
+To use MariaDB with data from MySQL we need to convert your database.
+`,
+                choices: [
+                    {
+                        name: 'yes',
+                        message: 'Okay, let\'s do that!'
+                    },
+                    {
+                        name: 'no',
+                        message: 'ABORT ABORT ABORT'
+                    }
+                ]
+            });
+
+            if (confirmConvert === 'yes') {
+                const pathToMySQLDumpFile = path.join(process.cwd(), 'mysql-database.sql');
+                const containerName = 'mysql-database';
+                task.output = 'Creating dump file from MySQL database...';
+
+                const existingContainers = await containerApi.ls({ formatToJSON: true, all: true });
+
+                if (existingContainers.some((c) => c.Names === containerName)) {
+                    await execAsyncSpawn(`docker container stop ${containerName}`);
+                    await execAsyncSpawn(`docker container rm ${containerName}`);
+                }
+
+                task.output = 'Starting MySQL server...';
+
+                await containerApi.run(
+                    {
+                        mounts: [
+                            `source=${ mysqlVolumeName },target=/var/lib/mysql`
+                        ],
+                        env: {
+                            MYSQL_PORT: 3306,
+                            MYSQL_ROOT_PASSWORD: 'scandipwa',
+                            MYSQL_USER: 'magento',
+                            MYSQL_PASSWORD: 'magento',
+                            MYSQL_DATABASE: 'magento'
+                        },
+                        command: [
+                            '--log_bin_trust_function_creators=1',
+                            '--default-authentication-plugin=mysql_native_password',
+                            '--max_allowed_packet=1GB',
+                            '--bind-address=0.0.0.0'
+                        ]
+                            .join(' '),
+                        securityOptions: [
+                            'seccomp=unconfined'
+                        ],
+                        name: containerName,
+                        image: `mysql:${ ctx.config.overridenConfiguration.configuration.mysql.version }`
+                    }
+                );
+
+                task.output = 'Waiting for MySQL to initialize...';
+
+                let mysqlReadyForConnections = false;
+
+                while (!mysqlReadyForConnections) {
+                    const mysqlOutput = await execAsyncSpawn(`docker logs ${containerName}`);
+                    if (mysqlOutput.includes('ready for connections')) {
+                        mysqlReadyForConnections = true;
+                        task.output = 'MySQL is ready!';
+                        break;
+                    } else if (mysqlOutput.includes('Initializing database files')) {
+                        task.output = `MySQL is initializing database files!
+Please wait, this will take some time and do not restart the MySQL container until initialization is finished!`;
+
+                        let mysqlFinishedInitialization = false;
+                        while (!mysqlFinishedInitialization) {
+                            const mysqlOutput = await execAsyncSpawn(`docker logs ${containerName}`);
+                            if (mysqlOutput.includes('init process done.') && !mysqlFinishedInitialization) {
+                                mysqlFinishedInitialization = true;
+                                task.output = 'MySQL is initialized!';
+                                break;
+                            }
+                            await sleep(2000);
+                        }
+                    }
+
+                    await sleep(2000);
+                }
+
+                task.output = 'Dumping MySQL database to dump file...';
+
+                await containerApi.exec(
+                    [
+                        'mysqldump',
+                        '--user=root',
+                        '--password=scandipwa',
+                        'magento',
+                        `--result-file=${ path.parse(pathToMySQLDumpFile).base }`
+                    ].join(' '),
+                    containerName,
+                    {},
+                    {
+                        callback: (t) => {
+                            task.output = t;
+                        }
+                    }
+                );
+
+                task.output = 'Copying dump file from container to system...';
+
+                await execAsyncSpawn(
+                    `docker cp ${containerName}:/${ path.parse(pathToMySQLDumpFile).base } ${ pathToMySQLDumpFile }`
+                );
+
+                task.output = 'Removing migration container...';
+
+                await execAsyncSpawn(
+                    `docker container stop ${containerName}`
+                );
+                await execAsyncSpawn(
+                    `docker container rm ${containerName}`
+                );
+
+                ctx.importDb = pathToMySQLDumpFile;
+
+                return task.newListr([
+                    createCacheFolder(),
+                    getSystemConfigTask(),
+                    getCachedPorts(),
+                    stopContainers(),
+                    setPrefix(),
+                    getProjectConfiguration(),
+                    // get fresh ports
+                    getAvailablePorts(),
+                    saveConfiguration(),
+                    pullImages(),
+                    dockerNetwork.tasks.createNetwork(),
+                    createVolumes(),
+                    {
+                        task: (ctx, task) => task.newListr([
+                            buildProjectImage(),
+                            buildDebugProjectImage()
+                        ], {
+                            concurrent: true
+                        })
+                    },
+                    checkPHPVersion(),
+                    getComposerVersionTask(),
+                    prepareFileSystem(),
+                    installMagentoProject(),
+                    enableMagentoComposerPlugins(),
+                    startServices(),
+                    connectToDatabase(),
+                    importDumpToDatabase(),
+                    {
+                        task: async (subCtx, subTask) => {
+                            const confirmDeleteOldVolume = await subTask.prompt({
+                                type: 'Select',
+                                message: 'Okay, looks like conversion went well, do you want to delete old mysql-data volume?',
+                                choices: [
+                                    {
+                                        name: 'yes',
+                                        message: 'Sure'
+                                    },
+                                    {
+                                        name: 'no',
+                                        message: 'No, I want to keep it'
+                                    }
+                                ]
+                            });
+
+                            if (confirmDeleteOldVolume === 'yes') {
+                                await execAsyncSpawn(`docker volume rm ${ mysqlVolumeName }`);
+
+                                const confirmDeleteDump = await subTask.prompt({
+                                    type: 'Select',
+                                    message: `Last question for today.
+Do you want to keep database dump created during conversion process? (${logger.style.file(pathToMySQLDumpFile)})`,
+                                    choices: [
+                                        {
+                                            name: 'yes',
+                                            message: 'I don\'t need it, thanks'
+                                        },
+                                        {
+                                            name: 'no',
+                                            message: 'I would like to keep it, thanks'
+                                        }
+                                    ]
+                                });
+
+                                if (confirmDeleteDump === 'yes') {
+                                    await fs.promises.rm(pathToMySQLDumpFile);
+                                }
+                            }
+                        }
+                    }
+                ]);
+            }
+            throw new KnownError('ABORTING');
+        }
+
+        task.skip();
+    },
+    options: {
+        bottomBar: 10
+    }
+});
+
+module.exports = convertMySQLDatabaseToMariaDB;

package/lib/tasks/docker/image/image-api.d.ts

@@ -0,0 +1,34 @@
+import { ExecAsyncSpawnOptions } from '../../../util/exec-async-command';
+
+export interface ImagesLsOptions<T extends boolean = false> {
+    all?: boolean
+    digests?: boolean
+    filter?: string | string[]
+    format?: string
+    quiet?: boolean
+    noTrunc?: boolean
+    formatToJSON?: T
+}
+
+export interface ImagesLsResult {
+    Containers: string
+    CreatedAt: string
+    CreatedSince: string
+    Digest: string
+    ID: string
+    Repository: string
+    SharedSize: string
+    Size: string
+    Tag: string
+    UniqueSize: string
+    VirtualSize: string
+}
+
+export function ls(
+    options?: ImagesLsOptions,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<string>
+export function ls(
+    options?: ImagesLsOptions<true>,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<ImagesLsResult[]>

package/lib/tasks/docker/image/image-api.js

@@ -0,0 +1,49 @@
+const { execAsyncSpawn } = require('../../../util/exec-async-command');
+
+/**
+ *
+ * @param {import('./image-api').ImagesLsOptions} options
+ * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions} execOptions
+ */
+const ls = async (options, execOptions = {}) => {
+    const {
+        all,
+        filter,
+        format,
+        formatToJSON = false,
+        noTrunc,
+        quiet,
+        digests
+    } = options;
+
+    const allArg = all && '--all';
+    const filterArg = filter && typeof filter === 'string'
+        ? `--filter=${filter}`
+        : filter && Array.isArray(filter) && filter.every((f) => typeof f === 'string') && filter.map((f) => `--filter=${f}`).join(' ');
+    const formatArg = !formatToJSON && format
+        ? `--format=${format}`
+        : formatToJSON && '--format=\'{{json .}}\'';
+    const digestsArg = digests && '--digests';
+    const noTruncArg = noTrunc && '--no-trunc';
+    const quietArg = quiet && '--quiet';
+
+    const args = [
+        allArg,
+        filterArg,
+        formatArg,
+        digestsArg,
+        noTruncArg,
+        quietArg
+    ].filter(Boolean).join(' ');
+
+    if (formatToJSON) {
+        const result = await execAsyncSpawn(`docker image ls ${args}`, execOptions);
+        return JSON.parse(`[${result.split('\n').join(', ')}]`);
+    }
+
+    return execAsyncSpawn(`docker image ls ${args}`, execOptions);
+};
+
+module.exports = {
+    ls
+};

package/lib/tasks/docker/image/index.js

@@ -0,0 +1,5 @@
+const imageApi = require('./image-api');
+
+module.exports = {
+    imageApi
+};

package/lib/tasks/docker/index.js

@@ -1,6 +1,5 @@
 const containers = require('./containers');
 const network = require('./network');
-const volumes = require('./volumes');
 
 /**
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
@@ -8,17 +7,6 @@ const volumes = require('./volumes');
 const startServices = () => ({
     title: 'Starting Docker services',
     task: (ctx, task) => task.newListr([
-        {
-            title: 'Preparing services',
-            task: (ctx, task) => task.newListr([
-                network.createNetwork(),
-                volumes.createVolumes()
-            ], {
-                concurrent: true,
-                exitOnError: true,
-                ctx
-            })
-        },
         containers.startContainers(),
         containers.checkContainersAreRunning()
     ], {
@@ -33,7 +21,7 @@ const startServices = () => ({
 const stopServices = () => ({
     task: (ctx, task) => task.newListr([
         containers.stopContainers(),
-        network.removeNetwork()
+        network.tasks.removeNetwork()
     ])
 });
 

package/lib/tasks/docker/network/index.js

@@ -0,0 +1,7 @@
+const networkApi = require('./network-api');
+const tasks = require('./tasks');
+
+module.exports = {
+    networkApi,
+    tasks
+};

package/lib/tasks/docker/network/network-api.d.ts

@@ -0,0 +1,103 @@
+import { ExecAsyncSpawnOptions } from '../../../util/exec-async-command';
+
+export interface NetworkLsOptions<T extends boolean = false> {
+    filter?: string | string[]
+    format?: string
+    formatToJSON?: T
+    quiet?: boolean
+    noTrunc?: boolean
+}
+
+export interface NetworkLsResult {
+    CreatedAt: string
+    Driver: string
+    ID: string
+    IPv6: string
+    Internal: string
+    Labels: string
+    Name: string
+    Scope: string
+}
+
+export function ls(
+    options?: NetworkLsOptions,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<string>
+export function ls(
+    options?: NetworkLsOptions<true>,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<NetworkLsResult[]>
+
+export interface NetworkInspectOptions<T extends boolean = false> {
+    network: string | string[]
+    format?: string
+    formatToJSON?: T
+    verbose?: boolean
+}
+
+export interface NetworkInspectResult {
+    Name: string
+    Id: string
+    Created: string
+    Scope: string
+    EnableIPv6: boolean
+    IPAM: {
+        Driver: string
+        Options: Record<string, unknown>
+        Config: {
+            Subnet: string
+            Gateway: string
+        }[]
+    }
+    Internal: boolean
+    Attachable: boolean
+    Ingress: boolean
+    ConfigFrom: {
+        Network: string
+    }
+    ConfigOnly: boolean
+    Containers: Record<string, {
+        Name: string
+        EndpointID: string
+        MacAddress: string
+        IPv4Address: string
+        IPv6Address: string
+    }>
+    Options: Record<string, unknown>
+    Labels: Record<string, unknown>
+}
+
+export function inspect(
+    options?: NetworkInspectOptions,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<string>
+
+export function inspect(
+    options?: NetworkInspectOptions<true>,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<NetworkInspectResult>
+
+export interface NetworkCreateOptions {
+    network: string | string[]
+    attachable?: boolean
+    auxAddress?: string
+    configFrom?: string
+    configOnly?: string
+    driver?: string
+    gateway?: string
+    ingress?: boolean
+    internal?: boolean
+    ipRange?: string
+    ipamDriver?: string
+    ipamOpt?: string
+    ipv6?: boolean
+    label?: string
+    opt?: string
+    scope?: string
+    subnet?: string
+}
+
+export function create(
+    options?: NetworkCreateOptions,
+    execOptions?: ExecAsyncSpawnOptions<false>
+): Promise<string>

package/lib/tasks/docker/network/network-api.js

@@ -0,0 +1,99 @@
+const { execAsyncSpawn } = require('../../../util/exec-async-command');
+
+/**
+ * @param {import('./network-api').NetworkLsOptions} options
+ * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions} execOptions
+ */
+const ls = async (options, execOptions = {}) => {
+    const {
+        filter,
+        format,
+        formatToJSON = false,
+        noTrunc,
+        quiet
+    } = options;
+
+    const formatArg = !formatToJSON && format
+        ? `--format=${format}`
+        : formatToJSON && '--format=\'{{json .}}\'';
+    const filterArg = filter && typeof filter === 'string'
+        ? `--filter=${filter}`
+        : filter && Array.isArray(filter) && filter.every((f) => typeof f === 'string') && filter.map((f) => `--filter=${f}`).join(' ');
+    const noTruncArg = noTrunc && '--no-trunc';
+    const quietArg = quiet && '--quiet';
+    const args = [
+        filterArg,
+        formatArg,
+        noTruncArg,
+        quietArg
+    ].filter(Boolean).join(' ');
+
+    if (formatToJSON) {
+        const result = await execAsyncSpawn(`docker network ls ${args}`, execOptions);
+        return JSON.parse(`[${result.split('\n').join(', ')}]`);
+    }
+
+    return execAsyncSpawn(`docker network ls ${args}`, execOptions);
+};
+
+/**
+ * @param {import('./network-api').NetworkInspectOptions} options
+ * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions} execOptions
+ */
+const inspect = async (options, execOptions = {}) => {
+    const {
+        network,
+        format,
+        formatToJSON = false,
+        verbose
+    } = options;
+
+    const formatArg = !formatToJSON && format
+        ? `--format=${format}`
+        : formatToJSON && '--format=\'{{json .}}\'';
+    const verboseArg = verbose && '--verbose';
+    const networks = typeof network === 'string'
+        ? network
+        : network.join(' ');
+
+    const args = [
+        formatArg,
+        verboseArg
+    ].filter(Boolean).join(' ');
+
+    if (formatToJSON) {
+        const result = await execAsyncSpawn(`docker network inspect ${args} ${networks}`, execOptions);
+        return JSON.parse(result);
+    }
+
+    return execAsyncSpawn(`docker network inspect ${args} ${networks}`, execOptions);
+};
+
+/**
+ * @param {import('./network-api').NetworkCreateOptions} options
+ * @param {import('../../../util/exec-async-command').ExecAsyncSpawnOptions} execOptions
+ */
+const create = async (options, execOptions = {}) => {
+    const {
+        network,
+        driver
+    } = options;
+
+    const networks = typeof network === 'string'
+        ? network
+        : network.join(' ');
+
+    const driverArg = driver && `--driver='${ driver }'`;
+
+    const args = [
+        driverArg
+    ].filter(Boolean).join(' ');
+
+    return execAsyncSpawn(`docker network create ${args} ${networks}`, execOptions);
+};
+
+module.exports = {
+    ls,
+    inspect,
+    create
+};

package/lib/tasks/docker/{network.js → network/tasks.js}

@@ -1,21 +1,26 @@
 const logger = require('@scandipwa/scandipwa-dev-utils/logger');
-const KnownError = require('../../errors/known-error');
-const { execAsyncSpawn } = require('../../util/exec-async-command');
+const KnownError = require('../../../errors/known-error');
+const UnknownError = require('../../../errors/unknown-error');
+const { execAsyncSpawn } = require('../../../util/exec-async-command');
+const networkApi = require('./network-api');
 
 /**
- * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ * @type {() => import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
  */
 const createNetwork = () => ({
     title: 'Deploying Docker network',
     task: async ({ config: { docker } }, task) => {
-        const networkList = (await execAsyncSpawn("docker network ls --format '{{.Name}}'")).split('\n');
+        const networkList = await networkApi.ls({ formatToJSON: true });
 
-        if (networkList.includes(docker.network.name)) {
+        if (networkList.some((network) => network.Name === docker.network.name)) {
             task.skip();
             return;
         }
         try {
-            await execAsyncSpawn(`docker network create --driver=bridge ${ docker.network.name }`);
+            await networkApi.create({
+                network: docker.network.name,
+                driver: 'bridge'
+            });
         } catch (e) {
             if (e.includes('could not find an available, non-overlapping IPv4 address pool')) {
                 const pruneNetworks = await task.prompt({
@@ -34,19 +39,21 @@ Do you want remove all custom networks not used by at least one container?`
 You need to delete unused networks yourself.
 Use command ${logger.style.command('docker network prune')}`);
             }
+
+            throw new UnknownError(`Unable to create network!\n\n${e}`);
         }
     }
 });
 
 /**
- * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ * @type {() => import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
  */
 const removeNetwork = () => ({
     title: 'Removing Docker network',
     task: async ({ config: { docker } }, task) => {
-        const networkList = (await execAsyncSpawn("docker network ls --format '{{.Name}}'")).split('\n');
+        const networkList = await networkApi.ls({ formatToJSON: true });
 
-        if (!networkList.includes(docker.network.name)) {
+        if (!networkList.some((network) => network.Name === docker.network.name)) {
             task.skip();
             return;
         }
@@ -56,7 +63,7 @@ const removeNetwork = () => ({
 });
 
 /**
- * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ * @type {() => import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
  */
 const pruneNetworks = () => ({
     title: 'Removing custom networks not used by at least one container',