@scandipwa/magento-scripts 1.16.0-alpha.4 → 2.0.0-alpha.0

package/lib/tasks/magento/setup-magento/waiting-for-varnish.js

@@ -1,14 +1,48 @@
+const logger = require('@scandipwa/scandipwa-dev-utils/logger');
+const semver = require('semver');
+const fs = require('fs');
+const path = require('path');
 const { request } = require('smol-request');
 const KnownError = require('../../../errors/known-error');
+const { execAsyncSpawn } = require('../../../util/exec-async-command');
+const { NginxParser } = require('../../../util/nginx-logs-parser');
 const sleep = require('../../../util/sleep');
 
+const pathToWorkingHealthCheckPhp = path.join(__dirname, '..', '..', 'php', 'working_health_check.php');
+const pathToProjectsHealthCheckPhp = path.join(process.cwd(), 'pub', 'health_check.php');
+const pathToHealthCheckBackupPhp = path.join(process.cwd(), 'pub', 'health_check-super-backup-file.php');
+
+/**
+ * @param {import('../../../../typings/context').ListrContext} ctx
+ */
+const getIsHealthCheckRequestBroken = async (ctx) => {
+    const { nginx } = ctx.config.docker.getContainers(ctx.ports);
+
+    const nginxLogs = await execAsyncSpawn(`docker logs ${nginx.name}`);
+
+    const healthCheckRequests = nginxLogs
+        .split('\n')
+        .filter((line) => line.includes('"GET /health_check.php HTTP/1.1"'));
+
+    // eslint-disable-next-line max-len
+    const parser = new NginxParser('$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"');
+
+    const parsedRequests = healthCheckRequests.map((line) => parser.parseLine(line));
+
+    return parsedRequests.every((parsedRequest) => parsedRequest.status !== '200');
+};
+
 /**
  * @returns {import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
  */
 const waitingForVarnish = () => ({
     title: 'Waiting for Varnish to return code 200',
-    skip: (ctx) => !ctx.config.overridenConfiguration.configuration.varnish.enabled,
-    task: async (ctx) => {
+    skip: (ctx) => ctx.debug || !ctx.config.overridenConfiguration.configuration.varnish.enabled,
+    task: async (ctx, task) => {
+        const pureMagentoVersion = ctx.magentoVersion.match(/^([0-9]+\.[0-9]+\.[0-9]+)/)[1];
+
+        const isMagento23 = semver.satisfies(pureMagentoVersion, '<2.4');
+
         let tries = 0;
         while (tries < 10) {
             try {
@@ -28,6 +62,94 @@ const waitingForVarnish = () => ({
             }
         }
 
+        if (tries === 10 && isMagento23) {
+            const isHealthCheckRequestBroken = await getIsHealthCheckRequestBroken(ctx);
+            if (isHealthCheckRequestBroken) {
+                const confirm = await task.prompt({
+                    type: 'Select',
+                    message: `We detected that your Magento instance is experiencing problems with Varnish server.
+Health Check request is returning status 500.
+
+Do you want to try resolving this issue by replacing ${logger.style.file('./pub/health_check.php')} file content with content from newer Magento version (2.4.4)?`,
+                    choices: [
+                        {
+                            name: 'yes',
+                            message: 'Okay, let\'s try that'
+                        },
+                        {
+                            name: 'no',
+                            message: 'I will try to fix this myself, thanks for the info!'
+                        }
+                    ]
+                });
+
+                if (confirm === 'yes') {
+                    await fs.promises.writeFile(
+                        pathToHealthCheckBackupPhp,
+                        await fs.promises.readFile(pathToProjectsHealthCheckPhp, 'utf-8'),
+                        'utf-8'
+                    );
+
+                    task.output = `Backup made! Available at ${logger.style.file(pathToHealthCheckBackupPhp)}`;
+
+                    await fs.promises.writeFile(
+                        pathToProjectsHealthCheckPhp,
+                        await fs.promises.readFile(pathToWorkingHealthCheckPhp, 'utf-8'),
+                        'utf-8'
+                    );
+
+                    task.output = `${logger.style.file('./pub/health_check.php')} content is replaced, waiting for nginx logs...`;
+
+                    let connectionFixed = false;
+
+                    for (let i = 0; i < 10; i++) {
+                        try {
+                            const response = await request(`http://localhost:${ctx.ports.sslTerminator}/`, {
+                                responseType: 'headers'
+                            });
+
+                            if (response.status !== 200) {
+                                await sleep(2000);
+                            } else {
+                                connectionFixed = true;
+                                break;
+                            }
+                        } catch (e) {
+                            await sleep(200);
+                        }
+                    }
+
+                    if (connectionFixed) {
+                        task.output = 'Connection looks good!';
+                        const secondConfirm = await task.prompt({
+                            type: 'Select',
+                            message: `Okay, looks like it helped!
+Do you want to keep backed up ${logger.style.file('health_check.php')} file?`,
+                            choices: [
+                                {
+                                    name: 'no',
+                                    message: 'I don\'t think I will need it anymore, thanks'
+                                },
+                                {
+                                    name: 'yes',
+                                    message: 'I will keep it, maybe it will be useful (X to doubt)'
+                                }
+                            ]
+                        });
+
+                        if (secondConfirm === 'no') {
+                            await fs.promises.rm(pathToHealthCheckBackupPhp);
+                        }
+                    }
+
+                    return;
+                }
+
+                task.skip('User choose to fix issue himself');
+                return;
+            }
+        }
+
         if (tries === 10) {
             throw new KnownError(`After 20 seconds website is still responding with non-200 code, which might indicate issue with setup.
 Or Varnish is still loading...
@@ -35,7 +157,10 @@ Or Varnish is still loading...
 Please check the logs!`);
         }
     },
-    exitOnError: false
+    exitOnError: false,
+    options: {
+        bottomBar: 10
+    }
 });
 
 module.exports = waitingForVarnish;

package/lib/tasks/mysql/connect-to-mysql.js

@@ -1,4 +1,4 @@
-const mysql = require('mysql2/promise');
+const mysql2 = require('mysql2/promise');
 const UnknownError = require('../../errors/unknown-error');
 const { execAsyncSpawn } = require('../../util/exec-async-command');
 const sleep = require('../../util/sleep');
@@ -10,10 +10,14 @@ const { createMagentoDatabase } = require('./create-magento-database');
 const waitForMySQLInitialization = () => ({
     title: 'Waiting for MySQL to initialize',
     task: async (ctx, task) => {
-        const { mysql: { name } } = ctx.config.docker.getContainers();
+        const { mysql } = ctx.config.docker.getContainers();
+
+        task.title = `Waiting for ${mysql._} to initialize`;
+
         let mysqlReadyForConnections = false;
+
         while (!mysqlReadyForConnections) {
-            const mysqlOutput = await execAsyncSpawn(`docker logs ${name}`);
+            const mysqlOutput = await execAsyncSpawn(`docker logs ${mysql.name}`);
             if (mysqlOutput.includes('ready for connections')) {
                 mysqlReadyForConnections = true;
                 break;
@@ -23,7 +27,7 @@ Please wait, this will take some time and do not restart the MySQL container unt
 
                 let mysqlFinishedInitialization = false;
                 while (!mysqlFinishedInitialization) {
-                    const mysqlOutput = await execAsyncSpawn(`docker logs ${name}`);
+                    const mysqlOutput = await execAsyncSpawn(`docker logs ${mysql.name}`);
                     if (mysqlOutput.includes('init process done.') && !mysqlFinishedInitialization) {
                         mysqlFinishedInitialization = true;
                         break;
@@ -47,19 +51,22 @@ const gettingMySQLConnection = () => ({
     title: 'Getting MySQL connection',
     task: async (ctx, task) => {
         const { config: { docker }, ports } = ctx;
-        const { mysql: { env } } = docker.getContainers();
+        const { mysql } = docker.getContainers(ctx.ports);
         let tries = 0;
         const maxTries = 20;
         const errors = [];
+
+        task.title = `Getting ${mysql._} connection`;
+
         while (tries < maxTries) {
             tries++;
             try {
-                const connection = await mysql.createConnection({
+                const connection = await mysql2.createConnection({
                     host: '127.0.0.1',
                     port: ports.mysql,
-                    user: env.MYSQL_USER,
-                    password: env.MYSQL_PASSWORD,
-                    database: env.MYSQL_DATABASE
+                    user: mysql.env.MYSQL_USER,
+                    password: mysql.env.MYSQL_PASSWORD,
+                    database: mysql.env.MYSQL_DATABASE
                 });
 
                 ctx.mysqlConnection = connection;
@@ -78,22 +85,40 @@ const gettingMySQLConnection = () => ({
     }
 });
 
+/**
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const terminatingExistingConnection = () => ({
+    title: 'Terminating existing Database connection',
+    skip: (ctx) => !ctx.mysqlConnection,
+    task: (ctx) => {
+        ctx.mysqlConnection.destroy();
+    }
+});
+
 /**
  * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
  */
 const connectToMySQL = () => ({
     title: 'Connecting to MySQL server',
     skip: (ctx) => ctx.skipSetup,
-    task: (ctx, task) => task.newListr([
-        waitForMySQLInitialization(),
-        createMagentoDatabase(),
-        gettingMySQLConnection()
-    ], {
-        concurrent: false,
-        rendererOptions: {
-            collapse: true
-        }
-    }),
+    task: (ctx, task) => {
+        const { mysql } = ctx.config.docker.getContainers();
+
+        task.title = `Connecting to ${mysql._} server`;
+
+        return task.newListr([
+            waitForMySQLInitialization(),
+            createMagentoDatabase(),
+            terminatingExistingConnection(),
+            gettingMySQLConnection()
+        ], {
+            concurrent: false,
+            rendererOptions: {
+                collapse: true
+            }
+        });
+    },
     options: {
         bottomBar: 10
     }

package/lib/tasks/mysql/import-dump-to-mysql.js

@@ -2,23 +2,100 @@
 const logger = require('@scandipwa/scandipwa-dev-utils/logger');
 const KnownError = require('../../errors/known-error');
 const UnknownError = require('../../errors/unknown-error');
-const { execAsyncSpawn } = require('../../util/exec-async-command');
+const { execAsyncSpawn, execCommandTask } = require('../../util/exec-async-command');
 const pathExists = require('../../util/path-exists');
+const connectToMySQL = require('./connect-to-mysql');
 
 /**
  * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
  */
-const importDumpToMySQL = () => ({
-    title: 'Importing Database Dump To MySQL',
+const copyDatabaseDumpIntoContainer = () => ({
+    title: 'Copying database dump into container',
     task: async (ctx, task) => {
-        if (!await pathExists(ctx.importDb)) {
-            throw new KnownError(`Dump file at ${ctx.importDb} does not exist. Please provide correct relative path to the file`);
-        }
+        const { config: { docker }, ports } = ctx;
+        const { mysql } = docker.getContainers(ports);
+
+        return task.newListr(
+            execCommandTask(`docker cp ${ctx.importDb} ${mysql.name}:/dump.sql`, {
+                logOutput: true
+            })
+        );
+    }
+});
 
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const runSetGlobalLogBinTrustFunctionCreatorsCommand = () => ({
+    task: async (ctx, task) => {
         const { config: { docker }, ports } = ctx;
+        const { mysql } = docker.getContainers(ports);
 
+        return task.newListr(
+            execCommandTask(`docker exec ${mysql.name} bash -c 'mysql -uroot -p${mysql.env.MYSQL_ROOT_PASSWORD} -e "SET GLOBAL log_bin_trust_function_creators = 1;"'`)
+        );
+    }
+});
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const deleteDatabaseBeforeImportingDumpPrompt = () => ({
+    title: 'Deleting magento database before importing dump',
+    task: async (ctx, task) => {
+        const deleteDatabaseMagentoChoice = await task.prompt({
+            type: 'Select',
+            message: `Before importing database dump, would you like to delete existing database?
+
+It is possible that dump might interfere with existing data in database.
+
+Note that you will lose your existing database!`,
+            choices: [
+                {
+                    name: 'delete',
+                    message: 'YES I AM SURE I WANT TO DELETE magento DATABASE!'
+                },
+                {
+                    name: 'skip',
+                    message: 'Skip this step'
+                }
+            ]
+        });
+
+        if (deleteDatabaseMagentoChoice === 'delete') {
+            await ctx.mysqlConnection.query('DROP DATABASE IF EXISTS magento;');
+            await ctx.mysqlConnection.query('CREATE DATABASE magento;');
+            return;
+        }
+        task.skip();
+    }
+});
+
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const executeImportDumpSQL = () => ({
+    task: async (ctx, task) => {
+        const { config: { docker }, ports } = ctx;
         const { mysql } = docker.getContainers(ports);
 
+        const userCredentialsForMySQLCLI = await task.prompt({
+            type: 'Select',
+            message: 'Which user do you want to use to import db in MySQL client?',
+            choices: [
+                {
+                    name: `--user=root --password=${mysql.env.MYSQL_ROOT_PASSWORD}`,
+                    message: `root (${logger.style.command('Probably safest option')})`
+                },
+                {
+                    name: `--user=${mysql.env.MYSQL_USER} --password=${mysql.env.MYSQL_PASSWORD}`,
+                    message: `${mysql.env.MYSQL_USER}`
+                }
+            ]
+        });
+
+        const importCommand = `docker exec ${mysql.name} bash -c "mysql ${userCredentialsForMySQLCLI} magento < ./dump.sql"`;
+
         const startImportTime = Date.now();
         const tickInterval = setInterval(() => {
             task.title = `Importing Database Dump To MySQL, ${Math.floor((Date.now() - startImportTime) / 1000)}s in progress...`;
@@ -26,18 +103,7 @@ const importDumpToMySQL = () => ({
 
         try {
             await execAsyncSpawn(
-                `docker cp ${ctx.importDb} ${mysql.name}:/dump.sql`
-            );
-
-            await execAsyncSpawn(
-                `docker exec ${mysql.name} bash -c 'mysql -uroot -p${mysql.env.MYSQL_ROOT_PASSWORD} -e "SET GLOBAL log_bin_trust_function_creators = 1;"'`
-            );
-
-            /**
-             * Using `mysql` instead of `mysqlimport` because `mysqlimport` has permission issues during import
-             */
-            await execAsyncSpawn(
-                `docker exec ${mysql.name} bash -c "mysql -umagento -pmagento magento < ./dump.sql"`,
+                importCommand,
                 {
                     callback: (t) => {
                         task.output = t;
@@ -45,24 +111,94 @@ const importDumpToMySQL = () => ({
                 }
             );
         } catch (e) {
-            if (e.includes('Unknown collation: \'utf8mb4_0900_ai_ci\'')) {
-                throw new KnownError(`Error happened during database dump import!
+            if (e.message.includes('Unknown collation: \'utf8mb4_0900_ai_ci\'')) {
+                const confirmFixingCollation = await task.prompt({
+                    type: 'Select',
+                    message: `We got the following error while trying to import ${logger.style.file('dump.sql')}!
+
+${e.message}
+
+To fix this error we suggest running the following commands:
+${logger.style.command('sed -i \'s/utf8mb4_0900_ai_ci/utf8mb4_general_ci/g\' dump.sql')}
+`,
+                    choices: [
+                        {
+                            name: 'yes',
+                            message: 'Yes, run the following commands, I reaaaalllyy want dump to work! (this will not edit original dump.sql)'
+                        },
+                        {
+                            name: 'no',
+                            message: 'Okay, I got it. Will try to fix myself'
+                        }
+                    ]
+                });
+
+                if (confirmFixingCollation === 'yes') {
+                    task.output = 'Running fix command...';
+                    await execAsyncSpawn(`docker exec ${mysql.name} bash -c "sed -i 's/utf8mb4_0900_ai_ci/utf8mb4_general_ci/g' dump.sql"`);
 
-${e}
+                    task.output = 'Trying to import dump again...';
+                    try {
+                        await execAsyncSpawn(
+                            importCommand,
+                            {
+                                callback: (t) => {
+                                    task.output = t;
+                                }
+                            }
+                        );
+
+                        return;
+                    } catch (e) {
+                        throw new KnownError(`Fixing ${logger.style.file('dump.sql')} collations did not helped, we got the following error:
+${e.message}`);
+                    }
+                } else {
+                    throw new KnownError(`Database dump import unsuccessful!
+
+${e.message}
 
 You can try replacing all occurrences of ${logger.style.misc('utf8mb4_0900_ai_ci')} with ${logger.style.misc('utf8mb4_general_ci')} in your ${logger.style.file(ctx.importDb)} file!`);
+                }
             }
 
             throw new UnknownError(`Unexpected error during dump import.\n\n${e}`);
+        } finally {
+            clearInterval(tickInterval);
         }
-
-        clearInterval(tickInterval);
-
-        task.title = 'Database imported!';
     },
     options: {
         bottomBar: 10
     }
 });
 
+/**
+ * @type {() => import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const importDumpToMySQL = () => ({
+    title: 'Importing Database Dump To MySQL',
+    task: async (ctx, task) => {
+        if (!await pathExists(ctx.importDb)) {
+            throw new KnownError(`Dump file at ${ctx.importDb} does not exist. Please provide correct relative path to the file`);
+        }
+
+        return task.newListr([
+            copyDatabaseDumpIntoContainer(),
+            deleteDatabaseBeforeImportingDumpPrompt(),
+            runSetGlobalLogBinTrustFunctionCreatorsCommand(),
+            executeImportDumpSQL(),
+            connectToMySQL(),
+            {
+                task: () => {
+                    task.title = 'Database imported!';
+                }
+            }
+        ], {
+            rendererOptions: {
+                collapse: false
+            }
+        });
+    }
+});
+
 module.exports = importDumpToMySQL;

package/lib/tasks/php/php-container.js

@@ -0,0 +1,108 @@
+/* eslint-disable no-use-before-define */
+const { deepmerge } = require('../../util/deepmerge');
+const { containerApi } = require('../docker/containers');
+
+/**
+ * @param {import('../../../typings/context').ListrContext} ctx
+ * @param {import('../../util/exec-async-command').ExecAsyncSpawnOptions<false> & { useXDebugContainer?: boolean }} [options]
+ * @param {string} command
+ */
+const runPHPContainerCommand = async (ctx, command, options = {}) => {
+    const { php } = ctx.config.docker.getContainers(ctx.ports);
+
+    const containers = await containerApi.ls({ formatToJSON: true, all: true });
+
+    if (containers.some((c) => c.Names === php.name)) {
+        return execPHPContainerCommand(ctx, command, options);
+    }
+
+    return containerApi.run(
+        deepmerge(
+            php,
+            {
+                detach: false,
+                rm: true
+            },
+            options.useXDebugContainer
+                ? {
+                    image: `${ php.image }.xdebug`
+                }
+                : {},
+            {
+                command
+            },
+        ),
+        options
+    );
+};
+
+/**
+ * @param {string} command
+ * @param {import('../../util/exec-async-command').ExecAsyncSpawnOptions<false> & { useXDebugContainer?: boolean }} [options]
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const runPHPContainerCommandTask = (command, options = {}) => ({
+    title: `Running command "${command}"`,
+    task: (ctx, task) => runPHPContainerCommand(ctx, command, {
+        callback: !ctx.verbose ? undefined : (t) => {
+            task.output = t;
+        },
+        ...options
+    }),
+    option: {
+        bottomBar: 10
+    }
+});
+
+/**
+ *
+ * @param {import('../../../typings/context').ListrContext} ctx
+ * @param {import('../../util/exec-async-command').ExecAsyncSpawnOptions<false> & { useXDebugContainer?: boolean }} [options]
+ * @param {string} command
+ */
+const execPHPContainerCommand = async (ctx, command, options = {}) => {
+    const { php } = ctx.config.docker.getContainers(ctx.ports);
+
+    const containers = await containerApi.ls({ formatToJSON: true, all: true });
+
+    if (!containers.some((c) => c.Names === php.name)) {
+        return runPHPContainerCommand(ctx, command, options);
+    }
+
+    return containerApi.exec(
+        command,
+        php.name,
+        deepmerge(
+            php,
+            options.env ? {
+                env: options.env
+            } : {}
+        ),
+        options
+    );
+};
+
+/**
+ * @param {string} command
+ * @param {import('../../util/exec-async-command').ExecAsyncSpawnOptions<false> & { useXDebugContainer?: boolean, title?: string }} [options]
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const execPHPContainerCommandTask = (command, options = {}) => ({
+    title: typeof options.title === 'string' && options.title === '' ? undefined : options.title || `Running command "${command}"`,
+    task: (ctx, task) => execPHPContainerCommand(ctx, command, {
+        callback: !ctx.verbose ? undefined : (t) => {
+            task.output = t;
+        },
+        ...options
+    }),
+    option: {
+        bottomBar: 10
+    }
+});
+
+module.exports = {
+    runPHPContainerCommand,
+    runPHPContainerCommandTask,
+    execPHPContainerCommand,
+    execPHPContainerCommandTask
+};