@scandipwa/magento-scripts 1.16.0 → 2.0.0-alpha.1

package/lib/tasks/magento/setup-magento/index.js

@@ -10,8 +10,8 @@ const setUrlRewrite = require('./set-url-rewrite');
 const increaseAdminSessionLifetime = require('./increase-admin-session-lifetime');
 const magentoTask = require('../../../util/magento-task');
 const urnHighlighter = require('./urn-highlighter');
-const waitingForVarnish = require('./waiting-for-varnish');
 const adjustFullPageCache = require('./adjust-full-page-cache');
+const updateEnvPHP = require('../../php/update-env-php');
 
 /**
  * @param {Object} [options]
@@ -33,6 +33,7 @@ const setupMagento = (options = {}) => ({
         return task.newListr([
             flushRedisConfig(),
             waitingForRedis(),
+            updateEnvPHP(),
             migrateDatabase(),
             {
                 title: 'Configuring Magento settings',
@@ -50,8 +51,7 @@ const setupMagento = (options = {}) => ({
             disable2fa(),
             urnHighlighter(),
             adjustFullPageCache(),
-            magentoTask('cache:flush'),
-            waitingForVarnish()
+            magentoTask('cache:flush')
         ], {
             concurrent: false,
             exitOnError: true,

package/lib/tasks/magento/setup-magento/install-magento.js

@@ -24,50 +24,58 @@ const installMagento = ({ isDbEmpty = false } = {}) => ({
                 magentoConfiguration
             },
             ports,
-            mysqlConnection
+            databaseConnection
         } = ctx;
 
-        const response = await mysqlConnection.query(
-            'select * from admin_user where username=\'admin\';'
+        const isLinux = ctx.platform === 'linux';
+        const isNativeLinux = isLinux && !ctx.isWsl;
+        const hostMachine = isNativeLinux ? '127.0.0.1' : 'host.docker.internal';
+
+        const [tableResponse] = await databaseConnection.query(
+            'SELECT * FROM information_schema.tables WHERE table_schema = \'magento\' AND table_name = \'admin_user\' LIMIT 1;'
         );
 
-        const usersWithUsernameAdmin = response && response.length > 0 && response[0];
-
-        if (usersWithUsernameAdmin && usersWithUsernameAdmin.length > 0) {
-            const confirmDeleteAdminUsers = await task.prompt({
-                type: 'Select',
-                message: `In order to install Magento in database you will need to delete admin user with username ${logger.style.command('admin')}`,
-                choices: [
-                    {
-                        name: 'delete-all',
-                        message: `Delete all admin users (${logger.style.code('Recommended')})`
-                    },
-                    {
-                        name: 'delete-only-admin',
-                        message: `Delete only admin user with ${logger.style.command('admin')} username`
-                    }
-                ]
-            });
+        if (tableResponse.length > 0) {
+            const response = await databaseConnection.query(
+                'select * from admin_user where username=\'admin\';'
+            );
+
+            const usersWithUsernameAdmin = response && response.length > 0 && response[0];
+
+            if (usersWithUsernameAdmin && usersWithUsernameAdmin.length > 0) {
+                const confirmDeleteAdminUsers = await task.prompt({
+                    type: 'Select',
+                    message: `In order to install Magento in database you will need to delete admin user with username ${logger.style.command('admin')}`,
+                    choices: [
+                        {
+                            name: 'delete-all',
+                            message: `Delete all admin users (${logger.style.code('Recommended')})`
+                        },
+                        {
+                            name: 'delete-only-admin',
+                            message: `Delete only admin user with ${logger.style.command('admin')} username`
+                        }
+                    ]
+                });
 
-            await mysqlConnection.query('SET FOREIGN_KEY_CHECKS = 0;');
+                await databaseConnection.query('SET FOREIGN_KEY_CHECKS = 0;');
 
-            if (confirmDeleteAdminUsers === 'delete-all') {
-                await mysqlConnection.query(`
+                if (confirmDeleteAdminUsers === 'delete-all') {
+                    await databaseConnection.query(`
                     TRUNCATE TABLE admin_user;
                 `);
-            } else {
-                await mysqlConnection.query(`
+                } else {
+                    await databaseConnection.query(`
                     DELETE FROM admin_user WHERE username='admin';
                 `);
-            }
+                }
 
-            await mysqlConnection.query('SET FOREIGN_KEY_CHECKS = 1;');
+                await databaseConnection.query('SET FOREIGN_KEY_CHECKS = 1;');
+            }
         }
 
-        const { mysql: { env } } = docker.getContainers(ports);
-        const envPhpData = await envPhpToJson(process.cwd(), {
-            magentoVersion: ctx.magentoVersion
-        });
+        const { mariadb: { env } } = docker.getContainers(ports);
+        const envPhpData = await envPhpToJson(ctx);
 
         const envPhpHaveEncryptionKey = envPhpData && envPhpData.crypt && envPhpData.crypt.key && envPhpData.crypt.key;
 
@@ -89,7 +97,7 @@ const installMagento = ({ isDbEmpty = false } = {}) => ({
 
         const elasticsearchConfiguration = ` \
 --search-engine='elasticsearch7' \
---elasticsearch-host='127.0.0.1' \
+--elasticsearch-host='${ hostMachine }' \
 --elasticsearch-port='${ ports.elasticsearch }'`;
 
         /**
@@ -108,25 +116,24 @@ const installMagento = ({ isDbEmpty = false } = {}) => ({
                 ${ !isMagento23 ? elasticsearchConfiguration : '' } \
                 ${encryptionKeyOption || ''} \
                 --session-save=redis \
-                --session-save-redis-host='127.0.0.1' \
+                --session-save-redis-host='${ hostMachine }' \
                 --session-save-redis-port='${ ports.redis }' \
                 --session-save-redis-log-level='3' \
                 --session-save-redis-max-concurrency='30' \
                 --session-save-redis-db='1' \
                 --session-save-redis-disable-locking='1' \
                 --cache-backend='redis' \
-                --cache-backend-redis-server='127.0.0.1' \
+                --cache-backend-redis-server='${ hostMachine }' \
                 --cache-backend-redis-port='${ ports.redis }' \
                 --cache-backend-redis-db='0't \
-                --db-host='127.0.0.1:${ ports.mysql }' \
-                --db-name='${ env.MYSQL_DATABASE }' \
-                --db-user='${ env.MYSQL_USER }' \
-                --db-password='${ env.MYSQL_PASSWORD }' \
+                --db-host='${ hostMachine }:${ ports.mariadb }' \
+                --db-name='${ env.MARIADB_DATABASE }' \
+                --db-user='${ env.MARIADB_USER }' \
+                --db-password='${ env.MARIADB_PASSWORD }' \
                 --backend-frontname='${ magentoConfiguration.adminuri }' \
                 --no-interaction`;
 
-                await runMagentoCommand(command, {
-                    magentoVersion,
+                await runMagentoCommand(ctx, command, {
                     throwNonZeroCode: true,
                     callback: !ctx.verbose ? undefined : (t) => {
                         task.output = t;

package/lib/tasks/magento/setup-magento/migrate-database.js

@@ -8,6 +8,7 @@ const upgradeMagento = require('./upgrade-magento');
 const varnishConfigSetup = require('./varnish-config');
 const pathExists = require('../../../util/path-exists');
 const updateEnvPHP = require('../../php/update-env-php');
+const UnknownError = require('../../../errors/unknown-error');
 
 /**
  * @param {Object} [options]
@@ -17,12 +18,9 @@ const updateEnvPHP = require('../../php/update-env-php');
 const migrateDatabase = (options = {}) => ({
     title: 'Migrating database',
     task: async (ctx, task) => {
-        const {
-            magentoVersion,
-            mysqlConnection
-        } = ctx;
+        const { databaseConnection } = ctx;
 
-        const [[{ tableCount }]] = await mysqlConnection.query(`
+        const [[{ tableCount }]] = await databaseConnection.query(`
             SELECT count(*) AS tableCount
             FROM INFORMATION_SCHEMA.TABLES
             WHERE TABLE_SCHEMA = 'magento';
@@ -35,9 +33,9 @@ const migrateDatabase = (options = {}) => ({
         ) {
             if (options.onlyInstallMagento) {
                 ctx.isSetupUpgradeNeeded = false;
-                return task.newListr(
+                return task.newListr([
                     installMagento({ isDbEmpty: true })
-                );
+                ]);
             }
 
             return task.newListr([
@@ -57,8 +55,7 @@ const migrateDatabase = (options = {}) => ({
             });
         }
 
-        const { code } = await runMagentoCommand('setup:db:status', {
-            magentoVersion,
+        const { code, result } = await runMagentoCommand(ctx, 'setup:db:status', {
             throwNonZeroCode: false
         });
 
@@ -67,7 +64,6 @@ const migrateDatabase = (options = {}) => ({
             ctx.isSetupUpgradeNeeded = false;
             // no setup is needed, but still to be sure configure ES
             return task.newListr([
-                updateEnvPHP(),
                 varnishConfigSetup(),
                 configureElasticsearch()
             ], {
@@ -107,7 +103,6 @@ const migrateDatabase = (options = {}) => ({
         }
         case 2: {
             return task.newListr([
-                updateEnvPHP(),
                 varnishConfigSetup(),
                 configureElasticsearch(),
                 upgradeMagento()
@@ -121,9 +116,7 @@ const migrateDatabase = (options = {}) => ({
             });
         }
         default: {
-        // TODO: handle these statuses ?
-            task.title = 'Migrating database failed: manual action is required!';
-            break;
+            throw new UnknownError(`Migrating database failed: manual action is required!\n\n${result}`);
         }
         }
     },

package/lib/tasks/magento/setup-magento/set-base-url.js

@@ -14,7 +14,7 @@ module.exports = () => ({
                     ssl
                 }
             },
-            mysqlConnection
+            databaseConnection
         } = ctx;
         const isNgrok = host.endsWith('ngrok.io');
         const enableSecureFrontend = ssl.enabled ? '1' : '0';
@@ -29,6 +29,6 @@ module.exports = () => ({
             { path: 'web/secure/use_in_frontend', value: enableSecureFrontend },
             { path: 'web/secure/use_in_adminhtml', value: enableSecureFrontend },
             { path: 'web/cookie/cookie_domain', value: null }
-        ], { mysqlConnection, task });
+        ], { databaseConnection, task });
     }
 });

package/lib/tasks/magento/setup-magento/set-deployment-mode.js

@@ -6,10 +6,10 @@ const runMagentoCommand = require('../../../util/run-magento');
  */
 module.exports = () => ({
     title: 'Switching Magento mode',
-    task: async ({ magentoVersion, config: { magentoConfiguration: { mode } } }, task) => {
-        const { result } = await runMagentoCommand('deploy:mode:show', {
-            throwNonZeroCode: false,
-            magentoVersion
+    task: async (ctx, task) => {
+        const { config: { magentoConfiguration: { mode } } } = ctx;
+        const { result } = await runMagentoCommand(ctx, 'deploy:mode:show', {
+            throwNonZeroCode: false
         });
 
         if (result.includes(mode)) {

package/lib/tasks/magento/setup-magento/set-url-rewrite.js

@@ -5,13 +5,13 @@ const { updateTableValues } = require('../../../util/database');
  */
 const setUrlRewrite = () => ({
     title: 'Setting up url-rewrites',
-    task: async ({ mysqlConnection }, task) => {
+    task: async ({ databaseConnection }, task) => {
         await updateTableValues('core_config_data', [
             {
                 path: 'web/seo/use_rewrites',
                 value: '1'
             }
-        ], { mysqlConnection, task });
+        ], { databaseConnection, task });
     }
 });
 

package/lib/tasks/magento/setup-magento/varnish-config.js

@@ -6,7 +6,7 @@ const getIsWsl = require('../../../util/is-wsl');
  * @returns {import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
  */
 const varnishConfigSetup = () => ({
-    title: 'Varnish setup',
+    title: 'Varnish Database setup',
     task: async (ctx, task) => {
         const {
             config: {
@@ -18,18 +18,20 @@ const varnishConfigSetup = () => ({
                     }
                 }
             },
-            mysqlConnection,
-            ports
+            databaseConnection,
+            ports,
+            debug
         } = ctx;
 
         const isLinux = os.platform() === 'linux';
         const isWsl = await getIsWsl();
+        const isNativeLinux = isLinux && !isWsl;
 
-        if (varnishEnabled) {
+        if (!debug && varnishEnabled) {
             await updateTableValues('core_config_data', [
                 {
                     path: 'system/full_page_cache/varnish/backend_host',
-                    value: 'localhost'
+                    value: !isNativeLinux ? 'host.docker.internal' : 'localhost'
                 },
                 {
                     path: 'system/full_page_cache/varnish/backend_port',
@@ -37,16 +39,16 @@ const varnishConfigSetup = () => ({
                 },
                 {
                     path: 'system/full_page_cache/varnish/access_list',
-                    value: (!isLinux || isWsl) ? 'host.docker.internal,localhost' : 'localhost'
+                    value: !isNativeLinux ? 'host.docker.internal,localhost' : 'localhost'
                 },
                 {
                     path: 'system/full_page_cache/caching_application',
                     value: '2'
                 }
-            ], { mysqlConnection, task });
+            ], { databaseConnection, task });
         } else {
             // delete varnish configuration if exists
-            await mysqlConnection.query(`
+            await databaseConnection.query(`
                 DELETE FROM core_config_data WHERE path LIKE '%varnish%';
             `);
 
@@ -57,7 +59,7 @@ const varnishConfigSetup = () => ({
                     path: 'system/full_page_cache/caching_application',
                     value: '0'
                 }
-            ], { mysqlConnection, task });
+            ], { databaseConnection, task });
         }
     },
     options: {

package/lib/tasks/magento/setup-magento/waiting-for-varnish.js

@@ -1,14 +1,48 @@
+const logger = require('@scandipwa/scandipwa-dev-utils/logger');
+const semver = require('semver');
+const fs = require('fs');
+const path = require('path');
 const { request } = require('smol-request');
 const KnownError = require('../../../errors/known-error');
+const { execAsyncSpawn } = require('../../../util/exec-async-command');
+const { NginxParser } = require('../../../util/nginx-logs-parser');
 const sleep = require('../../../util/sleep');
 
+const pathToWorkingHealthCheckPhp = path.join(__dirname, '..', '..', 'php', 'working_health_check.php');
+const pathToProjectsHealthCheckPhp = path.join(process.cwd(), 'pub', 'health_check.php');
+const pathToHealthCheckBackupPhp = path.join(process.cwd(), 'pub', 'health_check-super-backup-file.php');
+
+/**
+ * @param {import('../../../../typings/context').ListrContext} ctx
+ */
+const getIsHealthCheckRequestBroken = async (ctx) => {
+    const { nginx } = ctx.config.docker.getContainers(ctx.ports);
+
+    const nginxLogs = await execAsyncSpawn(`docker logs ${nginx.name}`);
+
+    const healthCheckRequests = nginxLogs
+        .split('\n')
+        .filter((line) => line.includes('"GET /health_check.php HTTP/1.1"'));
+
+    // eslint-disable-next-line max-len
+    const parser = new NginxParser('$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"');
+
+    const parsedRequests = healthCheckRequests.map((line) => parser.parseLine(line));
+
+    return parsedRequests.every((parsedRequest) => parsedRequest.status !== '200');
+};
+
 /**
  * @returns {import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
  */
 const waitingForVarnish = () => ({
     title: 'Waiting for Varnish to return code 200',
-    skip: (ctx) => !ctx.config.overridenConfiguration.configuration.varnish.enabled,
-    task: async (ctx) => {
+    skip: (ctx) => ctx.debug || !ctx.config.overridenConfiguration.configuration.varnish.enabled,
+    task: async (ctx, task) => {
+        const pureMagentoVersion = ctx.magentoVersion.match(/^([0-9]+\.[0-9]+\.[0-9]+)/)[1];
+
+        const isMagento23 = semver.satisfies(pureMagentoVersion, '<2.4');
+
         let tries = 0;
         while (tries < 10) {
             try {
@@ -28,6 +62,94 @@ const waitingForVarnish = () => ({
             }
         }
 
+        if (tries === 10 && isMagento23) {
+            const isHealthCheckRequestBroken = await getIsHealthCheckRequestBroken(ctx);
+            if (isHealthCheckRequestBroken) {
+                const confirm = await task.prompt({
+                    type: 'Select',
+                    message: `We detected that your Magento instance is experiencing problems with Varnish server.
+Health Check request is returning status 500.
+
+Do you want to try resolving this issue by replacing ${logger.style.file('./pub/health_check.php')} file content with content from newer Magento version (2.4.4)?`,
+                    choices: [
+                        {
+                            name: 'yes',
+                            message: 'Okay, let\'s try that'
+                        },
+                        {
+                            name: 'no',
+                            message: 'I will try to fix this myself, thanks for the info!'
+                        }
+                    ]
+                });
+
+                if (confirm === 'yes') {
+                    await fs.promises.writeFile(
+                        pathToHealthCheckBackupPhp,
+                        await fs.promises.readFile(pathToProjectsHealthCheckPhp, 'utf-8'),
+                        'utf-8'
+                    );
+
+                    task.output = `Backup made! Available at ${logger.style.file(pathToHealthCheckBackupPhp)}`;
+
+                    await fs.promises.writeFile(
+                        pathToProjectsHealthCheckPhp,
+                        await fs.promises.readFile(pathToWorkingHealthCheckPhp, 'utf-8'),
+                        'utf-8'
+                    );
+
+                    task.output = `${logger.style.file('./pub/health_check.php')} content is replaced, waiting for nginx logs...`;
+
+                    let connectionFixed = false;
+
+                    for (let i = 0; i < 10; i++) {
+                        try {
+                            const response = await request(`http://localhost:${ctx.ports.sslTerminator}/`, {
+                                responseType: 'headers'
+                            });
+
+                            if (response.status !== 200) {
+                                await sleep(2000);
+                            } else {
+                                connectionFixed = true;
+                                break;
+                            }
+                        } catch (e) {
+                            await sleep(200);
+                        }
+                    }
+
+                    if (connectionFixed) {
+                        task.output = 'Connection looks good!';
+                        const secondConfirm = await task.prompt({
+                            type: 'Select',
+                            message: `Okay, looks like it helped!
+Do you want to keep backed up ${logger.style.file('health_check.php')} file?`,
+                            choices: [
+                                {
+                                    name: 'no',
+                                    message: 'I don\'t think I will need it anymore, thanks'
+                                },
+                                {
+                                    name: 'yes',
+                                    message: 'I will keep it, maybe it will be useful (X to doubt)'
+                                }
+                            ]
+                        });
+
+                        if (secondConfirm === 'no') {
+                            await fs.promises.rm(pathToHealthCheckBackupPhp);
+                        }
+                    }
+
+                    return;
+                }
+
+                task.skip('User choose to fix issue himself');
+                return;
+            }
+        }
+
         if (tries === 10) {
             throw new KnownError(`After 20 seconds website is still responding with non-200 code, which might indicate issue with setup.
 Or Varnish is still loading...
@@ -35,7 +157,10 @@ Or Varnish is still loading...
 Please check the logs!`);
         }
     },
-    exitOnError: false
+    exitOnError: false,
+    options: {
+        bottomBar: 10
+    }
 });
 
 module.exports = waitingForVarnish;

package/lib/tasks/php/php-container.js

@@ -0,0 +1,108 @@
+/* eslint-disable no-use-before-define */
+const { deepmerge } = require('../../util/deepmerge');
+const { containerApi } = require('../docker/containers');
+
+/**
+ * @param {import('../../../typings/context').ListrContext} ctx
+ * @param {import('../../util/exec-async-command').ExecAsyncSpawnOptions<false> & { useXDebugContainer?: boolean }} [options]
+ * @param {string} command
+ */
+const runPHPContainerCommand = async (ctx, command, options = {}) => {
+    const { php } = ctx.config.docker.getContainers(ctx.ports);
+
+    const containers = await containerApi.ls({ formatToJSON: true, all: true });
+
+    if (containers.some((c) => c.Names === php.name)) {
+        return execPHPContainerCommand(ctx, command, options);
+    }
+
+    return containerApi.run(
+        deepmerge(
+            php,
+            {
+                detach: false,
+                rm: true
+            },
+            options.useXDebugContainer
+                ? {
+                    image: `${ php.image }.xdebug`
+                }
+                : {},
+            {
+                command
+            },
+        ),
+        options
+    );
+};
+
+/**
+ * @param {string} command
+ * @param {import('../../util/exec-async-command').ExecAsyncSpawnOptions<false> & { useXDebugContainer?: boolean }} [options]
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const runPHPContainerCommandTask = (command, options = {}) => ({
+    title: `Running command "${command}"`,
+    task: (ctx, task) => runPHPContainerCommand(ctx, command, {
+        callback: !ctx.verbose ? undefined : (t) => {
+            task.output = t;
+        },
+        ...options
+    }),
+    option: {
+        bottomBar: 10
+    }
+});
+
+/**
+ *
+ * @param {import('../../../typings/context').ListrContext} ctx
+ * @param {import('../../util/exec-async-command').ExecAsyncSpawnOptions<false> & { useXDebugContainer?: boolean }} [options]
+ * @param {string} command
+ */
+const execPHPContainerCommand = async (ctx, command, options = {}) => {
+    const { php } = ctx.config.docker.getContainers(ctx.ports);
+
+    const containers = await containerApi.ls({ formatToJSON: true, all: true });
+
+    if (!containers.some((c) => c.Names === php.name)) {
+        return runPHPContainerCommand(ctx, command, options);
+    }
+
+    return containerApi.exec(
+        command,
+        php.name,
+        deepmerge(
+            php,
+            options.env ? {
+                env: options.env
+            } : {}
+        ),
+        options
+    );
+};
+
+/**
+ * @param {string} command
+ * @param {import('../../util/exec-async-command').ExecAsyncSpawnOptions<false> & { useXDebugContainer?: boolean, title?: string }} [options]
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const execPHPContainerCommandTask = (command, options = {}) => ({
+    title: typeof options.title === 'string' && options.title === '' ? undefined : options.title || `Running command "${command}"`,
+    task: (ctx, task) => execPHPContainerCommand(ctx, command, {
+        callback: !ctx.verbose ? undefined : (t) => {
+            task.output = t;
+        },
+        ...options
+    }),
+    option: {
+        bottomBar: 10
+    }
+});
+
+module.exports = {
+    runPHPContainerCommand,
+    runPHPContainerCommandTask,
+    execPHPContainerCommand,
+    execPHPContainerCommandTask
+};