@scandipwa/magento-scripts 2.4.0-alpha.1 → 2.4.0-alpha.2

package/lib/config/docker.js

@@ -10,6 +10,27 @@ const logger = require('@scandipwa/scandipwa-dev-utils/logger')
 const defaultMagentoUser = require('../tasks/database/default-magento-user')
 const defaultOsEnv = require('./services/opensearch/default-os-env')
 
+/**
+ * @param {Partial<Record<'rw' | 'ro' | 'cached', boolean>>} directives
+ */
+const volumeDirectives = (directives) => {
+    const directivesResult = Object.entries(directives)
+        .filter(([name, value]) => value === true)
+        .map(([name]) => name)
+        .join(',')
+
+    return directivesResult ? `:${directivesResult}` : ''
+}
+
+/**
+ * @param {{source: string, target: string, rw?: boolean, ro?: boolean, cached?: boolean}} options
+ */
+const containerVolume = (options) => {
+    const { source, target, ...directives } = options
+
+    return `${source}:${target}${volumeDirectives(directives)}`
+}
+
 /**
  * @param {import('../../typings/context').ListrContext} ctx
  * @param {import('../../typings/context').ListrContext['config']['overridenConfiguration']} overridenConfiguration
@@ -68,8 +89,17 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
 
     const { isDockerDesktop } = ctx
 
-    const volumeDirective = isDockerDesktop ? ':cached' : ''
-
+    if (isDockerDesktop) {
+        volumes.php = {
+            name: `${prefix}_project-data`,
+            driver: 'local',
+            opt: {
+                type: 'none',
+                device: path.join(magentoDir),
+                o: 'bind'
+            }
+        }
+    }
     /**
      * @param {Record<string, number>} ports
      */
@@ -85,7 +115,7 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
             : {}
 
         /**
-         * @type {Record<string, import('../tasks/docker/containers/container-api').ContainerRunOptions & { _?: string, forwardedPorts?: string[], remoteImages?: string[], connectCommand?: string[], description?: string, pullImage?: boolean }>}
+         * @type {Record<string, import('../tasks/docker/containers/container-api').ContainerRunOptions & { _?: string, forwardedPorts?: string[], remoteImages?: string[], connectCommand?: string[], description?: string, pullImage?: boolean, dependsOn?: string[] }>}
          */
         const dockerConfig = {
             php: {
@@ -104,10 +134,29 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                 ],
                 network: isDockerDesktop ? network.name : 'host',
                 mountVolumes: [
-                    `${magentoDir}:${containerMagentoDir}${volumeDirective}`,
-                    `${volumes.composer_cache.name}:/composer/home/cache`,
-                    `${php.iniPath}:/usr/local/etc/php/php.ini${volumeDirective}`,
-                    `${php.fpmConfPath}:/usr/local/etc/php-fpm.d/zz-docker.conf${volumeDirective}`
+                    containerVolume({
+                        source: isDockerDesktop ? volumes.php.name : magentoDir,
+                        target: containerMagentoDir,
+                        rw: true,
+                        cached: isDockerDesktop
+                    }),
+                    containerVolume({
+                        source: volumes.composer_cache.name,
+                        target: '/composer/home/cache',
+                        rw: true
+                    }),
+                    containerVolume({
+                        source: php.iniPath,
+                        target: '/usr/local/etc/php/php.ini',
+                        ro: true,
+                        cached: isDockerDesktop
+                    }),
+                    containerVolume({
+                        source: php.fpmConfPath,
+                        target: '/usr/local/etc/php-fpm.d/zz-docker.conf',
+                        ro: true,
+                        cached: isDockerDesktop
+                    })
                 ],
                 env: deepmerge(composerAuthEnv, php.env || {}),
                 restart: 'on-failure:5',
@@ -115,6 +164,7 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                 remoteImages: [configuration.php.baseImage],
                 name: `${prefix}_php`,
                 connectCommand: ['/bin/sh'],
+                dependsOn: ['mariadb', 'redis', 'elasticsearch'],
                 user:
                     (ctx.platform === 'linux' && isDockerDesktop) ||
                     !isDockerDesktop
@@ -137,11 +187,35 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                 ],
                 network: isDockerDesktop ? network.name : 'host',
                 mountVolumes: [
-                    `${magentoDir}:${containerMagentoDir}${volumeDirective}`,
-                    `${volumes.composer_cache.name}:/composer/home/cache`,
-                    `${php.iniPath}:/usr/local/etc/php/php.ini${volumeDirective}`,
-                    `${php.debugFpmConfPath}:/usr/local/etc/php-fpm.d/zz-docker.conf${volumeDirective}`,
-                    `${php.debugIniPath}:/usr/local/etc/php/conf.d/00-xdebug.ini${volumeDirective}`
+                    containerVolume({
+                        source: isDockerDesktop ? volumes.php.name : magentoDir,
+                        target: containerMagentoDir,
+                        rw: true,
+                        cached: isDockerDesktop
+                    }),
+                    containerVolume({
+                        source: volumes.composer_cache.name,
+                        target: '/composer/home/cache',
+                        rw: true
+                    }),
+                    containerVolume({
+                        source: php.iniPath,
+                        target: '/usr/local/etc/php/php.ini',
+                        ro: true,
+                        cached: isDockerDesktop
+                    }),
+                    containerVolume({
+                        source: php.debugFpmConfPath,
+                        target: '/usr/local/etc/php-fpm.d/zz-docker.conf',
+                        ro: true,
+                        cached: isDockerDesktop
+                    }),
+                    containerVolume({
+                        source: php.debugIniPath,
+                        target: '/usr/local/etc/php/conf.d/00-xdebug.ini',
+                        ro: true,
+                        cached: isDockerDesktop
+                    })
                 ],
                 env: deepmerge(composerAuthEnv, php.env || {}),
                 restart: 'on-failure:5',
@@ -149,6 +223,7 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                 pullImage: false,
                 name: `${prefix}_php_with_xdebug`,
                 connectCommand: ['/bin/sh'],
+                dependsOn: ['mariadb', 'redis', 'elasticsearch'],
                 user:
                     (ctx.platform === 'linux' && isDockerDesktop) ||
                     !isDockerDesktop
@@ -172,20 +247,23 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                 healthCheck: {
                     cmd: 'service nginx status'
                 },
-                /**
-                 * Mount volumes directly on linux
-                 */
                 mountVolumes: [
-                    `${path.join(
-                        cacheDir,
-                        'ssl-terminator',
-                        'conf.d'
-                    )}:/etc/nginx/conf.d${volumeDirective}`,
-                    `${path.join(
-                        cacheDir,
-                        'ssl-terminator',
-                        'fastcgi_params'
-                    )}:/etc/nginx/fastcgi_params${volumeDirective}`
+                    containerVolume({
+                        source: path.join(cacheDir, 'ssl-terminator', 'conf.d'),
+                        target: '/etc/nginx/conf.d',
+                        ro: true,
+                        cached: isDockerDesktop
+                    }),
+                    containerVolume({
+                        source: path.join(
+                            cacheDir,
+                            'ssl-terminator',
+                            'fastcgi_params'
+                        ),
+                        target: '/etc/nginx/fastcgi_params',
+                        ro: true,
+                        cached: isDockerDesktop
+                    })
                 ],
                 restart: 'on-failure:5',
                 network: isDockerDesktop ? network.name : 'host',
@@ -193,7 +271,8 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                     nginx.version ? `nginx:${nginx.version}` : nginx.image
                 }`,
                 name: `${prefix}_ssl-terminator`,
-                command: "nginx -g 'daemon off;'"
+                command: "nginx -g 'daemon off;'",
+                dependsOn: ['nginx']
             },
             nginx: {
                 _: 'Nginx',
@@ -212,21 +291,29 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                 healthCheck: {
                     cmd: 'service nginx status'
                 },
-                /**
-                 * Mount volumes directly on linux
-                 */
                 mountVolumes: [
-                    `${path.join(
-                        cacheDir,
-                        'nginx',
-                        'conf.d'
-                    )}:/etc/nginx/conf.d${volumeDirective}`,
-                    `${magentoDir}:${containerMagentoDir}${volumeDirective}`,
-                    `${path.join(
-                        cacheDir,
-                        'ssl-terminator',
-                        'fastcgi_params'
-                    )}:/etc/nginx/fastcgi_params${volumeDirective}`
+                    containerVolume({
+                        source: path.join(cacheDir, 'nginx', 'conf.d'),
+                        target: '/etc/nginx/conf.d',
+                        ro: true,
+                        cached: isDockerDesktop
+                    }),
+                    containerVolume({
+                        source: isDockerDesktop ? volumes.php.name : magentoDir,
+                        target: containerMagentoDir,
+                        rw: true,
+                        cached: isDockerDesktop
+                    }),
+                    containerVolume({
+                        source: path.join(
+                            cacheDir,
+                            'ssl-terminator',
+                            'fastcgi_params'
+                        ),
+                        target: '/etc/nginx/fastcgi_params',
+                        ro: true,
+                        cached: isDockerDesktop
+                    })
                 ],
                 restart: 'on-failure:5',
                 network: isDockerDesktop ? network.name : 'host',
@@ -234,7 +321,8 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                     nginx.version ? `nginx:${nginx.version}` : nginx.image
                 }`,
                 name: `${prefix}_nginx`,
-                command: "nginx -g 'daemon off;'"
+                command: "nginx -g 'daemon off;'",
+                dependsOn: ['php', 'phpWithXdebug']
             },
             redis: {
                 _: 'Redis',
@@ -254,16 +342,21 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
             mariadb: {
                 _: 'MariaDB',
                 healthCheck: {
-                    cmd: 'mariadb-admin ping --silent'
+                    cmd: `${mariadb.binAdminFileName} ping --silent`
                 },
                 ports: [`127.0.0.1:${ports.mariadb}:3306`],
                 forwardedPorts: [`127.0.0.1:${ports.mariadb}:3306`],
                 mountVolumes: [
-                    `${volumes.mariadb.name}:/var/lib/mysql`,
-                    `${path.join(
-                        baseConfig.cacheDir,
-                        'mariadb.cnf'
-                    )}:/etc/mysql/my.cnf${volumeDirective}`
+                    containerVolume({
+                        source: volumes.mariadb.name,
+                        target: '/var/lib/mysql'
+                    }),
+                    containerVolume({
+                        source: path.join(baseConfig.cacheDir, 'mariadb.cnf'),
+                        target: '/etc/mysql/my.cnf',
+                        ro: true,
+                        cached: isDockerDesktop
+                    })
                 ],
                 env: {
                     MARIADB_ROOT_PASSWORD: 'scandipwa'
@@ -297,10 +390,16 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                 },
                 ports: [`127.0.0.1:${ports.elasticsearch}:9200`],
                 forwardedPorts: [`127.0.0.1:${ports.elasticsearch}:9200`],
-                mounts: [
+                mountVolumes: [
                     searchengine === 'elasticsearch'
-                        ? `source=${volumes.elasticsearch.name},target=/usr/share/elasticsearch/data`
-                        : `source=${volumes.opensearch.name},target=/usr/share/opensearch/data`
+                        ? containerVolume({
+                              source: volumes.elasticsearch.name,
+                              target: '/usr/share/elasticsearch/data'
+                          })
+                        : containerVolume({
+                              source: volumes.opensearch.name,
+                              target: '/usr/share/opensearch/data'
+                          })
                 ],
                 env:
                     searchengine === 'elasticsearch'
@@ -343,7 +442,12 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                           `127.0.0.1:${ports.maildevWeb}`,
                           `127.0.0.1:${ports.maildevSMTP}`
                       ],
-                mountVolumes: [`${volumes.maildev.name}:/tmp/maildev`],
+                mountVolumes: [
+                    containerVolume({
+                        source: volumes.maildev.name,
+                        target: '/tmp/maildev'
+                    })
+                ],
                 env: {
                     MAILDEV_SMTP_PORT: isDockerDesktop
                         ? '1025'
@@ -384,10 +488,11 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                 }`,
                 name: `${prefix}_varnish`,
                 mountVolumes: [
-                    `${path.join(
-                        cacheDir,
-                        'varnish'
-                    )}:/etc/varnish${volumeDirective}`
+                    containerVolume({
+                        source: path.join(cacheDir, 'varnish'),
+                        target: '/etc/varnish',
+                        ro: true
+                    })
                 ],
                 ports: isDockerDesktop
                     ? [
@@ -413,8 +518,11 @@ module.exports = async (ctx, overridenConfiguration, baseConfig) => {
                 tmpfs: ['/var/lib/varnish/varnishd:exec'],
                 description: `Varnish HealthCheck status: ${logger.style.command(
                     varnish.healthCheck ? 'enabled' : 'disabled'
-                )}`
+                )}`,
+                dependsOn: ['nginx']
             }
+
+            dockerConfig.sslTerminator.dependsOn.push('varnish')
         }
 
         if (newRelic.enabled) {

package/lib/config/port-config.js

@@ -98,18 +98,54 @@ const getPortsConfig = async (ports, options = {}) => {
     if (useNonOverlappingPorts) {
         p = p.concat(await getUsedByOtherCMAProjectsPorts())
     }
-    const availablePorts = Object.fromEntries(
-        await Promise.all(
-            Object.entries(mergedPorts).map(async ([name, port]) => {
-                const availablePort = await getPort(port, {
-                    portIgnoreList: p
-                })
-
-                return [name, availablePort]
-            })
-        )
+
+    /**
+     * @type {Record<string, string>}
+     */
+    const portsToCheck = Object.entries(mergedPorts).reduce(
+        (acc, [name, port]) => {
+            if (acc[port]) {
+                let i = 0
+                while (acc[port + i]) {
+                    i++
+                }
+
+                return {
+                    ...acc,
+                    [port + i]: name
+                }
+            } else {
+                return {
+                    ...acc,
+                    [port]: name
+                }
+            }
+        },
+        {}
     )
 
+    /**
+     * @type {Record<string, number>}
+     */
+    const availablePorts = {}
+
+    for (const [port, name] of Object.entries(portsToCheck)) {
+        const portInt = Number.parseInt(port)
+        const portIgnoreList = p.concat(
+            Object.keys(availablePorts).map((item) => Number.parseInt(item))
+        )
+
+        const getPortResult = await getPort(portInt, {
+            portIgnoreList
+        })
+
+        if (typeof getPortResult === 'number') {
+            availablePorts[name] = getPortResult
+        } else {
+            throw new Error(`No available port found for ${name} (${portInt})`)
+        }
+    }
+
     return availablePorts
 }
 

package/lib/config/services/elasticsearch/default-es-env.js

@@ -5,5 +5,5 @@ module.exports = {
     'bootstrap.memory_lock': true,
     'xpack.security.enabled': false,
     'discovery.type': 'single-node',
-    ES_JAVA_OPTS: '-Xms512m -Xmx512m'
+    ES_JAVA_OPTS: '-Xms2g -Xmx2g'
 }

package/lib/config/services/mariadb/versions/mariadb-10.2.js

@@ -3,7 +3,9 @@
  */
 const mariadb102 = () => ({
     image: 'mariadb:10.2',
-    useOptimizerSwitch: false
+    useOptimizerSwitch: false,
+    binFileName: 'mysql',
+    binAdminFileName: 'mysqladmin'
 })
 
 module.exports = mariadb102

package/lib/config/services/mariadb/versions/mariadb-10.3.js

@@ -3,7 +3,9 @@
  */
 const mariadb103 = () => ({
     image: 'mariadb:10.3',
-    useOptimizerSwitch: false
+    useOptimizerSwitch: false,
+    binFileName: 'mysql',
+    binAdminFileName: 'mysqladmin'
 })
 
 module.exports = mariadb103

package/lib/config/services/mariadb/versions/mariadb-10.4.js

@@ -3,7 +3,9 @@
  */
 const mariadb104 = () => ({
     image: 'mariadb:10.4',
-    useOptimizerSwitch: true
+    useOptimizerSwitch: true,
+    binFileName: 'mysql',
+    binAdminFileName: 'mysqladmin'
 })
 
 module.exports = mariadb104

package/lib/config/services/mariadb/versions/mariadb-10.6.js

@@ -3,7 +3,9 @@
  */
 const mariadb106 = () => ({
     image: 'mariadb:10.6',
-    useOptimizerSwitch: true
+    useOptimizerSwitch: true,
+    binFileName: 'mysql',
+    binAdminFileName: 'mysqladmin'
 })
 
 module.exports = mariadb106

package/lib/config/services/mariadb/versions/mariadb-11.4.js

@@ -3,7 +3,9 @@
  */
 const mariadb114 = () => ({
     image: 'mariadb:11.4',
-    useOptimizerSwitch: true
+    useOptimizerSwitch: true,
+    binFileName: 'mariadb',
+    binAdminFileName: 'mariadb-admin'
 })
 
 module.exports = mariadb114

package/lib/config/services/mariadb/versions/mariadb-11.6.js

@@ -3,7 +3,9 @@
  */
 const mariadb116 = () => ({
     image: 'mariadb:11.6',
-    useOptimizerSwitch: true
+    useOptimizerSwitch: true,
+    binFileName: 'mariadb',
+    binAdminFileName: 'mariadb-admin'
 })
 
 module.exports = mariadb116

package/lib/config/services/opensearch/default-os-env.js

@@ -4,7 +4,7 @@
 module.exports = {
     'bootstrap.memory_lock': true,
     'discovery.type': 'single-node',
-    OPENSEARCH_JAVA_OPTS: '-Xms512m -Xmx2048m',
+    OPENSEARCH_JAVA_OPTS: '-Xms2g -Xmx2g',
     DISABLE_SECURITY_PLUGIN: true,
     DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI: true
 }

package/lib/config/services/php/extensions/xdebug.js

@@ -7,6 +7,7 @@ module.exports = {
     command: ({
         version = ''
     }) => `apk add --no-cache --virtual .build-deps \\$PHPIZE_DEPS \
+&& apk add --no-cache linux-headers \
 && pecl install xdebug${version ? `-${version}` : ''} \
 && docker-php-ext-enable xdebug \
 && apk del -f .build-deps`,

package/lib/config/templates/nginx.template.conf

@@ -1,13 +1,13 @@
 upstream fastcgi_backend {
     # use tcp connection
-    server <%= it.hostMachine %>:<%= it.ports.fpm %>;
+    server <%= it.phpNetwork %>:<%= it.fpmPort %>;
 
     keepalive 16;
 }
 
 # Define available upstreams
 upstream fastcgi_backend_xdebug {
-    server <%= it.hostMachine %>:<%= it.ports.fpmXdebug %>;
+    server <%= it.phpWithXdebugNetwork %>:<%= it.fpmXdebugPort %>;
 
     keepalive 16;
 }

package/lib/config/templates/php-fpm.template.conf

@@ -4,7 +4,7 @@ log_level = debug
 
 [www]
 clear_env = no
-user = nobody
+user = www-data
 ;; Commented because if it is not set, php-fpm will use default group.
 ;; https://www.php.net/manual/en/install.fpm.configuration.php#group
 ; group = nobody

package/lib/config/templates/ssl-terminator.template.conf

@@ -4,7 +4,7 @@ set_real_ip_from 172.0.0.0/8;
 real_ip_recursive on;
 
 upstream app_backend {
-    server <%= it.hostMachine %>:<% if (!it.debug && it.config.configuration.varnish.enabled) { %><%= it.ports.varnish %><% } else { %><%= it.ports.app %><% } %>;
+    server <%= it.backendNetwork %>:<%= it.backendPort %>;
 
     keepalive 16;
 }

package/lib/tasks/database/create-magento-database.js

@@ -7,11 +7,12 @@ const { containerApi } = require('../docker/containers')
 const createMagentoDatabase = () => ({
     title: 'Creating Magento database',
     task: async (ctx, task) => {
+        const { configuration } = ctx.config.overridenConfiguration
         const { mariadb } = ctx.config.docker.getContainers()
         task.title = `Creating Magento database in ${mariadb._}`
 
         await containerApi.exec({
-            command: `mariadb -uroot -p${mariadb.env.MARIADB_ROOT_PASSWORD} -h 127.0.0.1 -e "CREATE DATABASE IF NOT EXISTS magento;"`,
+            command: `${configuration.mariadb.binFileName} -uroot -p${mariadb.env.MARIADB_ROOT_PASSWORD} -h 127.0.0.1 -e "CREATE DATABASE IF NOT EXISTS magento;"`,
             container: mariadb.name
         })
     }

package/lib/tasks/database/import-remote-db/ssh/index.js

@@ -1,5 +1,5 @@
 const os = require('os')
-const { NodeSSH } = require('node-ssh')
+const { NodeSSH } = require('node-ssh-no-cpu-features')
 const pathExists = require('../../../../util/path-exists')
 const regularSSHServer = require('./regular-server')
 const readymageSSH = require('./readymage')

package/lib/tasks/database/import-remote-db/ssh/readymage.js

@@ -4,7 +4,7 @@ const { execAsyncSpawn } = require('../../../../util/exec-async-command')
 const databaseDumpCommandWithOptions = require('./database-dump-command')
 
 /**
- * @returns {import('listr2').ListrTask<import('../../../../../typings/context').ListrContext & { ssh: import('node-ssh').NodeSSH }>}
+ * @returns {import('listr2').ListrTask<import('../../../../../typings/context').ListrContext & { ssh: import('node-ssh-no-cpu-features').NodeSSH }>}
  */
 const readymageSSH = () => ({
     task: async (ctx, task) => {

package/lib/tasks/database/import-remote-db/ssh/regular-server.js

@@ -4,7 +4,7 @@ const { execAsyncSpawn } = require('../../../../util/exec-async-command')
 const databaseDumpCommandWithOptions = require('./database-dump-command')
 const KnownError = require('../../../../errors/known-error')
 /**
- * @returns {import('listr2').ListrTask<import('../../../../../typings/context').ListrContext & { ssh: import('node-ssh').NodeSSH }>}
+ * @returns {import('listr2').ListrTask<import('../../../../../typings/context').ListrContext & { ssh: import('node-ssh-no-cpu-features').NodeSSH }>}
  */
 const regularSSHServer = () => ({
     task: async (ctx, task) => {

package/lib/tasks/docker/containers/container-api.d.ts

@@ -63,6 +63,11 @@ export interface ContainerExecOptions {
      * Allocate a pseudo-TTY
      */
     tty?: boolean
+
+    /**
+     * Keep STDIN open even if not attached
+     */
+    interactive?: boolean
 }
 
 export function exec<T>(

package/lib/tasks/docker/containers/container-api.js

@@ -103,7 +103,7 @@ const run = (options, execOptions = {}) =>
  * @param {import('./container-api').ContainerExecOptions} options
  */
 const execCommand = (options) => {
-    const { command, container, env, tty, user, workdir } = options
+    const { command, container, env, tty, user, workdir, interactive } = options
     const envArgs = !env
         ? ''
         : Object.entries(env)
@@ -112,6 +112,7 @@ const execCommand = (options) => {
     const ttyArg = tty ? '--tty' : ''
     const userArg = user ? `--user=${user}` : ''
     const workdirArg = workdir ? `--workdir=${workdir}` : ''
+    const interactiveArg = interactive ? '--interactive' : ''
 
     const dockerCommand = [
         'docker',
@@ -119,6 +120,7 @@ const execCommand = (options) => {
         'exec',
         envArgs,
         ttyArg,
+        interactiveArg,
         userArg,
         workdirArg,
         container,