@scandipwa/magento-scripts 2.2.3 → 2.3.1

package/lib/tasks/database/fix-db.js

@@ -1,4 +1,4 @@
-const configureElasticsearch = require('../magento/setup-magento/configure-elasticsearch')
+const configureElasticsearch = require('../magento/setup-magento/configure-searchengine')
 const deleteAdminUsers = require('../magento/setup-magento/delete-admin-users')
 const deleteCustomers = require('../magento/setup-magento/delete-customers')
 const deleteOrders = require('../magento/setup-magento/delete-orders')

package/lib/tasks/magento/setup-magento/configure-searchengine.js

@@ -0,0 +1,392 @@
+const semver = require('semver')
+const path = require('path')
+const {
+    updateTableValues,
+    isTableExists,
+    insertTableValues
+} = require('../../../util/database')
+const getJsonfileData = require('../../../util/get-jsonfile-data')
+const runComposerCommand = require('../../../util/run-composer')
+
+const magentoModuleElasticSearch8 = 'magento/module-elasticsearch-8'
+
+const searchEngineConfigurationInCoreConfigDataKeys = (
+    searchEngineEdition = 'elasticsearch7'
+) => ({
+    catalogSearchEngine: 'catalog/search/engine',
+    catalogSearchSearchEngineServerHostname: `catalog/search/${searchEngineEdition}_server_hostname`,
+    catalogSearchSearchEngineServerPort: `catalog/search/${searchEngineEdition}_server_port`,
+    catalogSearchSearchEngineIndexPrefix: `catalog/search/${searchEngineEdition}_index_prefix`,
+    catalogSearchSearchEngineEnableAuth: `catalog/search/${searchEngineEdition}_enable_auth`,
+    catalogSearchSearchEngineServerTimeout: `catalog/search/${searchEngineEdition}_server_timeout`
+})
+
+/**
+ * @param {string} openSearchVersion
+ * @returns {number}
+ */
+const mapOpenSearchVersionToElasticSearchVersion = (openSearchVersion) => {
+    const { major: parsedOSMajorVersion } = semver.parse(openSearchVersion) || {
+        major: 1
+    }
+
+    if (parsedOSMajorVersion === 2) {
+        return 8
+    }
+
+    return 7
+}
+
+/**
+ * @param {import('../../../../typings/context').ListrContext} ctx
+ */
+const isNeedToInstallElasticSearch8Module = async (ctx) => {
+    /**
+     * @type {{ packages: { name: string, version: string }[] } | null}
+     */
+    const composerLockData = await getJsonfileData(
+        path.join(ctx.config.baseConfig.magentoDir, 'composer.lock')
+    )
+
+    if (!composerLockData) {
+        return true
+    }
+
+    return !composerLockData.packages.some(
+        ({ name }) => name === magentoModuleElasticSearch8
+    )
+}
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
+ */
+const configureElasticSearchInDatabase = () => ({
+    title: 'Configuring Elasticsearch',
+    skip: async (ctx) => {
+        const { ports, isDockerDesktop } = ctx
+        const hostMachine = !isDockerDesktop
+            ? '127.0.0.1'
+            : 'host.docker.internal'
+
+        const { major: parsedESMajorVersion } = semver.parse(
+            ctx.elasticSearchVersion
+        ) || { major: 7 }
+
+        const coreConfigDataSearchEngineKeys =
+            searchEngineConfigurationInCoreConfigDataKeys(
+                `elasticsearch${parsedESMajorVersion}`
+            )
+
+        const isCoreConfigDataExists = await isTableExists(
+            'magento',
+            'core_config_data',
+            ctx
+        )
+
+        if (isCoreConfigDataExists) {
+            const elasticsearchConfig = await ctx.databaseConnection.query(
+                `SELECT path,value
+                FROM core_config_data
+                WHERE path='${coreConfigDataSearchEngineKeys.catalogSearchEngine}'
+                OR path='${coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerHostname}'
+                OR path='${coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerPort}';`
+            )
+
+            const mappedElasticSearchConfig = elasticsearchConfig.reduce(
+                (acc, { path, value }) => ({ ...acc, [path]: value }),
+                {}
+            )
+
+            return ![
+                mappedElasticSearchConfig[
+                    coreConfigDataSearchEngineKeys.catalogSearchEngine
+                ] === `elasticsearch${parsedESMajorVersion}`,
+                mappedElasticSearchConfig[
+                    coreConfigDataSearchEngineKeys
+                        .catalogSearchSearchEngineServerHostname
+                ] === hostMachine,
+                mappedElasticSearchConfig[
+                    coreConfigDataSearchEngineKeys
+                        .catalogSearchSearchEngineServerPort
+                ] === `${ports.elasticsearch}`
+            ].includes(false)
+        }
+
+        return true
+    },
+    task: async (ctx, task) => {
+        const { ports, databaseConnection, isDockerDesktop } = ctx
+        const hostMachine = !isDockerDesktop
+            ? '127.0.0.1'
+            : 'host.docker.internal'
+
+        const { major: parsedESMajorVersion } = semver.parse(
+            ctx.elasticSearchVersion
+        ) || { major: 7 }
+
+        const coreConfigDataSearchEngineKeys =
+            searchEngineConfigurationInCoreConfigDataKeys(
+                `elasticsearch${parsedESMajorVersion}`
+            )
+
+        const elasticsearchConfig = {
+            [coreConfigDataSearchEngineKeys.catalogSearchEngine]: `elasticsearch${parsedESMajorVersion}`,
+            [coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerHostname]:
+                hostMachine,
+            [coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerPort]: `${ports.elasticsearch}`
+        }
+
+        const elasticsearchDynamicConfig = {
+            [coreConfigDataSearchEngineKeys.catalogSearchEngine]: `elasticsearch${parsedESMajorVersion}`,
+            [coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerHostname]:
+                hostMachine,
+            [coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerPort]: `${ports.elasticsearch}`
+        }
+
+        await insertTableValues(
+            'core_config_data',
+            Object.entries(elasticsearchConfig).map(([path, value]) => ({
+                path,
+                value
+            })),
+            { databaseConnection }
+        )
+
+        await updateTableValues(
+            'core_config_data',
+            Object.entries(elasticsearchDynamicConfig).map(([path, value]) => ({
+                path,
+                value
+            })),
+            {
+                databaseConnection,
+                task: {
+                    skip() {
+                        // do nothing
+                    }
+                }
+            }
+        )
+    }
+})
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
+ */
+const installElasticSearch8Module = () => ({
+    title: 'Installing Magento ElasticSearch8 Module',
+    task: async (ctx, task) => {
+        await runComposerCommand(
+            ctx,
+            `require ${magentoModuleElasticSearch8} --update-with-all-dependencies`,
+            {
+                callback: !ctx.verbose
+                    ? undefined
+                    : (t) => {
+                          task.output = t
+                      }
+            }
+        )
+    },
+    options: {
+        bottomBar: 10
+    }
+})
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
+ */
+const configureOpenSearchInDatabase = () => ({
+    title: 'Configuring OpenSearch',
+    skip: async (ctx) => {
+        const { ports, isDockerDesktop, magentoVersion } = ctx
+        const hostMachine = !isDockerDesktop
+            ? '127.0.0.1'
+            : 'host.docker.internal'
+
+        const pureMagentoVersion = magentoVersion.match(
+            /^([0-9]+\.[0-9]+\.[0-9]+)/
+        )[1]
+
+        // required to determine if OpenSearch can be used
+        // OpenSearch is supported in setup:install starting from Magento 2.4.6
+        // OpenSearch 1 based Magento should use ES 7 compatible setup
+        const isAtLeastMagento246 = semver.satisfies(
+            pureMagentoVersion,
+            '>=2.4.6'
+        )
+
+        const useElasticSearch6Configuration = semver.satisfies(
+            pureMagentoVersion,
+            '<=2.3.4'
+        )
+
+        let searchEngineMode = 'opensearch'
+
+        if (!isAtLeastMagento246) {
+            if (useElasticSearch6Configuration) {
+                searchEngineMode = 'elasticsearch6'
+            } else {
+                searchEngineMode = `elasticsearch${mapOpenSearchVersionToElasticSearchVersion(
+                    ctx.openSearchVersion
+                )}`
+            }
+        }
+
+        const coreConfigDataSearchEngineKeys =
+            searchEngineConfigurationInCoreConfigDataKeys(searchEngineMode)
+
+        const isCoreConfigDataExists = await isTableExists(
+            'magento',
+            'core_config_data',
+            ctx
+        )
+
+        if (isCoreConfigDataExists) {
+            const openSearchConfig = await ctx.databaseConnection.query(
+                `SELECT path,value
+                FROM core_config_data
+                WHERE path='${coreConfigDataSearchEngineKeys.catalogSearchEngine}'
+                OR path='${coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerHostname}'
+                OR path='${coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerPort}';`
+            )
+
+            const mappedOpenSearchConfig = openSearchConfig.reduce(
+                (acc, { path, value }) => ({ ...acc, [path]: value }),
+                {}
+            )
+
+            return ![
+                mappedOpenSearchConfig[
+                    coreConfigDataSearchEngineKeys.catalogSearchEngine
+                ] === searchEngineMode,
+                mappedOpenSearchConfig[
+                    coreConfigDataSearchEngineKeys
+                        .catalogSearchSearchEngineServerHostname
+                ] === hostMachine,
+                mappedOpenSearchConfig[
+                    coreConfigDataSearchEngineKeys
+                        .catalogSearchSearchEngineServerPort
+                ] === `${ports.elasticsearch}`
+            ].includes(false)
+        }
+
+        return true
+    },
+    task: async (ctx, task) => {
+        const { ports, databaseConnection, isDockerDesktop, magentoVersion } =
+            ctx
+        const hostMachine = !isDockerDesktop
+            ? '127.0.0.1'
+            : 'host.docker.internal'
+
+        const pureMagentoVersion = magentoVersion.match(
+            /^([0-9]+\.[0-9]+\.[0-9]+)/
+        )[1]
+
+        // required to determine if OpenSearch can be used
+        // OpenSearch is supported in setup:install starting from Magento 2.4.6
+        // OpenSearch 1 based Magento should use ES 7 compatible setup
+        const isAtLeastMagento246 = semver.satisfies(
+            pureMagentoVersion,
+            '>=2.4.6'
+        )
+
+        const compatibleElasticSearchVersion =
+            mapOpenSearchVersionToElasticSearchVersion(ctx.openSearchVersion)
+
+        const useElasticSearch6Configuration = semver.satisfies(
+            pureMagentoVersion,
+            '<=2.3.4'
+        )
+
+        let searchEngineMode = 'opensearch'
+
+        if (!isAtLeastMagento246) {
+            if (useElasticSearch6Configuration) {
+                searchEngineMode = 'elasticsearch6'
+            } else {
+                searchEngineMode = `elasticsearch${compatibleElasticSearchVersion}`
+            }
+        }
+
+        if (searchEngineMode !== 'opensearch') {
+            task.title = `Configuring OpenSearch (using Elasticsearch ${compatibleElasticSearchVersion} compatible configuration)`
+        }
+
+        const coreConfigDataSearchEngineKeys =
+            searchEngineConfigurationInCoreConfigDataKeys(searchEngineMode)
+
+        const opensearchConfig = {
+            [coreConfigDataSearchEngineKeys.catalogSearchSearchEngineIndexPrefix]:
+                'magento2',
+            [coreConfigDataSearchEngineKeys.catalogSearchSearchEngineEnableAuth]: 0,
+            [coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerTimeout]: 15
+        }
+
+        const openSearchDynamicConfig = {
+            [coreConfigDataSearchEngineKeys.catalogSearchEngine]:
+                searchEngineMode,
+            [coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerHostname]:
+                hostMachine,
+            [coreConfigDataSearchEngineKeys.catalogSearchSearchEngineServerPort]: `${ports.elasticsearch}`
+        }
+
+        await insertTableValues(
+            'core_config_data',
+            Object.entries(opensearchConfig).map(([path, value]) => ({
+                path,
+                value
+            })),
+            { databaseConnection }
+        )
+
+        await updateTableValues(
+            'core_config_data',
+            Object.entries(openSearchDynamicConfig).map(([path, value]) => ({
+                path,
+                value
+            })),
+            {
+                databaseConnection,
+                task: {
+                    skip() {
+                        // do nothing
+                    }
+                }
+            }
+        )
+    }
+})
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../../typings/context').ListrContext>}
+ */
+const configureSearchEngine = () => ({
+    task: async (ctx, task) => {
+        const { searchengine = 'elasticsearch' } =
+            ctx.config.overridenConfiguration.configuration
+
+        if (searchengine === 'opensearch') {
+            return task.newListr(configureOpenSearchInDatabase())
+        }
+
+        const { major: parsedESMajorVersion } = semver.parse(
+            ctx.elasticSearchVersion
+        ) || { major: 7 }
+
+        if (
+            parsedESMajorVersion === 8 &&
+            (await isNeedToInstallElasticSearch8Module(ctx))
+        ) {
+            return task.newListr([
+                installElasticSearch8Module(),
+                configureElasticSearchInDatabase()
+            ])
+        }
+
+        return task.newListr(configureElasticSearchInDatabase())
+    }
+})
+
+module.exports = configureSearchEngine

package/lib/tasks/magento/setup-magento/install-magento.js

@@ -23,7 +23,12 @@ const installMagento = ({ isDbEmpty = false } = {}) => ({
         }
         const {
             magentoVersion,
-            config: { magentoConfiguration },
+            config: {
+                magentoConfiguration,
+                overridenConfiguration: {
+                    configuration: { searchengine = 'elasticsearch' }
+                }
+            },
             ports,
             databaseConnection,
             isDockerDesktop
@@ -108,10 +113,50 @@ const installMagento = ({ isDbEmpty = false } = {}) => ({
 
         const isMagento23 = semver.satisfies(pureMagentoVersion, '<2.4')
 
-        const elasticsearchConfiguration = ` \
---search-engine='elasticsearch7' \
+        // required to determine if OpenSearch can be used
+        // OpenSearch is supported in setup:install starting from Magento 2.4.6
+        // OpenSearch 1 based Magento should use ES 7 compatible setup
+        const isAtLeastMagento246 = semver.satisfies(
+            pureMagentoVersion,
+            '>=2.4.6'
+        )
+
+        let searchEngineConfiguration
+
+        if (isAtLeastMagento246 && searchengine === 'opensearch') {
+            searchEngineConfiguration = ` \
+            --search-engine='opensearch' \
+            --opensearch-host='${hostMachine}' \
+            --opensearch-port='${ports.elasticsearch}'`
+        } else {
+            let parsedESMajorVersion = 7
+
+            if (searchengine === 'opensearch') {
+                // OpenSearch 1 is a fork of ES 7, so should be compatible
+                // OpenSearch 2 is based on ES 8, should work in theory
+                const parsedOSVersion = semver.parse(ctx.openSearchVersion) || {
+                    major: 1
+                }
+
+                if (parsedOSVersion.major === 2) {
+                    parsedESMajorVersion = 8
+                }
+
+                // when OpenSearch version 3 comes out this should be replaced
+            } else {
+                // we only expect to have ES 5, 6, 7 and 8 here
+                const parsedESVersion = semver.parse(
+                    ctx.elasticSearchVersion
+                ) || { major: 7 }
+
+                parsedESMajorVersion = parsedESVersion.major
+            }
+
+            searchEngineConfiguration = ` \
+--search-engine=elasticsearch${parsedESMajorVersion} \
 --elasticsearch-host='${hostMachine}' \
 --elasticsearch-port='${ports.elasticsearch}'`
+        }
 
         /**
          * @type {Array<Error>}
@@ -126,7 +171,7 @@ const installMagento = ({ isDbEmpty = false } = {}) => ({
                 --admin-email='${magentoConfiguration.email}' \
                 --admin-user='${magentoConfiguration.user}' \
                 --admin-password='${magentoConfiguration.password}' \
-                ${!isMagento23 ? elasticsearchConfiguration : ''} \
+                ${!isMagento23 ? searchEngineConfiguration : ''} \
                 ${encryptionKeyOption || ''} \
                 --session-save=redis \
                 --session-save-redis-host='${hostMachine}' \

package/lib/tasks/magento/setup-magento/migrate-database.js

@@ -2,7 +2,7 @@ const path = require('path')
 const installMagentoProject = require('../install-magento-project')
 const magentoTask = require('../../../util/magento-task')
 const runMagentoCommand = require('../../../util/run-magento')
-const configureElasticsearch = require('./configure-elasticsearch')
+const configureSearchEngine = require('./configure-searchengine')
 const installMagento = require('./install-magento')
 const upgradeMagento = require('./upgrade-magento')
 const varnishConfigSetup = require('./varnish-config')
@@ -47,7 +47,7 @@ const migrateDatabase = (options = {}) => ({
                     installMagento({ isDbEmpty: true }),
                     updateEnvPHP(),
                     varnishConfigSetup(),
-                    configureElasticsearch(),
+                    configureSearchEngine(),
                     upgradeMagento(),
                     magentoTask('cache:enable')
                 ],
@@ -75,7 +75,7 @@ const migrateDatabase = (options = {}) => ({
                 ctx.isSetupUpgradeNeeded = false
                 // no setup is needed, but still to be sure configure ES
                 return task.newListr(
-                    [varnishConfigSetup(), configureElasticsearch()],
+                    [varnishConfigSetup(), configureSearchEngine()],
                     {
                         concurrent: false,
                         exitOnError: true,
@@ -101,7 +101,7 @@ const migrateDatabase = (options = {}) => ({
                         installMagento(),
                         updateEnvPHP(),
                         varnishConfigSetup(),
-                        configureElasticsearch(),
+                        configureSearchEngine(),
                         upgradeMagento(),
                         magentoTask('cache:enable')
                     ],
@@ -119,7 +119,7 @@ const migrateDatabase = (options = {}) => ({
                 return task.newListr(
                     [
                         varnishConfigSetup(),
-                        configureElasticsearch(),
+                        configureSearchEngine(),
                         upgradeMagento()
                     ],
                     {

package/lib/tasks/magento/setup-magento/set-base-url.js

@@ -31,6 +31,7 @@ const setBaseUrlForScope = (scopeId, code, host) => ({
         const secureLocation = `${host}/` // SSL will work only on port 443, so you cannot run multiple projects with SSL at the same time.
         const httpUrl = `http://${location}`
         const httpsUrl = `https://${secureLocation}`
+        const scope = scopeId === 0 ? 'default' : 'websites'
         const table = 'core_config_data'
         const values = [
             {
@@ -63,7 +64,8 @@ const setBaseUrlForScope = (scopeId, code, host) => ({
                             ['path', '=', path]
                         ],
                         data: {
-                            value
+                            value,
+                            scope
                         }
                     },
                     ctx

package/lib/tasks/requirements/opensearch-version.js

@@ -0,0 +1,49 @@
+const UnknownError = require('../../errors/unknown-error')
+const { containerApi } = require('../docker/containers')
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const checkOpenSearchVersion = () => ({
+    title: 'Checking container OpenSearch version',
+    task: async (ctx, task) => {
+        const { opensearch } = ctx.config.overridenConfiguration.configuration
+        const { ports } = ctx
+
+        let openSearchVersionResponse
+
+        try {
+            openSearchVersionResponse = await containerApi.run({
+                ...opensearch,
+                command: 'opensearch --version',
+                detach: false,
+                rm: true,
+                ports: [`127.0.0.1:${ports.elasticsearch}:9200`],
+                memory: '512mb'
+            })
+        } catch (e) {
+            openSearchVersionResponse = e.message
+        }
+
+        const openSearchVersionResponseResult = openSearchVersionResponse.match(
+            /Version:\s(\d+\.\d+\.\d+)/i
+        )
+
+        if (
+            openSearchVersionResponseResult &&
+            openSearchVersionResponseResult.length > 0
+        ) {
+            const openSearchVersion = openSearchVersionResponseResult[1]
+
+            ctx.openSearchVersion = openSearchVersion
+            task.title = `Using OpenSearch version ${openSearchVersion} in container`
+        } else {
+            throw new UnknownError(
+                `Cannot retrieve OpenSearch Version!\n\n${openSearchVersionResponse}`
+            )
+        }
+    },
+    exitOnError: false
+})
+
+module.exports = checkOpenSearchVersion

package/lib/tasks/requirements/searchengine-version.js

@@ -0,0 +1,20 @@
+const checkElasticSearchVersion = require('./elasticsearch-version')
+const checkOpenSearchVersion = require('./opensearch-version')
+
+/**
+ * @returns {import('listr2').ListrTask<import('../../../typings/context').ListrContext>}
+ */
+const checkSearchEngineVersion = () => ({
+    task: async (ctx, task) => {
+        const { searchengine = 'elasticsearch' } =
+            ctx.config.overridenConfiguration.configuration
+        if (searchengine === 'opensearch') {
+            return task.newListr(checkOpenSearchVersion())
+        }
+
+        return task.newListr(checkElasticSearchVersion())
+    },
+    exitOnError: false
+})
+
+module.exports = checkSearchEngineVersion

package/lib/tasks/start.js

@@ -34,7 +34,6 @@ const {
 } = require('../util/instance-metadata')
 const waitingForVarnish = require('./magento/setup-magento/waiting-for-varnish')
 const checkPHPVersion = require('./requirements/php-version')
-const checkElasticSearchVersion = require('./requirements/elasticsearch-version')
 const volumes = require('./docker/volume/tasks')
 const convertMySQLDatabaseToMariaDB = require('./docker/convert-mysql-to-mariadb')
 const { cmaGlobalConfig } = require('../config/cma-config')
@@ -42,6 +41,7 @@ const { setProjectConfigTask } = require('./project-config')
 const {
     convertComposerHomeToComposerCacheVolume
 } = require('./docker/convert-composer-home-to-composer-cache-volume')
+const checkSearchEngineVersion = require('./requirements/searchengine-version')
 
 /**
  * @returns {import('listr2').ListrTask<import('../../typings/context').ListrContext>}
@@ -135,7 +135,7 @@ const configureProject = () => ({
             {
                 task: (ctx, subTask) =>
                     subTask.newListr(
-                        [checkPHPVersion(), checkElasticSearchVersion()],
+                        [checkPHPVersion(), checkSearchEngineVersion()],
                         {
                             concurrent: true
                         }

package/lib/util/config-file-validator.js

@@ -147,6 +147,15 @@ const elasticsearchConfigurationSchema = Joi.object({
     env: Joi.object().optional()
 })
 
+const opensearchConfigurationSchema = Joi.object({
+    image: Joi.string().optional(),
+    env: Joi.object().optional()
+})
+
+const searchEngineConfiguration = Joi.string().valid(
+    'elasticsearch',
+    'opensearch'
+)
 /**
  * @type {Joi.ObjectSchema<import('../../typings').ComposerConfiguration>}
  */
@@ -208,6 +217,8 @@ const configurationSchema = Joi.object({
     nginx: nginxConfigurationSchema.optional(),
     mariadb: mariadbConfigurationSchema.optional(),
     elasticsearch: elasticsearchConfigurationSchema.optional(),
+    opensearch: opensearchConfigurationSchema.optional(),
+    searchengine: searchEngineConfiguration.optional(),
     redis: serviceConfigurationSchema.optional(),
     composer: composerConfigurationSchema.optional(),
     varnish: varnishConfigurationSchema.optional(),