@sap/cds-dk 6.8.2 → 7.0.0

package/lib/build/buildTaskEngine.js

@@ -0,0 +1,355 @@
+const fs = require('fs')
+const path = require('path')
+const cds = require('../cds'), { _log: log } = cds
+const { sortMessagesSeverityAware, deduplicateMessages, CompilationError } = cds.compiler
+const { relativePaths, BuildError, resolveRequiredSapModels } = require('./util')
+const { SEVERITIES, LOG_LEVELS } = require('./constants')
+const BuildTaskFactory = require('./buildTaskFactory')
+const BuildTaskProviderFactory = require('./buildTaskProviderFactory')
+const BuildTaskHandlerInternal = require('./provider/buildTaskHandlerInternal')
+
+const COMPILATION_ERROR = 'CompilationError'
+const COMPILE_MESSAGE = 'CompileMessage'
+
+class BuildTaskEngine {
+    constructor(options) {
+        this._providerFactory = new BuildTaskProviderFactory(options)
+    }
+    get providerFactory() {
+        return this._providerFactory
+    }
+    get logger() {
+        return this.providerFactory.logger
+    }
+    get buildOptions() {
+        return this.providerFactory.buildOptions
+    }
+
+    async processTasks(tasks) {
+        const startTime = Date.now()
+        const messages = []
+
+        this.logger.log(`building project [${cds.root}], clean [${this.buildOptions.clean}]`)
+        this.logger.log(`cds [${cds.version}], compiler [${cds.compiler.version()}], home [${cds.home}]\n`)
+
+        if (!tasks) {
+            const buildTaskFactory = new BuildTaskFactory(this.buildOptions)
+            tasks = await buildTaskFactory.getTasks()
+        }
+
+        // validate required @sap namespace models - log only
+        const unresolved = BuildTaskEngine._resolveRequiredSapServices(tasks)
+        if (unresolved.length > 0) {
+            throw new BuildError(`Required CDS models [${unresolved.join(', ')}] cannot be resolved. Make sure up-to-date versions of the missing modules are installed.`)
+        }
+
+        // create build task handlers
+        const handlers = []
+        tasks.forEach((task) => {
+            if (task) {
+                const handler = this._createHandler(task)
+                handlers.push(handler)
+            }
+        })
+
+        // use resolved tasks
+        this.providerFactory.context.tasks = handlers.map(handler => handler.task)
+
+        try {
+            await this._executePrepare(handlers)
+            await this._executeCleanBuildTasks(handlers)
+
+            // throwing Exception in case of compilation errors
+            const buildResult = await this._executeBuildTasks(handlers)
+
+            await this._writeGenerationLog(handlers)
+            this._logBuildOutput(handlers)
+            this._logMessages([...BuildTaskEngine._getHandlerMessages(handlers), ...messages])
+            this._logTimer(startTime, Date.now())
+
+            return buildResult
+        } catch (error) {
+            this._logBuildOutput(handlers)
+
+            // cds CLI layer logs in case of an exception if invoked from CLI
+            if (!this.buildOptions.cli) {
+                this._logMessages([...BuildTaskEngine._getErrorMessages([error]), ...messages])
+            }
+
+            throw error
+        }
+    }
+
+    /**
+     * BuildTaskHandler#prepare has been deprecated and was never part of the public API.
+     * Currently only used by internal FioriBuildTaskHandler.
+     * @deprecated
+     * @param {*} handlers
+     * @returns
+     */
+    async _executePrepare(handlers) {
+        const handlerGroups = new Map()
+
+        // group handlers by type
+        handlers.forEach(handler => {
+            handlerGroups.has(handler.task.for) ? handlerGroups.get(handler.task.for).push(handler) : handlerGroups.set(handler.task.for, [handler])
+        })
+
+        const promises = []
+        for (let handlerGroup of handlerGroups.values()) {
+            promises.push(this._doPrepare(handlerGroup))
+        }
+        return Promise.all(promises)
+    }
+
+    /**
+     * @deprecated
+     * @param {*} handlerGroup
+     */
+    async _doPrepare(handlerGroup) {
+        for (let handler of handlerGroup) {
+            // prepare has been deprecated
+            if (handler instanceof BuildTaskHandlerInternal) {
+                this.logger._debug && this.logger.debug(`preparing, handler [${handler.constructor.name}], src [${relativePaths(cds.root, handler.task.src)}]`)
+                const result = await handler.prepare()
+                if (result === false) {
+                    break
+                }
+            }
+        }
+    }
+
+    async _executeCleanBuildTasks(handlers) {
+        if (this.buildOptions.clean) {
+            // clean entire build staging folder once
+            if (cds.env.build.target !== '.') {
+                const target = path.resolve(cds.root, cds.env.build.target)
+                this.logger._debug && this.logger.debug(`cleaning staging folder ${target}`)
+                await fs.promises.rm(target, { force: true, recursive: true })
+            }
+
+            const results = await Promise.allSettled(handlers.map((handler) => {
+                this.logger._debug && this.logger.debug(`cleaning, handler [${handler.constructor.name}], src [${relativePaths(cds.root, handler.task.src)}]`)
+                return handler.clean()
+            }))
+            // check for errors and throw exception
+            this._resolveHandlerResponse(results)
+        }
+    }
+
+    async _executeBuildTasks(handlers) {
+        // sort handlers based on priority in
+        handlers = handlers.sort((a, b) => {
+            return a.priority === b.priority ? 0 : a.priority > b.priority ? -1 : 1
+        })
+        // group handlers with same priority in order to execute in parallel
+        const buildPipeline = handlers.reduce((acc, handler) => {
+            if (acc.length === 0) {
+                acc.push([handler])
+            } else {
+                const currGroup = acc[acc.length - 1]
+                if (currGroup[0].priority === handler.priority) {
+                    currGroup.push(handler)
+                } else {
+                    acc.push([handler])
+                }
+            }
+            return acc
+        }, [])
+
+        const results = await this._executePipeline(buildPipeline)
+
+        // check for errors and throw exception - return results otherwise including any compiler and build status messages
+        return this._resolveHandlerResponse(results, BuildTaskEngine._getHandlerMessages(handlers))
+    }
+
+    async _executePipeline(pipeline) {
+        let allResults = []
+        for (const group of pipeline) {
+            const results = await Promise.allSettled(group.map((handler) => {
+                this.logger._debug && this.logger.debug(`building, handler [${handler.constructor.name}], src [${relativePaths(cds.root, handler.task.src)}]`)
+                return handler.build()
+                    .then(handlerResult => {
+                        return Promise.resolve({
+                            task: handler.task,
+                            result: handlerResult,
+                            messages: this._sortMessagesUnique(handler.messages)
+                        })
+                    })
+            }))
+            allResults = allResults.concat(results)
+        }
+        return allResults
+    }
+
+    _resolveHandlerResponse(results, handlerMessages = []) {
+        const errors = []
+        const resolvedResults = results.reduce((acc, r) => {
+            if (r.status === 'fulfilled') {
+                acc.push(r.value)
+            }
+            if (r.status === 'rejected' && r.reason) {
+                errors.push(r.reason)
+            }
+            return acc
+        }, [])
+
+        if (errors.length > 0) {
+            const error = errors.find(e => e.constructor.name !== COMPILATION_ERROR)
+            if (error) {
+                // throw original error, including BuildErrors
+                // for BuildErrors we do not merge other build messages that might exist in order to keep the original error intact
+                throw error
+            }
+
+            // propagate existing CompilationErrors
+            // merge all existing compilation messages into a single CompilationError
+            // compiler warning and info messages are returned as handler messages
+            const compileErrors = errors.filter(e => e.constructor.name === COMPILATION_ERROR)
+            const compileMessages = handlerMessages.filter(message => message.constructor.name === COMPILE_MESSAGE)
+            if (compileErrors.length) {
+                throw new CompilationError(this._sortMessagesUnique(BuildTaskEngine._getErrorMessages(compileErrors), compileMessages))
+            }
+        }
+        return resolvedResults
+    }
+
+    _createHandler(task) {
+        try {
+            const handler = this.providerFactory.createHandler(task)
+            handler.init()
+
+            if (!(handler instanceof BuildTaskHandlerInternal) && handler.priority !== 1) {
+                // Custom build handlers are executed before internal handlers to ensure
+                // that generated content cannot be overwritten by mistake
+                throw new Error(`Illegal priority for ${handler.constructor.name} encountered for custom handler - in this version only priority value '1' is allowed`)
+            }
+            this._logTaskHandler(handler)
+
+            return handler
+        } catch (error) {
+            log(error, { log: this.logger.log })
+            throw error
+        }
+    }
+
+    _logBuildOutput(handlers) {
+        // log all generated files
+        const files = BuildTaskEngine._getBuildOutput(handlers)
+        if (files.length > 0) {
+            this.logger.log(`done > wrote output to:\n   ${files.join("\n   ")}\n`)
+        }
+    }
+
+    async _writeGenerationLog(handlers) {
+        const outputFile = cds.env.build.outputfile || process.env.GENERATION_LOG
+        if (outputFile) {
+            const files = BuildTaskEngine._getBuildOutput(handlers)
+            this.logger.log(`writing generation log to [${outputFile}]\n`)
+            try {
+                await fs.promises.mkdir(path.dirname(outputFile), { recursive: true })
+                await fs.promises.writeFile(outputFile, files.join('\n'))
+            } catch (error) {
+                this.logger.error(`failed to write generation log`)
+                this.logger.error(error.stack || error)
+            }
+        }
+    }
+
+    static _getBuildOutput(handlers) {
+        const files = handlers.reduce((acc, handler) => acc.concat(handler.files), []).sort()
+        return files.map(file => {
+            if (path.isAbsolute(cds.env.build.target)) {
+                return file
+            }
+            return path.relative(cds.root, file)
+        })
+    }
+
+    _logTimer(start, end) {
+        this.logger.log(`build completed in ${end - start} ms\n`)
+    }
+
+    _logTaskHandler(handler) {
+        this.logger._debug && this.logger.debug(`handler ${handler.constructor.name}`)
+        this.logger._debug && this.logger.debug(`details src [${relativePaths(cds.root, handler.task.src)}], dest [${relativePaths(cds.root, handler.task.dest)}], use [${handler.task.use}], options [${JSON.stringify(handler.task.options)}]`) //NOSONAR
+    }
+
+    _logMessages(messages) {
+        if (messages.length > 0) {
+            const options = {
+                log: this.logger.log,
+                "log-level": this._getLogLevel() // ensures that for tests the correct cds.env is used
+            }
+            deduplicateMessages(messages)
+            log(messages, options)
+        }
+    }
+
+    static _resolveRequiredSapServices(tasks) {
+        const taskModelPaths = tasks.reduce((acc, task) => {
+            const model = task.options?.model
+            if (model) {
+                if (Array.isArray(model)) {
+                    model.forEach(m => acc.add(m))
+                } else {
+                    acc.add(model)
+                }
+            }
+            return acc
+        }, new Set())
+
+        return resolveRequiredSapModels([...taskModelPaths])
+    }
+
+    /**
+     * Returns a sorted and flattened list of all messages extracted from the given errors.
+     * @param {Array<Error>} errors
+     */
+    static _getErrorMessages(errors) {
+        let messages = []
+        // flatten all compile messages in order to filter duplicates and sort later on
+        errors.forEach(error => {
+            if (Array.isArray(error.errors) && error.errors.length > 0) {
+                messages = messages.concat(this._getErrorMessages(error.errors))
+            } else {
+                messages.push(error)
+            }
+        })
+        return messages
+    }
+
+    /**
+     * Returns compiler messages and validation messages issued by handlers.
+     * @param {Array<BuildTaskHandler>} handlers
+     */
+    static _getHandlerMessages(handlers) {
+        return handlers.reduce((acc, handler) => acc.concat(handler.messages), [])
+    }
+
+    /**
+     * Sort and filter the given errors of type CompileMessage or BuildMessage according to their severity and location,
+     * but leave any other errors untouched as part of the result array.<br>
+     * The log level passed as 'buildOptions.log-level' or defined in 'cds.env' is used to filter the given messages.
+     * @param  {...Error} messages
+     */
+    _sortMessagesUnique(...messages) {
+        const logLevelIdx = LOG_LEVELS.indexOf(this._getLogLevel())
+        // flatten
+        messages = messages.reduce((acc, m) => acc.concat(m), [])
+        // filter according to log-level
+        const filteredMessages = messages.filter(message => !message.severity || logLevelIdx >= SEVERITIES.indexOf(message.severity))
+        // remove duplicates
+        deduplicateMessages(filteredMessages)
+        // sort
+        return sortMessagesSeverityAware(filteredMessages)
+    }
+
+    /**
+     * Return user defined log level or default value 'warn'
+     */
+    _getLogLevel() {
+        return this.buildOptions["log-level"] || cds.env["log-level"]
+    }
+}
+module.exports = BuildTaskEngine

package/lib/build/buildTaskFactory.js

@@ -0,0 +1,284 @@
+const fs = require('fs')
+const path = require('path')
+const cds = require('../cds'), { _log:log } = cds
+const BuildTaskProviderFactory = require('./buildTaskProviderFactory')
+const { hasJavaNature, getProperty, redactCredentials, flatten, getDefaultModelOptions } = require('./util')
+const { FILE_EXT_CDS, BUILD_TASK_JAVA_CF, BUILD_TASK_JAVA } = require("./constants")
+
+class BuildTaskFactory {
+    constructor(options) {
+        this._providerFactory = new BuildTaskProviderFactory(options)
+    }
+    get providerFactory() {
+        return this._providerFactory
+    }
+    get logger() {
+        return this.providerFactory.logger
+    }
+    get buildOptions() {
+        return this.providerFactory.buildOptions
+    }
+
+    // the following order for determining build tasks is used
+    // 1. create from command line input, e.g. cds build/all --for hana --src db --model srv --dest db
+    // 2. read using cds.env.build.tasks
+    // 3. create from cds.env.folders config data
+    async getTasks() {
+        try {
+            const tasks = await this._createTasks()
+
+            // log build tasks
+            this.logger.log(`the following build tasks will be executed`)
+            let tasksOutput = "   {\n     \"build\": {\n       \"target\": \"" + cds.env.build.target + "\",\n       \"tasks\": ["
+            for (let i = 0; i < tasks.length; i++) {
+                tasksOutput += "\n         " + BuildTaskFactory._stringifyTask(tasks[i]) + (i + 1 < tasks.length ? "," : "")
+            }
+            tasksOutput += "\n       ]\n     }\n   }\n"
+            this.logger.log(tasksOutput)
+
+            // for testing purposes
+            this.logger.debug("cds.env used for build:")
+            this.logger._debug && this.logger.debug(redactCredentials(cds.env))
+
+            // always resolve tasks for input validation
+            const resolvedTasks = this.providerFactory.resolveTasks(tasks)
+            if (this.buildOptions.resolve) {
+                return resolvedTasks
+            }
+            return tasks
+        } catch (e) {
+            this.logger.log('')
+            // cds CLI layer is doing the logging if invoked using CLI
+            if (!this.buildOptions.cli) {
+                log(e, { withStack: true, log: (message) => this.logger.log(message) })
+            }
+            throw e
+        }
+    }
+
+    async _createTasks() {
+        this.logger.debug(`determining build tasks for project [${cds.root}].`)
+
+        // clearing model cache (details https://github.tools.sap/cap/cds/pull/181) is no longer required
+        // because of changes https://github.tools.sap/cap/cds/pull/1121
+        // cds.resolve.cache = {}
+
+        let tasks = BuildTaskFactory._getExistingTasks()
+        if (tasks.length === 0) {
+            tasks = await this.providerFactory.lookupTasks()
+        } else {
+            // 1. apply default values including task.for and task.use and ensure that for all tasks a provider exists - throwing error otherwise
+            await this.providerFactory.applyTaskDefaults(tasks)
+            // ensure that dependencies get wired up before filtering
+            await this.providerFactory.lookupTasks(tasks)
+        }
+
+        // 3. filters the list of build tasks and adapts according to given CLI options
+        // Note: A new task might get created, e.g. 'cds build --for hana' will enforce a hana build even if sqlite has been configured
+        let existingTasks = tasks
+        tasks = this._applyCliOptionsFilter(tasks)
+        if (tasks.length === 0) {
+            return tasks
+        }
+        // a new array is returned to indicate new build tasks have been created
+        if (tasks !== existingTasks) {
+            // a different task shall be executed
+            await this.providerFactory.applyTaskDefaults(tasks)
+        }
+
+        // 2. add dependencies
+        existingTasks = [...tasks]
+        await this.providerFactory.lookupTasks(tasks)
+        if (tasks.length > existingTasks.length) {
+            const newTasks = tasks.filter(task => !existingTasks.includes(task))
+            // a dependant task was added
+            this._applyCliOptions(newTasks)
+        }
+
+        // obligatory task defaults shared by all tasks
+        BuildTaskFactory._applyCommonTaskDefaults(tasks)
+
+        // ensure correct values for optional build task properties, error for missing mandatory properties
+        BuildTaskFactory._validateBuildTasks(tasks)
+
+        this._setDefaultBuildTargetFolder(tasks)
+        return tasks
+    }
+
+    static _getExistingTasks() {
+        return Array.isArray(getProperty(cds.env, 'build.tasks')) ? JSON.parse(JSON.stringify(cds.env.build.tasks)) : []
+    }
+
+    static _applyCommonTaskDefaults(tasks) {
+        const modelPaths = getDefaultModelOptions()
+
+        tasks.forEach(task => {
+            this._setTaskModelOptions(task, modelPaths)
+            if (!task.src) {
+                throw new Error(`Invalid build task definition - value of property 'src' is missing in [${task.for || task.use}].`)
+            }
+        })
+    }
+
+    static _validateBuildTasks(tasks) {
+        tasks.forEach(task => {
+            if (!task.src) {
+                throw new Error(`Invalid build task definition - value of property 'src' is missing in [${task.for || task.use}].`)
+            }
+        })
+    }
+
+    _setDefaultBuildTargetFolder(tasks) {
+        const task = tasks.find(task => task.for === BUILD_TASK_JAVA_CF || task.for === BUILD_TASK_JAVA)
+        const srv = task ? task.src : BuildTaskFactory._getModuleFolder(flatten([cds.env.folders.srv])) || "srv"
+
+        // Java projects use "." as the default build target folder
+        if (hasJavaNature([path.join(cds.root, srv), cds.root]) && BuildTaskFactory._adaptBuildTargetSettingForJava()) {
+            this.logger.debug("using inplace build for java project instead of default staging build")
+        }
+    }
+
+    /**
+    * Use inplace build for java projects if build.target has not been configured.
+    * @returns {boolean} true if changed, false otherwise
+    */
+    static _adaptBuildTargetSettingForJava() {
+        if (cds.env.build.target !== ".") {
+            // filter user settings of cds.env
+            const userEnv = cds.env.for("cds", cds.root, false)
+
+            // use helper as env.build might be undefined
+            if (!userEnv.build?.target) {
+                cds.env.build.target = "."
+                return true
+            }
+        }
+        return false
+    }
+
+    _applyCliOptionsFilter(tasks) {
+        const buildOptions = this.buildOptions
+        // filter tasks using either option for, use, src
+        let resultTasks = tasks.filter(task => {
+            return (!buildOptions.use || buildOptions.use === task.use)
+                && (!buildOptions.for || buildOptions.for === task.for)
+                && (!buildOptions.src || buildOptions.src === task.src)
+        })
+        if (resultTasks.length === 0) {
+            if (buildOptions.for || buildOptions.use) {
+                const task = {}
+                if ((buildOptions.for)) {
+                    task.for = buildOptions.for
+                }
+                if ((buildOptions.use)) {
+                    task.use = buildOptions.use
+                }
+                if (buildOptions.src) {
+                    task.src = buildOptions.src
+                }
+                resultTasks.push(task)
+            }
+        } else if (resultTasks.length < tasks.length) {
+            // return the same array as long as it contains a subset of the given tasks
+            tasks.length = 0
+            resultTasks.forEach(task => tasks.push(task))
+            resultTasks = tasks
+        }
+        this._applyCliOptions(resultTasks)
+        return resultTasks
+    }
+
+    _applyCliOptions(tasks) {
+        const buildOptions = this.buildOptions
+        // apply remaining cli options to filtered tasks
+        tasks.forEach(task => {
+            if (buildOptions.dest) {
+                task.dest = buildOptions.dest
+            }
+            if (buildOptions.opts) {
+                const opts = BuildTaskFactory._scanTaskOptionParams(buildOptions.opts)
+                task.options = task.options ? Object.assign(task.options, opts) : opts
+            }
+        })
+    }
+
+    static _setTaskModelOptions(task, defaultModelPaths) {
+        task.options = task.options || {}
+        if (!task.options.model || Array.isArray(task.options.model) && task.options.model.length === 0) {
+            defaultModelPaths = new Set(defaultModelPaths)
+            if (task.src) {
+                defaultModelPaths.add(task.src)
+            }
+            task.options.model = [...defaultModelPaths]
+        } else if (!Array.isArray(task.options.model)) {
+            task.options.model = [task.options.model]
+        }
+    }
+
+    static _stringifyTask(task) {
+        // ensures identical order of properties
+        const order = ["for", "use", "src", "dest", "options"]
+        const keys = Object.keys(task).sort((a, b) => order.indexOf(a) - order.indexOf(b))
+
+        return keys.reduce((acc, key, idx) => {
+            // render either "for" OR "use" value
+            return acc + (!key.match("^_") ? (key !== "use" || keys[idx - 1] !== "for" ? (acc !== "{" ? ", " : "") + JSON.stringify(key) + ":" + JSON.stringify(task[key]) : "") : "")
+        }, "{") + "}"
+    }
+
+    static _scanTaskOptionParams(optsParams) {
+        // need to create new regex every call since a constant would keep the match state
+        const quoteRegex = /([\w-]+)=([\w/.]+|\[([\w/,.]+)\])/g
+
+        // captures a=1             => a:1
+        //          a=[x,y,z]       => a:[x,y,z]
+        //          a=1,b=[x,y,z]   => a:1 b=[x,y,z]
+        let match = quoteRegex.exec(optsParams)
+        const taskOptions = {}
+
+        while (match != null) {
+            const key = match[1]
+            const value = match[3] || match[2]
+            const valueArray = value.split(",")
+            taskOptions[key] = valueArray.length > 1 ? valueArray.map((entry) => entry.trim()) : value
+            match = quoteRegex.exec(optsParams)
+        }
+        return taskOptions
+    }
+
+    /**
+     * For valid paths remove trailing '/'. Otherwise return as is - important!!
+     * @param {*} dir
+     */
+    static _normalizePath(dir) {
+        return typeof dir === "string" ? dir.replace(/\/$/, '') : dir
+    }
+
+    /**
+     * Determines the module folder from the past list that may represent files or folders w or w/o .cds file extension.
+     * @param {Array} filesOrFolders
+     */
+    static _getModuleFolder(filesOrFolders) {
+        const resources = [...filesOrFolders]
+        filesOrFolders.forEach(fileOrFolder => {
+            if (path.extname(fileOrFolder) !== FILE_EXT_CDS) {
+                resources.push(fileOrFolder + FILE_EXT_CDS)
+            }
+        })
+        return resources.reduce((acc, resource) => {
+            if (!acc) {
+                let resourcePath = path.resolve(cds.root, resource)
+                if (fs.existsSync(resourcePath)) {
+                    if (fs.lstatSync(resourcePath).isDirectory()) {
+                        acc = resource
+                    } else {
+                        // represents file
+                        acc = path.dirname(resource)
+                    }
+                }
+            }
+            return acc
+        }, null)
+    }
+}
+module.exports = BuildTaskFactory