@sap/cds-dk 6.8.3 → 7.0.0

package/lib/build/provider/hana/index.js

@@ -0,0 +1,522 @@
+const fs = require('fs')
+const path = require('path')
+const cds = require('../../../cds')
+const to_hdbmigration = require('./2migration')
+const BuildTaskHandlerInternal = require('../buildTaskHandlerInternal')
+const { BuildError, relativePaths } = require('../../util')
+
+const { OUTPUT_MODE_NONE, OUTPUT_MODE, CONTENT_PACKAGE_JSON, CONTENT_HDBTABLEDATA, CSV_FILE_DETECTION,
+    CONTENT_ENV, CONTENT_DEFAULT_ENV_JSON, CONTENT_NODE_MODULES, OUTPUT_MODE_RESULT } = require('../../constants')
+const { ERROR, WARNING } = BuildTaskHandlerInternal
+
+const DEFAULT_COMPILE_DEST_FOLDER = path.normalize("src/gen")
+
+const FILE_EXT_CSV = ".csv"
+const FILE_EXT_HDBTABLEDATA = ".hdbtabledata"
+const FILE_EXT_HDBTABLE = ".hdbtable"
+const FILE_EXT_HDBMIGRATIONTABLE = ".hdbmigrationtable"
+
+const FILE_NAME_HDICONFIG = ".hdiconfig"
+const FILE_NAME_HDINAMESPACE = ".hdinamespace"
+const FILE_NAME_PACKAGE_JSON = "package.json"
+const PATH_LAST_DEV_CSN = "last-dev/csn.json"
+const FILE_NAME_UNDEPLOY_JSON = "undeploy.json"
+
+const DEPLOY_FORMAT = "deploy-format"
+
+class HanaModuleBuilder extends BuildTaskHandlerInternal {
+    init() {
+        if (this.hasBuildOption(OUTPUT_MODE, OUTPUT_MODE_RESULT)) {
+            this._result = {
+                dest: this.task.dest,
+                hana: []
+            }
+        }
+        this.task.options.compileDest = path.resolve(this.task.dest, this.task.options.compileDest || DEFAULT_COMPILE_DEST_FOLDER)
+    }
+
+    async build() {
+        const { src, dest } = this.task
+        const model = await this.model()
+        if (!model) {
+            return
+        }
+        // the order is important
+        // 1. compile
+        const hdiPlugins = await this._compileToHana(model)
+
+        if (!this.hasBuildOption(OUTPUT_MODE, OUTPUT_MODE_NONE)) {
+            // 2. copy resources
+            if (this.hasBuildOption(CSV_FILE_DETECTION, false)) {
+                await this._copyResources(src, dest)
+            } else {
+                await this._copyResourcesExt(src, dest, model)
+            }
+
+            // 3. create additional stuff in dest
+            await this._writeHdiConfig(hdiPlugins)
+            await this._writeHdiNamespace()
+
+            // TODO disabled as this contradicts the MTX domain concept which allows partial app deployments
+            //await this._writeUndeployJson()
+
+            if (!this.hasBuildOption(CONTENT_HDBTABLEDATA, false)) {
+                await this._compileToHdbtabledata(model)
+            }
+            if (!this.hasBuildOption(CONTENT_PACKAGE_JSON, false)) {
+                await this._writePackageJson()
+            }
+        }
+        return this._result
+    }
+
+    /**
+     * Deletes any content that has been created in folder '#this.task.dest/src/gen' by inplace mode.
+     * <br>
+     * Note: Content created in staging build will be deleted by the #BuildTaskEngine itself.
+     */
+    async clean() {
+        if (this.isStagingBuild()) {
+            return super.clean()
+        }
+        return fs.promises.rm(this.task.options.compileDest, { force: true, recursive: true })
+    }
+
+    /**
+     * Copies all files located at <src> (except HANA artifacts not contained in <db>/src/**) to the folder <dest>.
+     * '*.csv' files are read based on the corresponding CDS model file location and copied as flat list into folder '<dest>/src/gen>'.
+     *
+     * @param {string} src
+     * @param {string} dest
+     * @param {Object} model
+     */
+    async _copyResourcesExt(src, dest, model) {
+        const resources = Object.keys(await cds.deploy.resources(model)).reverse() // reverse to get reuse resources first and app resources last
+        const csvPath = path.join(src, "data")
+        // determine subfolder name used by the application for backward compatibility
+        const csvFolder = resources.some(res => res && res.startsWith(csvPath)) ? "data" : "csv"
+
+        // 1. copy csv files into 'src/gen/data' or 'src/gen/csv' subfolder
+        const dbSrc = path.join(src, 'src')
+
+        // await each copy as order is important - reuse resources first and app resources last
+        for (const res of resources) {
+            // do not duplicate resources that are already contained in db/src/**
+            if (res && /\.csv$/.test(res) && !res.startsWith(dbSrc)) {
+                await this.copy(res).to(path.join(this.task.options.compileDest, csvFolder, path.basename(res)))
+            }
+        }
+
+        // 2. staging build: copy files except *.cds, .env, default-env.json, ./node_modules/**
+        if (this.isStagingBuild()) {
+            let blockList = "\\.cds$|\\.csv$|\\.hdbtabledata$"
+            blockList += !this.hasBuildOption(CONTENT_ENV, true) ? "|\\.env($|\\..*$)" : ""
+            blockList += !this.hasBuildOption(CONTENT_DEFAULT_ENV_JSON, true) ? "|default-env\\.json$" : ""
+            blockList = new RegExp(blockList)
+
+            await this.copyNativeContent(src, dest, (entry) => {
+                if (entry.startsWith(dbSrc)) {
+                    // entire native content
+                    return true
+                }
+                if (fs.statSync(entry).isDirectory()) {
+                    return !/(\/|\\)node_modules(\/|\\)?$/.test(entry) ||
+                        (/(\/|\\)node_modules(\/|\\)?$/.test(entry) && this.hasBuildOption(CONTENT_NODE_MODULES, true))
+                }
+                return !blockList.test(entry)
+            })
+        }
+    }
+
+    /**
+     * Copies the entire content of the db module located in the given <src> folder to the folder <dest>.
+     * '*.csv' and '*.hdbtabledata' files located in a subfolder 'data' or 'csv' will be copied to '<dest>/src/gen/data>'||'<dest>/src/gen/csv>'
+     *
+     * @param {string} src
+     * @param {string} dest
+     */
+    async _copyResources(src, dest) {
+        const dbCsvDir = path.join(src, "csv")
+        const dbDataDir = path.join(src, "data")
+        const csvDirs = [dbCsvDir, dbDataDir]
+        const regexData = RegExp('\\.csv$|\\.hdbtabledata$')
+
+        if (this.isStagingBuild()) {
+            const regex = RegExp('\\.cds$|\\.csv$|\\.hdbtabledata$')
+
+            await this.copyNativeContent(src, dest, (entry) => {
+                if (fs.statSync(entry).isDirectory()) {
+                    return !/(\/|\\)node_modules(\/|\\)?$/.test(entry)
+                }
+                return (!regex.test(entry) && entry !== cds.env.build.outputfile) ||
+                    (regexData.test(entry) && !entry.startsWith(dbCsvDir) && !entry.startsWith(dbDataDir))
+            })
+        }
+
+        // handle *.csv and *.hdbtabledata located in '<dbSrc>/data' and '<dbSrc>/csv' folder
+        const allFiles = csvDirs.reduce((acc, csvDir) => {
+            return acc.concat(BuildTaskHandlerInternal._find(csvDir, (entry) => {
+                if (fs.statSync(entry).isDirectory()) {
+                    return false
+                }
+                return regexData.test(entry)
+            }))
+        }, [])
+
+        return Promise.all(allFiles.map((file) => {
+            return this.copy(file).to(path.join(this.task.options.compileDest, path.relative(src, file)))
+        }))
+    }
+
+    /**
+     * Generates *.hdbtabledata files in folder '#this.task.dest/src/gen' from *.csv files located in '#this.task.dest/src/**' folder.
+     * The generated *.hdbtabledata files will link to their *.csv counterparts using relative links. The *.csv files have either
+     * already been defined in the 'src' folder or they have been copied to '#this.task.dest/src/gen/**' folder if they have been
+     * created outside 'src' folder. If custom *.hdbtabledata files are found nothing is generated for this particular folder.
+     * <br>
+     * Note: *.csv and *.hdbtabledata need to be copied to '#this.task.dest/src/gen**' if required before this method is called.
+     * In inplace mode dest folder is refering to src folder.
+     *
+     * @param {object} model compiled csn
+     */
+    async _compileToHdbtabledata(model) {
+        const tableDataDirs = new Set()
+        const destSrcDir = path.join(this.task.dest, "src")
+
+        const allCsvFiles = BuildTaskHandlerInternal._find(destSrcDir, (entry) => {
+            if (fs.statSync(entry).isDirectory()) {
+                return true
+            }
+            if (/\.hdbtabledata$/.test(entry)) {
+                tableDataDirs.add(path.dirname(entry))
+            }
+            return /\.csv$/.test(entry)
+        })
+        if (allCsvFiles.length > 0) {
+            const csvDirs = allCsvFiles.map(path.dirname).reduce((dirs, dir) => {
+                if (!tableDataDirs.has(dir) && !dirs.includes(dir)) { // exclude any dir where a tabledata is present
+                    dirs.push(dir)
+                }
+                return dirs
+            }, [])
+
+            // ODM csv data comes with license comments, so strip these
+            if (!this.hasBuildOption("stripCsvComments", false)) {
+                await this._stripCsvComments(allCsvFiles)
+            }
+
+            const promises = []
+            const relDest = path.relative(this.task.dest, this.task.options.compileDest)
+            const options = { ...this.options(), logger: this.logger, dirs: csvDirs, baseDir: this.task.options.compileDest }
+
+            const toHdbtabledata = cds.compile.to.hdbtabledata ?? require(path.join(cds.home, 'bin/build/provider/hana/2tabledata')) // cds@6 compatibility
+            const tableDataResult = await toHdbtabledata(model, options)
+
+            for (let [tableData, { file, csvFolder }] of tableDataResult) {
+                // create .hdbtabledata side-by-side if .csv is contained in 'src/gen/**' subfolder
+                // otherwise create in 'src/gen'
+                let tableDataPath = csvFolder.startsWith(this.task.options.compileDest) ? csvFolder : this.task.options.compileDest
+                tableDataPath = path.join(tableDataPath, file)
+                if (this.hasBuildOption(OUTPUT_MODE, OUTPUT_MODE_RESULT)) {
+                    this._result.hana.push(path.join(relDest, file))
+                }
+                promises.push(this.write(tableData).to(tableDataPath))
+            }
+            await Promise.all(promises)
+        }
+    }
+
+    async _stripCsvComments(csvFiles) {
+        // Note: modification of csv files is only allowed for files located in the compile destination folder,
+        // meaning having their origin location at db/data/* or db/csv/*
+        const stripComments = cds.utils.csv?.stripComments ?? require(path.join(cds.home, 'bin/build/csv-reader')).stripComments  // cds@6 compatibility
+
+        for (const file of csvFiles) {
+            if (this.isStagingBuild() || file.startsWith(this.task.options.compileDest)) {
+                await stripComments(file)
+            }
+        }
+    }
+
+    /**
+     * Creates the hana artifacts from the given csn model and writes the files to the folder '<dest>/src/gen'.
+     *
+     * @param {object} model The compiled csn model
+     */
+    async _compileToHana(model) {
+        // compile to old format (.hdbcds) or new format (.hdbtable / .hdbview)
+        const format = this.getBuildOption(DEPLOY_FORMAT) || cds.env.requires.db?.[DEPLOY_FORMAT] || cds.env.hana?.[DEPLOY_FORMAT]
+        if (!cds.compile.to[format]) {
+            return Promise.reject(new Error(`Invalid deploy-format defined: ${format}`))
+        }
+
+        if (this.hasCdsEnvOption('features.journal', false) || format === 'hdbcds') {
+            return await this._compileToHdb(model, format)
+        } else {
+            return await this._compileToHdbmigration(model, format)
+        }
+    }
+
+    async _compileToHdb(model, format) {
+        const hdiPlugins = new Set()
+        const relDest = path.relative(this.task.dest, this.task.options.compileDest)
+        // enforces sqlNames option for compiler in tests
+        const options = { ...this.options(), sql_mapping: cds.env.sql.names }
+        const result = cds.compile.to[format](model, options)
+        const promises = []
+
+        for (const [content, key] of result) {
+            hdiPlugins.add(key.suffix || path.extname(key.file))
+            const file = key.file ? key.file : key.name + key.suffix
+            if (this.hasBuildOption(OUTPUT_MODE, OUTPUT_MODE_RESULT)) {
+                this._result.hana.push(path.join(relDest, file))
+            }
+            promises.push(this.write(content).to(path.join(this.task.options.compileDest, file)))
+        }
+        await Promise.all(promises)
+        return hdiPlugins
+    }
+
+    async _compileToHdbmigration(model, format) {
+        const hdiPlugins = new Set()
+        const relDestDir = path.relative(this.task.dest, this.task.options.compileDest)
+        const relDbDestDir = path.relative(cds.root, this.task.options.compileDest)
+        const dbSrcDir = path.join(this.task.src, "src")
+        const relDbSrcDir = path.relative(cds.root, dbSrcDir)
+        const lastDevCsnFolder = PATH_LAST_DEV_CSN
+        const lastDevCsnDir = path.join(this.task.src, lastDevCsnFolder)
+        let lastDev = null
+        const promises = []
+        const migrationTableFiles = []
+
+        if (fs.existsSync(lastDevCsnDir)) {
+            lastDev = JSON.parse((await fs.promises.readFile(lastDevCsnDir, 'utf-8')).toString())
+        }
+        // enforces sqlNames option for compiler in tests, pass options from cds env, ensures that the correct format is taken
+        const options = { ...this.options(), logger: this.logger, sql_mapping: cds.env.sql.names, "deploy-format": format }
+
+        const compilationResult = await to_hdbmigration(model, lastDev, dbSrcDir, options)
+        const definitions = compilationResult.definitions
+        const afterImage = compilationResult.afterImage
+
+        for (const { name, suffix, content, changed } of definitions) {
+            hdiPlugins.add(suffix)
+            const file = name + suffix
+            if (suffix === FILE_EXT_HDBMIGRATIONTABLE) {
+                migrationTableFiles.push(path.join(relDbSrcDir, file))
+                if (changed) {
+                    if (this.hasBuildOption(OUTPUT_MODE, OUTPUT_MODE_RESULT)) {
+                        this._result.hana.push(path.join("src", file))
+                    }
+                    promises.push(this.write(content).to(path.join(dbSrcDir, file)))
+                } else {
+                    this.logger._debug && this.logger.debug(`no change, keep existing ${file}`)
+                }
+            } else {
+                if (this.hasBuildOption(OUTPUT_MODE, OUTPUT_MODE_RESULT)) {
+                    this._result.hana.push(path.join(relDestDir, file))
+                }
+                promises.push(this.write(content).to(path.join(this.task.options.compileDest, file)))
+                if (suffix === FILE_EXT_HDBTABLE) {
+                    // issue an error in case a .hdbmigrationtable file already exists
+                    if (fs.existsSync(path.join(dbSrcDir, name + FILE_EXT_HDBMIGRATIONTABLE))) {
+                        throw new BuildError(`Multiple files exist defining the same HANA artifact - [${path.join(relDbSrcDir, name + FILE_EXT_HDBMIGRATIONTABLE)}, ${path.join(relDbDestDir, file)}].\nEither annotate the model entity using @cds.persistence.journal or undeploy the file [${path.join('src', name + FILE_EXT_HDBMIGRATIONTABLE)}] using an undeploy.json file.`)
+                    }
+                }
+            }
+        }
+        await Promise.all(promises)
+        await this._validateMigrationTableFiles()
+
+        // update last development version
+        if (afterImage) {
+            if (migrationTableFiles.length > 0) {
+                if (!HanaModuleBuilder._toEqualIgnoreMeta(lastDev, afterImage)) {
+                    await this.write(afterImage).to(lastDevCsnDir)
+                }
+
+                // add src/.hdiconfig if not existing
+                if (!fs.existsSync(path.join(dbSrcDir, FILE_NAME_HDICONFIG))) {
+                    const template = await HanaModuleBuilder._readTemplateAsJson('.hdiconfig-cloud')
+                    await this.write(template).to(path.join(dbSrcDir, FILE_NAME_HDICONFIG))
+                }
+            }
+        } else {
+            throw new BuildError(`Inconsistent CDS compilation results - file ${lastDevCsnFolder} missing`)
+        }
+        return hdiPlugins
+    }
+
+    async _writePackageJson() {
+        const packageJson = path.join(this.task.src, "package.json")
+        const exists = fs.existsSync(packageJson)
+
+        if (exists) {
+            this.logger._debug && this.logger.debug(`skip create [${relativePaths(cds.root, packageJson)}], already existing`)
+        }
+        if (this.isStagingBuild() && !exists) {
+            const content = await HanaModuleBuilder._readTemplateAsJson(FILE_NAME_PACKAGE_JSON)
+            await this.write(content).to(path.join(this.task.dest, FILE_NAME_PACKAGE_JSON))
+        }
+    }
+
+    /**
+     * Create .hdiconfig file in <dest>src/gen folder of db module.
+     */
+    async _writeHdiConfig(compileHdiPlugins) {
+        const DEFAULT_HDI_PLUGINS = [FILE_EXT_CSV, FILE_EXT_HDBTABLEDATA, FILE_EXT_HDBTABLE, ".hdbview", ".hdbindex", ".hdbconstraint"]
+        const undeployHdiPlugins = await this._readTypesFromUndeployJson() // see CAP issue #6222
+        const hdiPlugins = new Set([...DEFAULT_HDI_PLUGINS, ...undeployHdiPlugins, ...compileHdiPlugins])
+        let hdiConfigPath
+
+        // do not create the .hdiconfig file in 'gen/db' root folder for now
+        // if (this.isStagingBuild()
+        //     && !fs.existsSync(path.join(this.task.dest, FILE_NAME_HDICONFIG))
+        //     && !fs.existsSync(path.join(this.task.dest, 'cfg', FILE_NAME_HDICONFIG))) {
+        //     // in 'gen/db' if none exists
+        //     // ensures correct deployment of all artifacts including HANA native artifacts 
+        //     // without the need of creating a static .hdiconfig file in the 'db' folder
+
+        //     // add all known plugins
+        //     const defaultHdiConfig = await HanaModuleBuilder._readTemplateAsJson('.hdiconfig-cloud')
+        //     for (const plugin in defaultHdiConfig['file_suffixes']) {
+        //         hdiPlugins.add('.' + plugin)
+        //     }
+        //     hdiConfigPath = path.join(this.task.dest, FILE_NAME_HDICONFIG)
+        // } else {
+        // in 'gen/db/src/gen'
+        // ensures correct deployment of generated and undeploy.json artifacts
+        // a static .hdiconfig file in 'db' folder is required for any HANA native artifacts
+        hdiConfigPath = path.join(this.task.options.compileDest, FILE_NAME_HDICONFIG)
+        // }
+
+        const template = await HanaModuleBuilder._readTemplateAsJson('.hdiconfig-all')
+        let content = {
+            'file_suffixes': {}
+        }
+        // only use the required subset as SAP HANA cloud does not support all hdiPlugins
+        for (const key in template['file_suffixes']) {
+            if (hdiPlugins.has('.' + key)) {
+                content['file_suffixes'][key] = template['file_suffixes'][key]
+            }
+        }
+        if (Object.keys(content['file_suffixes']).length !== hdiPlugins.size) {
+            this.pushMessage(`'HANA database plugin not found for file suffix [${Array.from(hdiPlugins).join(',')}]`)
+        }
+        await this.write(content).to(hdiConfigPath)
+    }
+
+    /**
+     * Create .hdinamespace file in <dest>src/gen folder of db module.
+     */
+    async _writeHdiNamespace() {
+        // see issue #64 - add .hdinamespace file to prevent HDI from adding gen/ folder to the namespace.
+        const hdiNamespace = path.join(this.task.options.compileDest, FILE_NAME_HDINAMESPACE)
+        const content = await HanaModuleBuilder._readTemplateAsJson(FILE_NAME_HDINAMESPACE)
+        return await this.write(content).to(hdiNamespace)
+    }
+
+    /**
+     * Create undeploy.json file in <dest> folder of db module.
+     */
+    async _writeUndeployJson() {
+        if (this.isStagingBuild()) {
+            // see issue #64 - add .hdinamespace file to prevent HDI from adding gen/ folder to the namespace.
+            const undeployJsonDest = path.join(this.task.dest, FILE_NAME_UNDEPLOY_JSON)
+            const undeployJsonSrc = path.join(this.task.src, FILE_NAME_UNDEPLOY_JSON)
+            const templateEntries = await HanaModuleBuilder._readTemplateAsJson(FILE_NAME_UNDEPLOY_JSON)
+            let newEntries = []
+            if (fs.existsSync(undeployJsonSrc)) {
+                newEntries = await JSON.parse((await fs.promises.readFile(undeployJsonSrc, 'utf-8')).toString())
+                newEntries = Array.isArray(newEntries) ? newEntries : []
+                templateEntries.forEach(entry => {
+                    if (!newEntries.includes(entry)) {
+                        newEntries.push(entry)
+                    }
+                })
+            } else {
+                newEntries = templateEntries
+            }
+            // formatted output
+            let content = '[\n'
+            for (let i = 0; i < newEntries.length; i++) {
+                content += `  "${newEntries[i]}"${i + 1 < newEntries.length ? ',' : ''}\n`
+            }
+            content += ']'
+            await this.write(content).to(undeployJsonDest)
+        }
+    }
+
+    async _readTypesFromUndeployJson() {
+        const hdiPlugins = new Set()
+        const file = path.join(this.task.src, "undeploy.json")
+
+        if (fs.existsSync(file)) {
+            const undeployList = JSON.parse((await fs.promises.readFile(file)).toString(), 'utf-8')
+            if (Array.isArray(undeployList)) {
+                const hdiconfig = await HanaModuleBuilder._readTemplateAsJson('.hdiconfig-all')
+                const keys = new Set(Object.keys(hdiconfig['file_suffixes']).map(key => '.' + key))
+                undeployList.forEach(entry => {
+                    const extName = path.extname(entry)
+                    if (extName && !hdiPlugins.has(extName)) {
+                        if (keys.has(extName)) {
+                            hdiPlugins.add(extName)
+                        } else {
+                            this.pushMessage(`Ignoring invalid entry '${entry}' in undeploy.json file`, WARNING)
+                        }
+                    }
+                })
+            }
+        }
+        return hdiPlugins
+    }
+
+    async _validateMigrationTableFiles() {
+        const dbSrcDir = path.join(this.task.src, "src")
+        const migrationTableFiles = BuildTaskHandlerInternal._find(dbSrcDir, (res) => {
+            return fs.statSync(res).isFile() && path.extname(res) === FILE_EXT_HDBMIGRATIONTABLE
+        })
+        if (migrationTableFiles.length > 0) {
+            const parser = require('./migrationtable')
+
+            await Promise.all(migrationTableFiles.map(async file => {
+                try {
+                    const tableModel = await parser.read(file)
+                    if (/^>>>>>/m.test(tableModel.migrations.toString())) {
+                        // as this is not a build error, we do not abort cds build, instead only log as error
+                        this.pushMessage(`Current model changes require manual resolution. See migration file ${path.relative(cds.root, file)} for further details.`, ERROR)
+                    }
+                } catch (e) {
+                    throw new Error(`${path.relative(cds.root, file)}: ${e.toString()}`, ERROR)
+                }
+            }))
+        }
+    }
+
+    static async _readTemplateAsJson(template) {
+        const content = await fs.promises.readFile(path.join(__dirname, 'template', template), 'utf-8')
+        return JSON.parse(content.toString())
+    }
+
+    static _toEqualIgnoreMeta(csn1, csn2) {
+        function toString(csn) {
+            return JSON.stringify(csn, (k, v) => {
+                if (v?.creator) {
+                    // make sure it's the compiler meta tag
+                    if (k === 'meta' && v.creator?.startsWith('CDS Compiler'))
+                        return
+                }
+                return v
+            })
+        }
+
+        if (csn1 === csn2) {
+            return true
+        }
+        if (!csn1 || !csn2) {
+            return false
+        }
+        return toString(csn1) === toString(csn2)
+    }
+}
+module.exports = HanaModuleBuilder