@sap/cds-dk 4.4.2 → 4.5.0

package/CHANGELOG.md

@@ -6,6 +6,30 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## Version 4.5.0 - 2021-09-23
+
+### Added
+
+- `cds import` supports import of functions, action, functionimport and actionimport from both OData v2 and v4 edmx files.
+
+### Changed
+
+- `cds init` uses latest `Maven Java archetype` version `1.18.1` for creating Java projects.
+
+### Fixed
+
+- `cds watch` additional CLI args were not passed to launched process, e.g. `--odata x4`.
+- `cds import` fix for TypeError issue during OData v4 edmx conversion to csn.
+- `cds import` fix for multi-line documentation text in OData v2 edmx file.
+- `cds import` fix for v4 csn generation when non standard OData vocabularies are referred.
+- `cds add data`
+    - no longer fails for entities without keys
+    - no longer creates csv files for synthetic draft entities
+    - no longer creates entries for virtual fields
+    - no longer skips over entities marked with `@cds.persistence.skip`, which is true for 'external' entities created by `cds import`.
+- `cds init --add` handles comma list correctly.
+
+
 ## Version 4.4.2 - 2021-09-03
 
 ### Fixed

package/bin/activate.js

@@ -49,8 +49,7 @@ async function activate ([projectFolder], options = {}) {
 
     Object.assign(options, { projectFolder })
 
-    const client = require('@sap/cds-sidecar-client')
-    await client.apply(injection, options)
+    await require('../lib/client/activate').run(injection, options);
 }
 
 

package/bin/deploy/index.js

@@ -71,7 +71,7 @@ async function deploy (_model, options) { // NOSONAR
 
   const conf = cds.env.requires.db || {}
   const [,_kind,_db] = /(\w+)?(?::(.*))?/.exec(url||'')
-  const kind = _kind || conf.use || conf.kind
+  const kind = _kind || conf.dialect || conf.use || conf.kind
 
   try {
     var deployer = require ('./to-'+kind)

package/bin/deploy/to-sqlite.js

@@ -10,9 +10,9 @@ exports.deploy = function deploy_to_sqlite (_model, _db, options) {
     .then (db => db && db.disconnect()) // REVISIT: we should NOT require to disconnect explicitly
 }
 
-function _database4  (_db,_dbrl,conf) {
+function _database4 (_db,_dbrl,conf) {
     if (_db)  return _db
     if (_dbrl === 'sqlite')  return 'sqlite.db'
     if (_dbrl === 'sql')  return ':memory:'
-    return conf.credentials && conf.credentials.database
+    if (conf.credentials) return conf.credentials.database || conf.credentials.url
 }

package/bin/extend.js

@@ -49,17 +49,15 @@ module.exports = Object.assign(extend, {
 
 
 async function extend ([url], options = {}) {
-    const client = require('@sap/cds-sidecar-client')
-
     if (options.force) {
-        if (! await client.question.askBooleanQuestion('Using option --force will overwrite existing files. Continue (yN)? ', false, false)) {
+        if (! await require('../lib/client/helper/question').askBooleanQuestion('Using option --force will overwrite existing files. Continue (yN)? ', false, false)) {
             console.log('Extend cancelled')
             return
         }
     }
 
     Object.assign(options, { url })
-    client.extend(options)
+    await require('../lib/client/extend').run(options);
 }
 
 

package/bin/login.js

@@ -5,9 +5,9 @@ module.exports = Object.assign(login, {
     shortcuts: ['-s', '-d', '-p', '-c'],
     flags: ['--plain'],
     get help() {
-        const client = require('@sap/cds-sidecar-client')
-         const cdsExtendAndCdsActivate = `${bold('cds extend')} and ${bold('cds activate')}`
-        const keyring = client.config.keyringDesignation
+        const config = require('../lib/client/settings_manager').SettingsManager.config
+        const cdsExtendAndCdsActivate = `${bold('cds extend')} and ${bold('cds activate')}`
+        const keyring = config.keyringDesignation
         return `
 # SYNOPSIS
 
@@ -42,13 +42,13 @@ module.exports = Object.assign(login, {
 
     Authentication data for all logins is saved by <app-url> and <tenant-subdomain> in the
     desktop keyring (${keyring} on your platform) or, if requested, in config
-    file ${client.config.paths.auth}.
+    file ${config.paths.auth}.
 
     To make use of keyring storage, the 'keytar' Node.js module must be installed.
     A corresponding hint will be given in case the module is not found.
 
     Project settings are saved by project directory in config file
-    ${client.config.paths.settings}.
+    ${config.paths.settings}.
 
     Note that local storage of authentication data incurs a certain security risk:
     a malicious locally-running application might be able to perform any action with
@@ -96,7 +96,7 @@ module.exports = Object.assign(login, {
 # SEE ALSO
 
     *cds logout* to remove any authentication data
-`;
+`
     }})
 
 
@@ -108,8 +108,7 @@ async function login ([url], options = {}) {
     })
     delete options.plain
 
-    const client = require('@sap/cds-sidecar-client')
-    client.login(options)
+    await require('../lib/client/auth_manager').loginAndUpdateSettings(options)
 }
 
 

package/bin/logout.js

@@ -56,8 +56,7 @@ module.exports = Object.assign(logout, {
 async function logout ([projectFolder], options = {}) {
     Object.assign(options, { projectFolder })
 
-    const client = require('@sap/cds-sidecar-client')
-    client.logout(options)
+    await require('../lib/client/auth_manager').logout(options)
 }
 
 

package/bin/repl.js

@@ -27,7 +27,7 @@ function repl (_, options={}) {
   const info = colors ? require('../lib/util/term').info : s => s
   log(info('Welcome to cds repl v'+ cds.version))
 
-  const cdsr = require('repl').start ({...options, writer:_writer, ignoreUndefined:true })
+  const cdsr = cds.repl = require('repl').start ({...options, writer:_writer, ignoreUndefined:true })
 
   cds.lazify (Object.assign (cdsr.context,{ cds,
     Foo: lazy => new cds.entity ({name:'Foo'}),

package/bin/watch.js

@@ -59,22 +59,26 @@ let ignore = /node_modules|\.cds-services\.json$/
 async function watch ([cwd], {
   args = ['serve', 'all', '--with-mocks'],
   ext = extDefaults,
-  ...etc
+  ...options
 }={}) {
 
-  args.push (`--${Object.keys(etc).find(a => /^in-memory\??$/.test(a)) || 'in-memory?'}`)
-  const env = Object.assign (process.env, { _args: JSON.stringify(args) })
+  args.push (`--${Object.keys(options).find(a => /^in-memory\??$/.test(a)) || 'in-memory?'}`)
+  args.push (...process.argv.slice(3).filter(a => a !== cwd))
+  const env = Object.assign (process.env, {
+    _args: JSON.stringify(args),
+    _options: JSON.stringify(options)
+  })
   if (typeof ext !== 'string')  ext = extDefaults
-  if ('port' in etc) env.PORT = etc.port
+  if ('port' in options) env.PORT = options.port
   env.NODE_PATH = path.resolve (__dirname,'../node_modules') // allow global sqlite to be resolved
 
   let liveReload //= ...
   let firstTime = true
-  if (!/false|no/.test(etc['livereload'])) {
+  if (!/false|no/.test(options['livereload'])) {
     ext += ',css,gif,html,jpg,png,svg'
     liveReload = new (require ('../lib/watch/livereload-server'))
     env.CDS_LIVERELOAD_PATH = require.resolve ('connect-livereload')
-    env.CDS_LIVERELOAD_URL = await liveReload.start (etc['livereload'])
+    env.CDS_LIVERELOAD_URL = await liveReload.start (options['livereload'])
       .catch(err => { console.log(`Error starting live reload: ${err.message}`); liveReload = null })
   } else {
     ignore = RegExp(`app${path.sep}.*\\.(js|ts)|`+ignore.source) // exclude more frontend stuff
@@ -99,7 +103,7 @@ async function watch ([cwd], {
     watch: ['*','.env','.cdsrc.json'],
     script:__filename, cwd, env, ext, ignore,
     delay: parseInt(process.env.CDS_WATCH_DELAY) || 200,
-    options: etc
+    options
   }).on('restart', (files)=>{
     DEBUG && DEBUG (files)
     if (liveReload) liveReload.markPending(files)
@@ -115,7 +119,7 @@ async function watch ([cwd], {
     case 'listening':
       env.PORT = (msg.address && msg.address.port) || msg.port
       liveReload && liveReload.reload()
-      firstTime && etc['open'] && require('../lib/watch/open').openURL (etc['open'], env.PORT)
+      firstTime && options['open'] && require('../lib/watch/open').openURL (options['open'], env.PORT)
       firstTime = false
       break
     case 'EADDRINUSE': env.PORT = 0; _addr_in_use(msg); break

package/lib/client/activate.js

@@ -0,0 +1,243 @@
+const path = require('path');
+const axios = require('axios');
+const { fse } = require('@sap/cds-foss');
+
+const AuthManager = require('./auth_manager');
+const BaseCommand = require('./base_command');
+const { collectFileContent, collectFiles, toUnixPath } = require('./helper/fs_helper');
+const Logger = require('./helper/logging');
+
+const COMMAND_URL_SUFFIX = 'asyncActivate';
+const UPDATE_CUSTOM_FILE_URL_SUFFIX = 'updateCustomTenantContent';
+
+const I18N_WHITELIST = ['.properties', '.csv'];
+
+const JOB_STATUS = {
+    QUEUED: 'QUEUED',
+    RUNNING: 'RUNNING',
+    FINISHED: 'FINISHED',
+    FAILED: 'FAILED'
+};
+
+async function checkFinished(jobId, statusUrl, params, finisher) {
+    const requestOptions = Object.assign({
+        method: 'GET'
+    }, params.get('reqAuth'));
+
+    const { status, error } = (await axios(statusUrl, requestOptions)).data;
+
+    if ((status === JOB_STATUS.FINISHED || status === JOB_STATUS.FAILED) && !finisher.finished) {
+        finisher.finished = true;
+        clearInterval(finisher.timer);
+        if (status === JOB_STATUS.FAILED) {
+            return finisher.reject(new Error(`Job ${jobId} failed: ${error.message || JSON.stringify(error)}`));
+        }
+        Logger.log('Extension active');
+        finisher.resolve();
+    }
+}
+
+module.exports = class Activate extends BaseCommand {
+
+    static get COMMAND_URL_SUFFIX() { return COMMAND_URL_SUFFIX; }
+    static get UPDATE_CUSTOM_FILE_URL_SUFFIX() { return UPDATE_CUSTOM_FILE_URL_SUFFIX; }
+
+    static async run(injection, paramValues) {
+        try {
+            const params = await AuthManager.loginAndUpdateSettings(paramValues);
+
+            if (process.env.DEBUG) {
+                Logger.log('Activating extension with parameters ' + JSON.stringify({
+                    appUrl: params.get('appUrl'),
+                    projectFolder: params.get('projectFolder'),
+                    undeploy: params.get('undeploy')
+                }));
+            } else {
+                Logger.log(`Activating extension located at ${params.get('projectFolder')} to URL ${params.get('appUrl')}`);
+            }
+
+            if (!await Activate.checkProject(params.get('projectFolder'))) {
+                throw new Error(params.get('projectFolder') + ' is not a valid extension project');
+            }
+
+            try {
+                await Activate.runActivate(injection.cds, params);
+                if (params.get('wsdl')) {
+                    await Activate.runCustomFileUpload(params);
+                }
+            } catch (error) {
+                Activate.handleHttpError(error, params);
+            }
+        } catch (error) {
+            Logger.error(undefined, error);
+        }
+    }
+
+    static async collectExtensionFiles(cds, projectFolder) {
+
+        const absoluteFolder = projectFolder;
+        const dbFolder = path.join(absoluteFolder, 'db');
+        const srvFolder = path.join(absoluteFolder, 'srv');
+
+        const resolvedPathDb = cds.resolve(dbFolder, { root: dbFolder });
+        const resolvedPathSrv = cds.resolve(srvFolder, { root: srvFolder });
+
+        let resolvedPaths = [];
+        if (resolvedPathSrv) {
+            resolvedPaths = resolvedPaths.concat(resolvedPathSrv);
+        }
+
+        if (resolvedPathDb) {
+            resolvedPaths = resolvedPaths.concat(resolvedPathDb);
+        }
+
+        let allFiles = new Map();
+
+        if (resolvedPaths.length < 1) {
+            Logger.log('No extension files found');
+            return allFiles;
+        }
+
+        const csn = await cds.load(resolvedPaths);
+        const sources = csn.$sources || csn._sources;
+
+        const filteredSources = sources && sources.filter((entry) => {
+            if (! entry.startsWith(projectFolder)) {
+                /* Current file might be below the base model's node_modules, in which case compilation on server should succeed.
+                   Otherwise, compilation will probably fail, with this message giving a hint. */
+                Logger.debug(`Source file outside project folder will not be uploaded: ${entry}`);
+                return false;
+            }
+            const unixFilename = toUnixPath(entry);
+            return !/.*node_modules\/_base\/.*/.test(unixFilename);
+        }) || [];
+
+        const sourceFiles = await collectFileContent(filteredSources, projectFolder);
+        allFiles = new Map([...allFiles, ...sourceFiles]);
+
+        const modelFiles4I18n = { $sources: filteredSources };
+
+        let i18nFolders = cds.localize.folders4(modelFiles4I18n);
+        i18nFolders = i18nFolders.filter(i18nFolder => {
+            const unixFilename = toUnixPath(i18nFolder);
+            return !(/.*node_modules\/_base\/.*/.test(unixFilename));
+        });
+
+        i18nFolders.forEach(async(folder) => {
+            const i18nFiles = collectFiles(projectFolder, folder, entry => {
+                const ext = path.extname(entry);
+                return I18N_WHITELIST.includes(ext);
+            });
+            allFiles = new Map([...allFiles, ...i18nFiles]);
+        });
+
+        try {
+            const pkgJson = await fse.readFile(path.join(absoluteFolder, 'package.json'), 'utf-8');
+            allFiles.set('package.json', pkgJson);
+        } catch(e) {
+            Logger.log('No package.json file found');
+            Logger.error('No package.json file found', e);
+        }
+
+        return allFiles;
+    }
+
+    static collectCustomFiles(projectFolder) {
+        const wsdlFolder = path.join(projectFolder, 'wsdl');
+
+        try {
+            const wsdlFiles = collectFiles(projectFolder, wsdlFolder);
+            if (wsdlFiles.size === 0) {
+                Logger.log('No custom files found');
+            }
+            return wsdlFiles;
+        } catch (error) {
+            Logger.error(undefined, error);
+            throw new Error('No custom files found');
+        }
+    }
+
+    static async runActivate(cds, params) {
+        let activationUrl;
+
+        return Activate.collectExtensionFiles(cds, params.get('projectFolder'))
+        .then(allFiles => {
+            if (allFiles.size === 0) {
+                return null;
+            }
+
+            Logger.log('Activating extension...');
+
+            activationUrl = Activate.getMtxApiUrl(params.get('appUrl'), COMMAND_URL_SUFFIX);
+            const requestOptions = Object.assign({
+                method: 'POST',
+                headers: {},
+                data: {
+                    extension: [...allFiles],
+                    undeployExtension: params.get('undeploy')
+                }
+            }, params.get('reqAuth'));
+            requestOptions.headers['Content-Type'] = 'application/json';
+
+            return axios(activationUrl, requestOptions);
+        })
+        .then(response => {
+            if (response === null) {
+                return;
+            }
+
+            const { jobID: jobId, statusPath } = response.data;
+
+            if (jobId === undefined) {
+                throw new Error('Activation failed: did not receive job ID');
+            }
+
+            if (!statusPath) {
+                throw new Error(`Job ${jobId} failed: did not receive job-status path`);
+            }
+
+            Logger.debug(`Job ID: ${jobId}`);
+
+            return new Promise((resolve, reject) => {
+                const statusUrl = Activate.concatUrls(params.get('appUrl'), statusPath);
+                const finisher = {
+                    resolve,
+                    reject,
+                    finished: false,
+                    timer: null
+                };
+                finisher.timer = setInterval(checkFinished.bind(undefined, jobId, statusUrl, params, finisher), 250);
+            });
+        });
+    }
+
+    static async runCustomFileUpload(params) {
+        Logger.log(`Uploading custom files to ${params.get('appUrl')}...`);
+
+        const customFiles = Activate.collectCustomFiles(params.get('projectFolder'));
+
+        const customFilesSerialized = [];
+        customFiles.forEach((value, key) => {
+            customFilesSerialized.push({
+                category: "wsdl",
+                name: key,
+                content: value
+            });
+        });
+
+        const customFileUrl = Activate.getMtxApiUrl(params.get('appUrl'), Activate.UPDATE_CUSTOM_FILE_URL_SUFFIX);
+        const requestOptions = Object.assign({
+            method: 'POST',
+            headers: {},
+            data: {
+                content: customFilesSerialized
+            }
+        }, params.get('reqAuth'));
+        requestOptions.headers['Content-Type'] = 'application/json';
+
+        await axios(customFileUrl, requestOptions);
+
+        Logger.log('Custom file upload finished');
+    }
+
+}

package/lib/client/auth_manager.js

@@ -0,0 +1,115 @@
+const util = require('util');
+const axios = require('axios');
+
+const Logger = require('./helper/logging');
+const { SettingsManager } = require('./settings_manager');
+const { ParamCollection } = require('./params');
+
+function addTokenInfo(params, authData) {
+    params.set('token', authData.access_token);
+    if (authData.expires_in) {
+        params.set('tokenExpirationDate', Date.now() + authData.expires_in * 1000);
+    } else {
+        params.delete('tokenExpirationDate');
+    }
+    if (authData.refresh_token) {
+        params.set('refreshToken', authData.refresh_token);
+    } else {
+        params.delete('refreshToken');
+    }
+}
+function addUrlInfo(params, authData) {
+    params.set('passcodeUrl', authData.passcode_url);
+}
+function getTokenUrl(params) {
+    const tokenUrl = new URL(params.get('tokenUrl'));
+    tokenUrl.search = `subdomain=${params.get('subdomain')}`;
+
+    if (params.has('refreshToken')) {
+        tokenUrl.search += `&refresh_token=${params.get('refreshToken')}`;
+    } else if (params.has('passcode')) {
+        tokenUrl.search += `&passcode=${params.get('passcode')}`;
+    }
+    if (params.has('clientid')) {
+        tokenUrl.search += `&clientid=${params.get('clientid')}&clientsecret=${params.get('clientsecret')}`;
+    }
+    return tokenUrl.toString();
+}
+
+async function retrieveTokenOrPasscodeUrl(params) {
+    if (params.has('token')) {
+        if (params.get('renewLogin') || params.get('tokenExpirationDate') <= Date.now()) {
+            Logger.debug((params.get('renewLogin') ? 'Renewing' : 'Refreshing expired') + ' authentication token');
+            params.delete('token');
+            params.delete('tokenExpirationDate');
+        } else {
+            return;
+        }
+    }
+    if (!params.has('passcode') && !params.has('refreshToken') && params.has('passcodeUrl')) {
+        return;
+    }
+
+    let retry;
+    let authData;
+    do {
+        const tokenUrl = getTokenUrl(params);
+        Logger.debug(`Getting authentication token or passcode URL from ${(params.obfuscateQueryParams(tokenUrl))}`);
+
+        let response;
+        try {
+            response = await axios(tokenUrl.toString(), { method: 'GET' });
+        } catch (error) {
+            throw new Error(`Failed to get authentication token: ${error}.\nResponse body: ${util.inspect(response && response.data)}`);
+        }
+        authData = response.data;
+
+        retry = !authData.access_token && params.has('refreshToken') && (params.delete('refreshToken'), true);
+    } while (retry);
+
+    if (authData.access_token) {
+        addTokenInfo(params, authData);
+    } else {
+        addUrlInfo(params, authData);
+    }
+}
+
+
+module.exports = class AuthManager {
+
+    static async loginAndUpdateSettings(paramValues) {
+        SettingsManager.init();
+        const params = new ParamCollection(paramValues);
+        await SettingsManager.loadAndMergeSettings(params);
+        if (params.has('username')) {
+            params.set('reqAuth', { auth: { username: params.get('username'), password: params.get('password') } });
+        } else {
+            await retrieveTokenOrPasscodeUrl(params);
+            if (!params.has('token')) {
+                await SettingsManager.saveSettings(params);
+                throw new Error('Failed to login: no valid token or passcode provided. ' +
+                    'Get a passcode' + (params.get('passcodeUrl') ? ' from ' + params.get('passcodeUrl') : '') +
+                    ' and supply the code when re-running the command');
+            }
+            params.set('reqAuth', { headers: { Authorization: 'Bearer ' + params.get('token') } });
+        }
+
+        await SettingsManager.saveSettings(params); // saves token conditionally
+
+        return params;
+    }
+
+    static async logout(paramValues) {
+        SettingsManager.init();
+        const params = new ParamCollection(paramValues);
+        if (params.get('clearInvalid')) {
+            await SettingsManager.deleteInvalid();
+        } else {
+            await SettingsManager.deleteToken(params);
+            if (params.get('deleteSettings')) {
+                await SettingsManager.deleteSettingsWithoutToken(params);
+            }
+        }
+    }
+
+}

package/lib/client/base_command.js

@@ -0,0 +1,55 @@
+const path = require('path');
+const { fse } = require('@sap/cds-foss');
+
+const Logger = require('./helper/logging');
+
+const MTX_URL_PATH_PREFIX = 'mtx/v1/model/';
+const CUSTOM_FILE_URL_SUFFIX = 'customTenantContent';
+
+module.exports = class BaseCommand {
+
+    static get CUSTOM_FILE_URL_SUFFIX() { return CUSTOM_FILE_URL_SUFFIX; }
+
+    static async checkProject(folder) {
+        return fse.pathExists(path.join(folder, 'package.json'));
+    }
+
+    static async handleHttpError(error, params) {
+        if (!('isAxiosError' in error)) {
+            throw error;
+        }
+
+        switch (error.response.status) {
+            case 401:
+                Logger.error('Request failed: authorization might have expired. Please retry with valid passcode' +
+                    (params.has('passcodeUrl') ? ` from ${params.get('passcodeUrl')}` : ''), error);
+                break;
+            case 404:
+                Logger.error(`Request failed. Please check application URL ${params.get('appUrl')}`, error);
+                break;
+            default:
+                // Including 422 (compilation or native-extension error).
+                Logger.error(undefined, error);
+        }
+    }
+
+    static concatUrls(base, ...paths) {
+        function trimBeginSlash(str) {
+            return str.replace(/^\//, '');
+        }
+        function trimEndSlash(str) {
+            return str.replace(/\/$/, '');
+        }
+        function trimSlashes(str) {
+            return trimBeginSlash(trimEndSlash(str));
+        }
+        const concatenated = new URL(base);
+        concatenated.pathname = trimEndSlash(concatenated.pathname) + '/' +
+            paths.map((p, i) => i < paths.length - 1 ? trimSlashes(p) : trimBeginSlash(p)).join('/');
+        return concatenated.toString();
+    }
+
+    static getMtxApiUrl(url, action) {
+        return BaseCommand.concatUrls(url, MTX_URL_PATH_PREFIX, action);
+    }
+}