@sap/cds-compiler 4.1.2 → 4.2.4

package/lib/utils/moduleResolve.js

@@ -1,7 +1,15 @@
 // Custom resolve functionality for the CDS compiler
 //
 // See `internalDoc/ModuleResolution.md` for details on the algorithm.
+// See also <https://cap.cloud.sap/docs/cds/cdl#model-resolution>.
 // The algorithm is based on NodeJS's `require()`.
+//
+// For debugging purpose, if the option variable `options.traceFs` is set,
+// we log file lookups.  This makes it easier to debug why a file was not
+// found when setting custom search directories.
+//
+// You can set this option via your `.cdsrc.json` or `package.json`:
+// `{ "cds": { "cdsc": { "traceFs": true } } }`.
 
 'use strict';
 
@@ -14,18 +22,24 @@ const DEFAULT_ENCODING = 'utf-8';
 /**
  * Default lookup-extensions.  If a module "./index" is requested, then
  * "./index.cds" is checked first, then "index.csn" and so on.
+ *
+ * Keep in sync with documentation!
  */
 const extensions = [ '.cds', '.csn', '.json' ];
+/**
+ * Default module-lookup directories.  Used when resolving modules.
+ * Since version v4.2, this option can be configured.  Before that,
+ * only node_modules/ was a valid module-lookup directory.
+ *
+ * @type {string[]}
+ */
+const defaultLookupDirectories = [ 'node_modules/' ];
 
 /**
- * A global cds.home configuration can be set that forces the cds-compiler to
- * use a certain directory for all @sap/cds/ includes.
+ * A global `cds.home` or local `options.cdsHome` configuration can be set that
+ * forces the cds-compiler to use a certain directory for all @sap/cds/ includes.
  * This function handles such module paths.
  *
- * @todo Re-think:
- *   - Why can't a JAVA installation set a (symbolic) link?
- *   - Preferred to a local installation? Not the node-way!
- *
  * @param {string} modulePath
  * @param {CSN.Options} options
  * @returns {string}
@@ -43,127 +57,193 @@ function adaptCdsModule( modulePath, options = {} ) {
 }
 
 /**
- * @param {object} dep
- * @param {object} fileCache
+ * Create the module resolver, namely `resolveModule(dep)`.
+ *
  * @param {CSN.Options} options
+ * @param {object} fileCache
  * @param {object} messageFunctions
  */
-function resolveModule( dep, fileCache, options, messageFunctions ) {
+function makeModuleResolver( options, fileCache, messageFunctions ) {
   const _fs = cdsFs(fileCache, options.traceFs);
-  // let opts = { extensions, basedir: dep.basedir, preserveSymlinks: false };
-  // `preserveSymlinks` option does not really work -> provide workaround anyway...
-  // Hm, the resolve package also does not follow the node recommendation:
-  // "Using fs.stat() to check for the existence of a file before calling
-  // fs.open(), fs.readFile() or fs.writeFile() is not recommended"
+  /** @type {ResolveConfig} */
   const opts = {
     extensions,
-    basedir: dep.basedir,
     isFile: _fs.isFile,
     readFile: _fs.readFile,
     realpath: _fs.realpath,
+    lookupDirs: _getLookupDirectories( options, messageFunctions ),
   };
-  return new Promise( (fulfill, reject) => {
-    const lookupPath = adaptCdsModule(dep.module, options);
-    resolveCDS( lookupPath, opts, (err, res) => {
-      // console.log('RESOLVE', dep, res, err)
-      if (err) {
-        reject(err);
-      }
-      else {
-        const body = fileCache[res];
-        if (body === undefined || body === true) { // use fs if no or just temp entry
-          dep.absname = res;
-          _fs.realpath( res, cb );
+
+  return {
+    resolveModule,
+  };
+
+  function resolveModule( dep ) {
+    return new Promise( (fulfill, reject) => {
+      const lookupPath = adaptCdsModule( dep.module, options );
+      _resolveCDS( lookupPath, dep.basedir, opts, (err, res) => {
+        // console.log('RESOLVE', dep, res, err)
+        if (err) {
+          reject(err);
         }
-        else if (body && typeof body === 'object' && body.realname) {
-          // dep.absname = body.realname;
-          cb( null, body.realname ); // use fs.realpath name
+        else {
+          const body = fileCache[res];
+          if (body === undefined || body === true) { // use fs if no or just temp entry
+            dep.absname = res;
+            _fs.realpath( res, cb );
+          }
+          else if (body && typeof body === 'object' && body.realname) {
+            // dep.absname = body.realname;
+            cb( null, body.realname ); // use fs.realpath name
+          }
+          else {
+            // dep.absname = res;
+            cb( null, res );
+          }
+        }
+      });
+
+      function cb( err, res ) {
+        if (err) {
+          reject(err);
         }
         else {
-          // dep.absname = res;
-          cb( null, res );
+          if (dep.absname)
+            fileCache[dep.absname] = (dep.absname === res) || { realname: res };
+          dep.resolved = res;   // store in dep that module resolve was successful
+          for (const from of dep.usingFroms)
+            from.realname = res;
+          fulfill(res);
         }
       }
+    }).catch( () => {
+      _errorFileNotFound(dep, options, messageFunctions);
+      return false;
     });
-
-    function cb( err, res ) {
-      if (err) {
-        reject(err);
-      }
-      else {
-        if (dep.absname)
-          fileCache[dep.absname] = (dep.absname === res) || { realname: res };
-        dep.resolved = res;   // store in dep that module resolve was successful
-        for (const from of dep.usingFroms)
-          from.realname = res;
-        fulfill(res);
-      }
-    }
-  }).catch( () => {
-    _errorFileNotFound(dep, options, messageFunctions);
-    return false;
-  });
+  }
 }
 
 
 /**
- * @param {object} dep
- * @param {object} fileCache
+ * Create the synchronous module resolver, namely `resolveModuleSync(dep)`.
+ *
  * @param {CSN.Options} options
+ * @param {object} fileCache
  * @param {object} messageFunctions
  */
-function resolveModuleSync( dep, fileCache, options, messageFunctions ) {
+function makeModuleResolverSync( options, fileCache, messageFunctions ) {
   const _fs = cdsFs(fileCache, options.traceFs);
+  /** @type {ResolveConfig} */
   const opts = {
     extensions,
-    basedir: dep.basedir,
     isFile: _fs.isFileSync,
     readFile: _fs.readFileSync,
     realpath: _fs.realpathSync,
+    lookupDirs: _getLookupDirectories( options, messageFunctions ),
   };
 
-  let result = null;
-  let error = null;
-  const lookupPath = adaptCdsModule(dep.module, options);
-
-  resolveCDS( lookupPath, opts, (err, res) => {
-    if (err)
-      error = err;
-    if (res)
-      result = res;
-  });
+  return {
+    resolveModuleSync,
+  };
 
-  if (error) {
-    _errorFileNotFound(dep, options, messageFunctions);
-    return false;
-  }
+  function resolveModuleSync( dep ) {
+    let result = null;
+    let error = null;
+    const lookupPath = adaptCdsModule(dep.module, options);
 
-  const body = result ? fileCache[result] : undefined;
-  if (body === undefined || body === true) { // use fs if no or just temp entry
-    dep.absname = result;
-    _fs.realpathSync( result, (err, modulePath) => {
+    _resolveCDS( lookupPath, dep.basedir, opts, (err, res) => {
       if (err)
         error = err;
-      else
-        result = modulePath;
+      if (res)
+        result = res;
     });
+
+    if (error) {
+      _errorFileNotFound(dep, options, messageFunctions);
+      return false;
+    }
+
+    const body = result ? fileCache[result] : undefined;
+    if (body === undefined || body === true) { // use fs if no or just temp entry
+      dep.absname = result;
+      _fs.realpathSync( result, (err, modulePath) => {
+        if (err)
+          error = err;
+        else
+          result = modulePath;
+      });
+    }
+    else if (body && typeof body === 'object' && body.realname) {
+      result = body.realname;
+    }
+
+    if (error) {
+      _errorFileNotFound(dep, options, messageFunctions);
+      return false;
+    }
+
+    if (dep.absname)
+      fileCache[dep.absname] = (dep.absname === result) || { realname: result };
+    dep.resolved = result;   // store in dep that module resolve was successful
+    for (const from of dep.usingFroms)
+      from.realname = result;
+
+    return result;
   }
-  else if (body && typeof body === 'object' && body.realname) {
-    result = body.realname;
+}
+
+/**
+ * Get a list of module-lookup directories and ensure that user-provided lookup
+ * directories match our expectations / validate them.
+ *
+ * In case of errors (e.g. invalid options), returns the default list.
+ *
+ * @param {CSN.Options} options
+ * @param {object} messageFunctions
+ * @return {string[]}
+ */
+function _getLookupDirectories( options, messageFunctions ) {
+  const dirs = options?.moduleLookupDirectories || defaultLookupDirectories;
+
+  if (!Array.isArray(dirs)) {
+    messageFunctions.error('api-invalid-option', null, {
+      '#': 'type',
+      option: 'moduleLookupDirectories',
+      value: 'string[]',
+      othervalue: typeof options.moduleLookupDirectories,
+    });
+    return defaultLookupDirectories;
   }
 
-  if (error) {
-    _errorFileNotFound(dep, options, messageFunctions);
-    return false;
+  if (!dirs.includes('node_modules/')) {
+    // Special case of call-side-errors and to ensure old behavior.
+    dirs.push('node_modules/');
   }
 
-  if (dep.absname)
-    fileCache[dep.absname] = (dep.absname === result) || { realname: result };
-  dep.resolved = result;   // store in dep that module resolve was successful
-  for (const from of dep.usingFroms)
-    from.realname = result;
+  for (const dir of dirs) {
+    if (!dir.endsWith('/')) {
+      messageFunctions.error('api-invalid-lookup-dir', null, {
+        '#': 'slash',
+        option: 'moduleLookupDirectories',
+        value: dir,
+        othervalue: '/',
+      } );
+      return defaultLookupDirectories;
+    }
+    if (dir.startsWith('./') || dir.startsWith('../')) {
+      // Avoid relative directories, as we don't want to give the impression that they
+      // are resolved relative to the caller.
+      messageFunctions.error('api-invalid-lookup-dir', null, {
+        '#': 'relative',
+        option: 'moduleLookupDirectories',
+        value: dir,
+        othervalue: './',
+      });
+      return defaultLookupDirectories;
+    }
+  }
 
-  return result;
+  return dirs;
 }
 
 function _errorFileNotFound( dep, options, { error } ) {
@@ -201,16 +281,18 @@ function _errorFileNotFound( dep, options, { error } ) {
  * with existing code.  This may change at a later point.
  *
  * @param {string} moduleName Module to load, e.g. `./Include.cds` or `@sap/cds/common`.
- * @param {ResolveOptions} options
+ * @param {ResolveConfig} config
+ * @param {string} baseDir
  * @param {(err, result) => void} callback
  */
-function resolveCDS( moduleName, options, callback ) {
+function _resolveCDS( moduleName, baseDir, config, callback ) {
   const isWindows = (process.platform === 'win32');
-  let resolvedBaseDir = path.resolve(options.basedir);
+  let resolvedBaseDir = path.resolve(baseDir);
+  const lookupDirs = [ ...config.lookupDirs ];
 
   // NodeJS does not preserve symbolic links when resolving modules.
   // So neither do we.
-  options.realpath(resolvedBaseDir, (realPathErr, realPath) => {
+  config.realpath(resolvedBaseDir, (realPathErr, realPath) => {
     // There may be an error in resolving the symlink.
     // We ignore the error and simply use the original path.
     // Otherwise, cds-lsp tests would fail because they don't have real
@@ -224,7 +306,7 @@ function resolveCDS( moduleName, options, callback ) {
     if (isLocalFile(moduleName))
       loadFromLocalFileOrDirectory();
     else
-      loadNodeModules(resolvedBaseDir);
+      loadAsModule(resolvedBaseDir);
   }
 
   /**
@@ -269,7 +351,7 @@ function resolveCDS( moduleName, options, callback ) {
    * @param {(err, filepath: string|null) => void} cb
    */
   function loadAsFile( absoluteModulePath, cb ) {
-    const extensionsToTry = [ '' ].concat(options.extensions);
+    const extensionsToTry = [ '' ].concat(config.extensions);
     loadFileWithExtensions(extensionsToTry);
 
     /**
@@ -284,7 +366,7 @@ function resolveCDS( moduleName, options, callback ) {
         return;
       }
       const file = absoluteModulePath + exts.shift();
-      options.isFile(file, (err, foundAndIsFile) => {
+      config.isFile(file, (err, foundAndIsFile) => {
         if (!err && foundAndIsFile)
           cb(null, file);
         else
@@ -332,28 +414,29 @@ function resolveCDS( moduleName, options, callback ) {
   }
 
   /**
-   * Try to load the module from a node_modules directory.
-   * Start at absoluteDir and go through all parent directories.
+   * Try to load the module from a specified module-lookup directory such
+   * as `node_modules/`.
+   * Start at `absoluteDir` and go through all parent directories.
    *
    * @param {string} absoluteDir
    */
-  function loadNodeModules( absoluteDir ) {
-    const dirs = nodeModulesPaths(absoluteDir);
-    loadFromNodeDirs(dirs);
+  function loadAsModule( absoluteDir ) {
+    const dirGen = modulePaths(absoluteDir);
+    loadNextDir();
 
-    function loadFromNodeDirs( nodeDirs ) {
-      const dir = nodeDirs.shift();
-      if (!dir) {
+    function loadNextDir() {
+      const dir = dirGen.next();
+      if (dir.done) {
         // We're at root
         callback(makeNotFoundError(), null);
         return;
       }
-      const file = path.join(dir, moduleName);
+      const file = path.join(dir.value, moduleName);
       loadAsLocalFileOrDirectory(file, (err, filepath) => {
         if (!err && filepath)
           callback(null, filepath);
         else
-          loadFromNodeDirs(nodeDirs);
+          loadNextDir();
       });
     }
   }
@@ -368,7 +451,7 @@ function resolveCDS( moduleName, options, callback ) {
   function loadAndParsePackageJsonInDirectory( packageDir, cb ) {
     const file = path.join(packageDir, 'package.json');
 
-    options.readFile(file, DEFAULT_ENCODING, (err, content) => {
+    config.readFile(file, DEFAULT_ENCODING, (err, content) => {
       if (err) {
         cb(err, null);
         return;
@@ -385,16 +468,14 @@ function resolveCDS( moduleName, options, callback ) {
 
   /**
    * Get a list of all `node_modules` directories that MAY exist.
-   * Starting from absoluteStart upwards until at root.
+   * Starting from `absoluteStart` upwards until at root.
    *
-   * @param {string} absoluteStart
-   * @returns {string[]} Array of possible "node_modules" folders for the given path.
+   * @param {string} absoluteStart   *
+   * @return {Generator<string>} Possible module directories for the given path.
    */
-  function nodeModulesPaths( absoluteStart ) {
+  function* modulePaths( absoluteStart ) {
     // Use platform-dependent separator.  All NodeJS `path` methods use the system's path separator.
     const parts = absoluteStart.split(path.sep);
-    // Do NOT use global node_modules directories.
-    const dirs = [];
 
     // If we're on *nix systems, the first part is just an empty string ''
     // because the path is absolute.  Re-add it here because `path.join()`
@@ -403,12 +484,18 @@ function resolveCDS( moduleName, options, callback ) {
       parts[0] = '/';
 
     for (let i = parts.length - 1; i >= 0; i--) {
-      if (parts[i] === 'node_modules')
-        continue;
-      const dir = path.join(...parts.slice(0, i + 1), 'node_modules');
-      dirs.push(dir);
+      for (let j = 0; j < lookupDirs.length; ++j) {
+        const dir = lookupDirs[j];
+        if (dir && path.isAbsolute(dir)) {
+          // Only look up absolute paths once.
+          lookupDirs[j] = null;
+          yield dir;
+        }
+        else if (dir && parts[i] && !lookupDirs.includes(`${ parts[i] }/`)) {
+          yield path.join(...parts.slice(0, i + 1), dir);
+        }
+      }
     }
-    return dirs;
   }
 
   /**
@@ -417,7 +504,7 @@ function resolveCDS( moduleName, options, callback ) {
    * @returns {Error}
    */
   function makeNotFoundError() {
-    const moduleError = new Error(`Can't find module '${ moduleName }' from '${ options.basedir }'`);
+    const moduleError = new Error(`Can't find module '${ moduleName }' from '${ config.basedir }'`);
     // eslint-disable-next-line
     moduleError['code'] = 'MODULE_NOT_FOUND';
     return moduleError;
@@ -437,7 +524,7 @@ function isLocalFile( moduleName ) {
 }
 
 /**
- * Get the cds.main entry of the package.json
+ * Get the `cds.main` entry of the package.json
  * @param {object} pkg
  */
 function packageCdsMain( pkg ) {
@@ -447,8 +534,9 @@ function packageCdsMain( pkg ) {
 }
 
 /**
- * @typedef {object} ResolveOptions
- * @property {string} basedir
+ * @typedef {object} ResolveConfig
+ * @property {string[]} lookupDirs
+ *     Directories to look in for modules, e.g. node_modules/.
  * @property {string[]} extensions
  * @property {(path: string, callback: (err, foundAndIsFile) => void) => void} isFile
  * @property {(path: string, encoding, callback: (err, content) => void) => void} readFile
@@ -457,10 +545,11 @@ function packageCdsMain( pkg ) {
  */
 
 module.exports = {
-  resolveModule,
-  resolveModuleSync,
+  makeModuleResolver,
+  makeModuleResolverSync,
   // exported for unit tests
-  resolveCDS,
+  _resolveCDS,
+  _getLookupDirectories,
   isLocalFile,
   extensions,
 };

package/lib/utils/objectUtils.js

@@ -15,7 +15,7 @@ function copyPropIfExist( sourceObj, property, targetObj ) {
 
 /**
  * Takes an object and creates a dictionary out of it.
- * This avoid cases where e.g. properties named "toString" are interpreted
+ * This avoids cases where e.g. properties named "toString" are interpreted
  * as JS internal functions.
  *
  * @param {object} obj Object with prototype.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sap/cds-compiler",
-  "version": "4.1.2",
+  "version": "4.2.4",
   "description": "CDS (Core Data Services) compiler and backends",
   "homepage": "https://cap.cloud.sap/",
   "author": "SAP SE (https://www.sap.com)",
@@ -22,15 +22,15 @@
     "testci": "node scripts/verifyGrammarChecksum.js && mocha --reporter-option maxDiffSize=0 scripts/testLazyLoading.js && mocha --parallel --reporter-option maxDiffSize=0 test/ test3/",
     "testverbose": "node scripts/verifyGrammarChecksum.js && mocha --parallel test/ test3/",
     "test3": "node scripts/verifyGrammarChecksum.js && mocha --reporter-option maxDiffSize=0 test3/",
-    "deployTest3SQL": "deployRefs=true mocha --reporter-option maxDiffSize=0 test3/test.deploy.hana-sql.js",
-    "deployTest3": "deployRefs=true mocha --reporter-option maxDiffSize=0 test3/testDeployment.js",
-    "deployHanaDeltaSQL": "CDS_COMPILER_DEPLOY_HANA=1 mocha test3/test.deploy.hana-sql.delta.js",
-    "deployDiffs": "deployRefs=true mocha --reporter-option maxDiffSize=0 test3/deployDiffs.js",
+    "deployHanaSql": "CDS_COMPILER_DEPLOY_HANA=1 mocha --reporter-option maxDiffSize=0 test3/test.deploy.hana-sql.js",
+    "deployHdiHdbcds": "CDS_COMPILER_DEPLOY_HANA=1 mocha --reporter-option maxDiffSize=0 test3/test.deploy.hdi.hdbcds.js",
+    "deployGitDiffs": "CDS_COMPILER_DEPLOY_HANA=1 mocha --reporter-option maxDiffSize=0 test3/test.deploy.git-diffs.js",
     "gentest3": "cross-env MAKEREFS=${MAKEREFS:-'true'} mocha --reporter-option maxDiffSize=0 test3/testRefFiles.js",
     "coverage": "cross-env nyc mocha --reporter-option maxDiffSize=0 test/ test3/ && nyc report --reporter=lcov",
     "lint": "eslint bin/ benchmark/ lib/ test/ test3/ scripts/ types/ && node scripts/linter/lintGrammar.js && node scripts/linter/lintTests.js test3/ && node scripts/linter/lintMessages.js && node scripts/linter/lintMessageIdCoverage.js lib/ && markdownlint README.md CHANGELOG.md doc/ internalDoc/ && cd share/messages && markdownlint .",
     "tslint": "tsc --pretty -p .",
     "updateVocs": "node scripts/odataAnnotations/generateDictMain.js && npm run generateAllRefs",
+    "updateTocs": "node scripts/update-toc.js",
     "generateCompilerRefs": "cross-env MAKEREFS='true' mocha test/testCompiler.js",
     "generateEdmRefs": "cross-env MAKEREFS='true' mocha test/testEdmPositive.js",
     "generateForHanaRefs": "cross-env MAKEREFS='true' mocha test/testHanaTransformation.js",