@sap/cds-compiler 4.9.6 → 5.1.0

package/lib/compiler/lsp-api.js

@@ -1,5 +1,504 @@
 'use strict';
 
-// Placeholder for future LSP API.
+// API for `@sap/cds-lsp`.
+//
+// THIS FILE IS CONSIDERED INTERNAL!
+// We do not guarantee stability for any project besides the CAP LSP server.
+//
+// This files includes an iterator over "semantic tokens" in an XSN model.
+// "Semantic tokens" are identifiers, but also the "return" parameter.
+// See `internalDoc/lsp/IdentifierCrawling.md` for details.
 
-module.exports = {};
+const { CompilerAssertion } = require('../base/error');
+const $inferred = Symbol.for( 'cds.$inferred' );
+
+// TODO: Remove hints; they should not be necessary in the best case
+const HINTS = {
+  USING_ALIAS: 'using-alias',
+  DEFINITION_NAME: 'definition',
+  NAMESPACE_STATEMENT: 'namespace-statement',
+};
+
+// eslint-disable-next-line no-unused-vars
+class LspSemanticTokenEvent {
+  event; // 'reference' | 'definition',
+  semanticToken;
+  node;
+  hint; // TODO: Remove
+}
+
+/**
+ * All actions to report semantic tokens in a model.
+ */
+const artifactActions = {
+  __proto__: null,
+
+  // e.g. sources or services
+  artifacts: dictOf( artifactTokens ),
+  extensions: arrayOf( extensionTokens ),
+  namespace: namespaceTokens,
+  // e.g. via CSN input
+  vocabularies: dictOf( artifactTokens ),
+  definitions: dictOf( artifactTokens ),
+
+  extern: artifactTokens,
+  name: definitionNameTokens,
+  path: pathReferenceTokens,
+
+  type: artifactTokens,
+  target: artifactTokens,
+  targetAspect: artifactTokens,
+  targetElement: artifactTokens,
+  returns: returnsTokens,
+  items: artifactTokens,
+  elements: elementsTokens,
+
+  enum: dictOf( artifactTokens ),
+  foreignKeys: dictOf( artifactTokens ),
+  actions: dictOf( artifactTokens ),
+  params: dictOf( artifactTokens ),
+  mixin: dictOf( artifactTokens ),
+  excludingDict: dictOf( nameAsReference ),
+
+  // Don't crawl `$tableAliases`, as they are set multiple times in queries
+  // via different `$tableAliases`.
+  //   $tableAliases: null,
+
+  // NOT $queries, as that doesn't cover UNIONs (e.g. `orderBy` vs `$orderBy`)
+  query: artifactTokens,
+
+  from: artifactTokens,
+  includes: arrayOf( artifactTokens ),
+  columns: arrayOf( artifactTokens ),
+  expand: arrayOf( artifactTokens ),
+  inline: arrayOf( artifactTokens ),
+
+  args: argsTokens,
+  on: artifactTokens,
+  default: artifactTokens,
+  value: artifactTokens,
+  sym: enumSymToken,
+  where: artifactTokens,
+  groupBy: artifactTokens,
+  orderBy: artifactTokens,
+  having: artifactTokens,
+  suffix: artifactTokens,
+  limit: artifactTokens,
+  rows: artifactTokens,
+  offset: artifactTokens,
+
+  '@': annotationTokens,
+};
+
+/** Returns a generator that applies the given function on all entries and yields the result. */
+function dictOf( func ) {
+  return function* dictionary( dict ) {
+    for (const [ item ] of iterateGeneric({ dict }, 'dict'))
+      yield* func( item );
+  };
+}
+
+/** Returns a generator that applies the given function on all entries and yields the result. */
+function arrayOf( func ) {
+  return function* array( arr ) {
+    if (!Array.isArray(arr))
+      return;
+    for (const item of arr)
+      yield* func( item );
+  };
+}
+
+/** Generator equivalent of iterateGeneric of forEachGeneric() */
+function* iterateGeneric( obj, prop ) {
+  const dict = obj[prop];
+  if (!dict)
+    return;
+
+  for (const name in dict) {
+    obj = dict[name];
+    if (Array.isArray( obj )) {
+      for (const item of obj)
+        yield [ item, name, prop ]; // parser or source duplicates (e.g. USING vs definition)
+    }
+    else {
+      yield [ obj, name, prop ];
+      if (Array.isArray( obj.$duplicates )) { // redefinitions
+        for (const dup of obj.$duplicates)
+          yield [ dup, name, prop ];
+      }
+    }
+  }
+}
+
+/**
+ * A generator that yields all semantic tokens in an XSN model.
+ * Semantic tokens include identifiers (references/definitions) and the "returns" parameter.
+ *
+ * @param {XSN.Model} xsn
+ * @param {CSN.Options} options
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* traverseSemanticTokens( xsn, options ) {
+  if (!xsn)
+    throw new CompilerAssertion('Expected valid XSN model for traverseSemanticTokens(…)');
+  if (!options)
+    throw new CompilerAssertion('Expected valid options for traverseSemanticTokens(…)');
+
+  if (xsn.sources)
+    yield* dictOf( artifactTokens )( xsn.sources );
+}
+
+/**
+ * Report semantic tokens in artifacts, including definitions, elements, params, etc.
+ *
+ * @param {XSN.Artifact} art
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* artifactTokens( art ) {
+  if (!art || art.builtin || art.$inferred)
+    return null;
+
+  if (Array.isArray( art )) {
+    for (const entry of art)
+      yield* artifactTokens( entry );
+    return null;
+  }
+
+  for (const prop in art) {
+    if (artifactActions[prop])
+      yield* artifactActions[prop](art[prop], art);
+    else if (prop.charAt(0) === '@')
+      yield* artifactActions['@'](art[prop], art);
+  }
+
+  return null;
+}
+
+/**
+ * For an extension, yield all semantic tokens.
+ * We don't use `artifactTokens` for it, because extensions are a special case:
+ *  - they have a name, but actually refer to some other artifact.
+ *  - their artifacts such as elements may overlap with existing definitions, because
+ *    extensions are applied; if they were applied, `_parent` does not point to the
+ *    extension, which means we can't use it to skip them in `artifactTokens`.
+ *  - we only need to handle `annotate` and `extend` kinds specifically:
+ *    if an extension was not applied, pass it to `artifactTokens`;
+ *    if an extension was     applied, we only need to report its name (i.e. reference)
+ *    and traverse over all artifacts
+ *
+ * @param {XSN.Extension} ext
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* extensionTokens( ext ) {
+  if (ext.kind !== 'extend' && ext.kind !== 'annotate')
+    return null;
+
+  const wasApplied = ext.name._artifact && !ext.name._artifact.$inferred;
+  if (!wasApplied) {
+    yield* artifactTokens( ext );
+    return null;
+  }
+
+  yield* nameAsReference( ext );
+
+  // We need to traverse all dictionaries that could themselves contain
+  // extensions.  Enum extensions or columns don't need to be traversed,
+  // for example, because there can't be inner extensions.
+  yield* dictOf( extensionTokens )( ext.params );
+  yield* dictOf( extensionTokens )( ext.actions );
+  yield* dictOf( extensionTokens )( ext.elements );
+
+  if (ext.returns)
+    yield* extensionTokens( ext.returns );
+
+  // Artifact extensions are always definitions, and can't have nested `extend`s,
+  // hence no need to traverse them with `extensionTokens`.
+  yield* dictOf( artifactTokens )( ext.artifacts );
+
+  return null;
+}
+
+/**
+ * Report all semantic tokens in an annotation assignment.
+ *
+ * @param {XSN.Artifact} anno
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* annotationTokens( anno ) {
+  // TODO: Also report annotation names
+  if (anno.kind === '$annotation')
+    yield* annotationValueTokens( anno );
+}
+
+function* argsTokens( args, art ) {
+  if (Array.isArray(args)) {
+    // e.g. unnamed function arguments
+    yield* arrayOf( artifactTokens )( args );
+  }
+  else {
+    // e.g. named arguments
+    for (const [ param ] of iterateGeneric( art, 'args' )) {
+      yield* nameAsReference( param );
+      yield* artifactTokens( param );
+    }
+  }
+}
+
+function* enumSymToken( sym, expr ) {
+  yield {
+    event: 'reference',
+    semanticToken: expr.sym,
+    node: expr,
+    hint: undefined,
+  };
+}
+
+/**
+ * A namespace is always considered a reference and not a definition.
+ *
+ * @param {XSN.Artifact} def
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* namespaceTokens( def ) {
+  if (!def.name)
+    return null;
+
+  for (let i = 0; i < def.name.path.length; ++i) {
+    yield {
+      event: 'reference',
+      semanticToken: def.name.path[i],
+      node: def,
+      hint: (i === def.name.path.length - 1) ? HINTS.NAMESPACE_STATEMENT : null,
+    };
+  }
+
+  return null;
+}
+
+/**
+ * An annotation value may contain expressions which we need to report.
+ *
+ * @param {object} anno
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* annotationValueTokens( anno ) {
+  if (Array.isArray(anno)) {
+    for (const entry of anno)
+      yield* annotationValueTokens( entry );
+  }
+  else if (anno.$tokenTexts) {
+    yield* artifactTokens( anno );
+  }
+  else if (Array.isArray(anno.val)) {
+    yield* annotationValueTokens( anno.val );
+  }
+  else if (anno.struct) {
+    for (const [ struct ] of iterateGeneric( anno, 'struct' ))
+      yield* annotationValueTokens( struct );
+  }
+}
+
+/**
+ * A `returns` structure may contain sub-elements.  But we report the `returns`
+ * token as well, as it is considered a token with semantic value.
+ *
+ * @param {XSN.Artifact} art
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* returnsTokens( art ) {
+  if (art.kind === 'param') {
+    // report the `returns` parameter
+    yield {
+      event: 'definition',
+      semanticToken: art.name,
+      node: art,
+      hint: undefined,
+    };
+    yield* artifactTokens( art );
+  }
+}
+
+/**
+ * Report elements if they should be traversed.  They are not always traversed
+ * to avoid duplication due to `expand` and `columns` also being traversed.
+ *
+ * @param {Record<string, XSN.Artifact>} elements
+ * @param {XSN.Artifact} art
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* elementsTokens( elements, art ) {
+  if (shouldTraverseElements( art ))
+    yield* dictOf( artifactTokens )( elements );
+}
+
+/**
+ * Report all references in `ref`.
+ *
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* pathReferenceTokens( path, ref, user = ref, hint = null ) {
+  if (!path)
+    return null;
+
+  // don't report cds.Association/cds.Composition
+  // TODO: Or report the `Association` keyword, similar to `returns`?
+  if (path.length === 1 && ref._artifact?.category === 'relation')
+    return null;
+
+  yield* artifactTokens( path );
+
+  // parser prepends a fake `type of` segment, which we need to skip
+  const root = ref.scope === 'typeOf' ? 1 : 0;
+  for (let i = root; i < path.length; ++i) {
+    if (!path[i].$inferred) { // e.g. `id` when expanded from `$user`
+      yield {
+        event: 'reference',
+        semanticToken: path[i],
+        node: user,
+        hint,
+      };
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Some XSN nodes such as entries in `excludingDict` or named arguments are references
+ * but don't have a `path` property, only a `name` property.  Report such names
+ * as references.
+ *
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* nameAsReference( ref, hint = null ) {
+  if (!ref.name || ref.name.$inferred)
+    return null;
+
+  if (ref.name.path) {
+    yield* pathReferenceTokens( ref.name.path, ref.name, ref, hint );
+  }
+  else {
+    yield {
+      event: 'reference',
+      semanticToken: ref.name,
+      node: ref,
+      hint,
+    };
+  }
+  return null;
+}
+
+/**
+ * Traverse the name of a definition, and report N-1 path steps as references
+ * and of course the definition itself.
+ *
+ * @returns {Generator<LspSemanticTokenEvent>}
+ */
+function* definitionNameTokens( name, art ) {
+  if (!art.kind)
+    return null; // e.g. parameter references
+  if (art.kind === '$annotation')
+    return null; // annotation name, e.g. in `@anno: (elem)`
+
+  if ((name.$inferred && name.$inferred !== 'as') ||
+    art.kind === 'select' || art.kind === '$join') {
+    // Internal names such as numbers for SELECTs or the `$internal` names must
+    // not be reported.
+    return null;
+  }
+
+  if (art.kind === 'extend' || art.kind === 'annotate') {
+    yield* nameAsReference( art );
+    return null;
+  }
+
+  // Report references in a name (N-1 path steps).
+  for (let i = 0; i < name.path?.length - 1; ++i) {
+    yield {
+      event: 'reference',
+      semanticToken: name.path[i],
+      node: art,
+      hint: HINTS.DEFINITION_NAME,
+    };
+  }
+
+  const hint = art.kind === 'using' ? HINTS.USING_ALIAS : null;
+
+  if (name.path) {
+    // Only take the last path step; all others are considered references.
+    const implicitName = name.path[name.path.length - 1];
+    yield {
+      event: 'definition',
+      semanticToken: implicitName,
+      node: art,
+      hint,
+    };
+  }
+  else if (name.id) {
+    // Not all names have a path; some (e.g. parameters) only have an ID.
+    yield {
+      event: 'definition',
+      semanticToken: name,
+      node: art,
+      hint,
+    };
+  }
+
+  return null;
+}
+
+/**
+ * Returns true if `elements` of the given `art` should be traversed.
+ * Elements are _not_ traversed, e.g. for `expand`, to avoid duplicates.
+ *
+ * @returns {boolean}
+ */
+function shouldTraverseElements( art ) {
+  return (
+    // $expand: 'origin'   -> normal expansion
+    // $expand: 'annotate' -> additional annotation (needs to traverse annotation expressions)
+    art.$expand !== 'origin' && !art.elements[$inferred] && (
+      // sub-elements are always traversed except for `expand`, which is handled on its own.
+      art.kind === 'element' && !art.expand ||
+      // all non-query elements are traversed; because `_main` on bound actions may point
+      // to a query, we handle parameters explicitly.
+      art.kind === 'param' || !(art._main || art).$queries
+    )
+  );
+}
+
+/**
+ * Given a LspSemanticTokenEvent, returns a generator that yields the referenced
+ * object and its origin's until the deepest entry is found.
+ *
+ * @param obj
+ * @returns {Generator<*, void, *>}
+ */
+function* getSemanticTokenOrigin( obj ) {
+  let ref = obj.semanticToken;
+  if (obj.event === 'definition') {
+    ref = obj.node;
+  }
+  else {
+    if (!ref?._artifact)
+      return; // unknown -> abort
+    // take first artifact for duplicates (best effort)
+    ref = Array.isArray(ref._artifact) ? ref._artifact[0] : ref._artifact;
+    yield ref;
+  }
+
+  if (!ref._effectiveType)
+    return; // abort for unresolved references and cyclic ones
+
+  while (ref._origin) {
+    yield ref._origin;
+    ref = ref._origin;
+    if (!ref || typeof ref === 'string')
+      break;
+  }
+}
+
+module.exports = {
+  traverseSemanticTokens,
+  getSemanticTokenOrigin,
+};

package/lib/compiler/populate.js

@@ -83,6 +83,7 @@ function populate( model ) {
     effectiveType,
     getOrigin,
     getInheritedProp,
+    mergeSpecifiedForeignKeys,
   } );
   // let depth = 100;
 
@@ -115,7 +116,10 @@ function populate( model ) {
 
   /** Make sure that effectiveType() is called on all members and items */
   function traverseElementEnvironments( art ) {
-    // TODO: we could leave out foreign keys (but they are traversed via forEachMember)
+    // We leave out foreign keys (as they are traversed via forEachMember).
+    // Keys are handled in tweak-assocs.js
+    if (art.kind === 'key')
+      return;
     let type = effectiveType( art );
     while (type?.items)
       type = effectiveType( type.items );
@@ -529,9 +533,6 @@ function populate( model ) {
       if (!selem) {
         info( 'query-missing-element', [ ielem.name.location, art ], {
           '#': ielem.kind === 'enum' ? 'enum' : 'std', id,
-        }, {
-          std: 'Element $(ID) is missing in specified elements',
-          enum: 'Enum $(ID) is missing in specified enum values',
         } );
       }
       else {
@@ -560,6 +561,8 @@ function populate( model ) {
           setLink( ielem, 'elements$', selem.elements );
         if (selem.enum)
           setLink( ielem, 'enum$', selem.enum );
+        if (selem.foreignKeys)
+          setLink( ielem, 'foreignKeys$', selem.foreignKeys );
       }
     }
 
@@ -574,8 +577,57 @@ function populate( model ) {
       specifiedElement.$isSpecifiedElement = true;
       if (!specifiedElement.$replacement) {
         const loc = [ specifiedElement.name.location, specifiedElement ];
-        error( 'query-unspecified-element', loc, { id },
-               'Element $(ID) does not result from the query' );
+        error( 'query-unspecified-element', loc, { id } );
+      }
+    }
+  }
+
+  /**
+   * Merge _specified_ foreign keys with _inferred_ foreign keys in the given view/element,
+   * where specified elements can appear through CSN.
+   *
+   * We only copy annotations.
+   *
+   * This is important to ensure re-compilability.
+   *
+   * TODO: make this part of a revamped on-demand 'extend' functionality.
+   *
+   * @param art
+   */
+  function mergeSpecifiedForeignKeys( art ) {
+    if (!art.foreignKeys)
+      return; // TODO: Warn if there are no foreign keys?
+
+    let wasAnnotated = false;
+
+    for (const id in art.foreignKeys) {
+      const ielem = art.foreignKeys[id];  // inferred element
+      const selem = art.foreignKeys$[id]; // specified element
+      if (!selem) {
+        info( 'query-missing-element', [ ielem.name.location, art ], { '#': 'foreignKeys', id } );
+      }
+      else {
+        for (const prop in selem) {
+          // just annotation assignments and doc comments for foreign keys
+          if (prop.charAt(0) === '@' || prop === 'doc') {
+            ielem[prop] = selem[prop];
+            // required for gensrc mode of to-csn.js, otherwise the annotation
+            // may be lost during recompilation.
+            ielem[prop].$priority = 'annotate';
+            wasAnnotated = true;
+          }
+        }
+        selem.$replacement = true;
+      }
+    }
+    if (wasAnnotated)
+      setExpandStatusAnnotate( art, 'annotate' );
+
+    for (const id in art.foreignKeys$) {
+      const specifiedElement = art.foreignKeys$[id];
+      if (!specifiedElement.$replacement) {
+        const loc = [ specifiedElement.name.location, specifiedElement ];
+        error( 'query-unspecified-element', loc, { '#': 'foreignKeys', id } );
       }
     }
   }
@@ -816,15 +868,10 @@ function populate( model ) {
     const excludingDict = (colParent || query).excludingDict || Object.create( null );
 
     const envParent = wildcard._pathHead; // TODO: rename _pathHead to _columnParent
-    // console.log('S1:',location.line,location.col,
-    //             envParent&&!!envParent._origin&&envParent._origin.name)
     const env = wildcardColumnEnv( wildcard, query );
     if (!env)
       return;
 
-    // if (envParent) console.log('S2:',location.line,location.col,
-    //             envParent?.name,envParent?._origin?.name,
-    //             Object.keys(env),Object.keys(elements))
     for (const name in env) {
       const navElem = env[name];
       // TODO: remove all access to masked (use 'grep')
@@ -1069,10 +1116,10 @@ function populate( model ) {
     else {
       // referred (and probably inferred) assoc (without a user-provided target at that place)
       // HINT: consider bin/cdsv2m.js when changing the following message text
-      // No grouped and sub messages yet (TODO v5): mention at all target places with all assocs
+      // No grouped and sub messages yet (TODO v6): mention at all target places with all assocs
       const withAnno = annotationVal( exposed[0]['@cds.redirection.target'] );
       for (const proj of exposed) {
-        // TODO: def-ambiguous-target (just v5, as the current is infamous and used in options),
+        // TODO: def-ambiguous-target (just v6, as the current is infamous and used in options),
         message( 'redirected-implicitly-ambiguous',
                  [ weakLocation( proj.name.location ), proj ],
                  {

package/lib/compiler/propagator.js

@@ -13,7 +13,6 @@ const {
   forEachMember,
   forEachGeneric,
   isDeprecatedEnabled,
-  isBetaEnabled,
 } = require( '../base/model');
 const {
   setLink,
@@ -50,13 +49,17 @@ function propagate( model ) {
     targetAspect,
     cardinality: notWithExpand,
     on: notWithExpand,
-    foreignKeys: expensive,    // includes "notWithExpand", dictionary copy
+    // "expensive" includes "notWithExpand"
+    // required for places where we don't handle associations, such as in parameters;
+    // otherwise already expanded and rewritten.
+    foreignKeys: expensive,
     items,
+    // required for propagation in targetAspect; otherwise already expanded
     elements: expensive,
-    enum: expensive,
-    params: expensive,          // actually only with parent action
-    returns,
-    $filtered: annotation, // TODO(v5): Remove
+    // already expanded if necessary
+    //   enum: expensive,
+    //   params: expensive,          // actually only with parent action
+    //   returns,
     $enclosed: annotation,
   };
   const ruleToFunction = {
@@ -75,8 +78,7 @@ function propagate( model ) {
   const { warning, throwWithError } = model.$messageFunctions;
 
   forEachDefinition( model, run );
-  if (isBetaEnabled( model.options, 'v5preview' ))
-    forEachGeneric( model, 'vocabularies', run );
+  forEachGeneric( model, 'vocabularies', run );
 
   // TODO: move 'virtual' handling/checks to resolver if
   //       'deprecated.oldVirtualNotNullPropagation' is gone