@sap/cds-compiler 2.10.4 → 2.12.0

package/lib/language/errorStrategy.js

@@ -114,9 +114,10 @@ function sync( recognizer ) {
     }
     return;
   }
-  // TODO: expected token is identifier, current is KEYWORD
 
   if (nextTokens.contains(antlr4.Token.EPSILON)) {
+    // when exiting a (innermost) rule, remember the state to make
+    // getExpectedTokensForMessage() calculate the full "expected set"
     if (recognizer.$nextTokensToken !== token) {
       // console.log('SET:',token.type,recognizer.state,recognizer.$nextTokensToken && recognizer.$nextTokensToken.type)
       recognizer.$nextTokensToken = token;
@@ -126,13 +127,29 @@ function sync( recognizer ) {
     return;
   }
 
+  // Expected token is identifier, current is (reserved) KEYWORD:
+  // TODO: do not use this if "close enough" (1 char diff) to a keyword in nextTokens
+  //
+  // NOTE: it is important to do this only if EPSILON is not in `nextTokens`,
+  // which means that we cannot bring the better special syntax-fragile-ident
+  // in all cases.  Reason: high performance impact of the alternative,
+  // i.e. calling method Parser#isExpectedToken() = invoking the ATN
+  // interpreter to see behind EPSILON.
+  const identType = recognizer.constructor.Identifier;
+  if (keywordRegexp.test( token.text ) && nextTokens.contains( identType )) {
+    recognizer.message( 'syntax-fragile-ident', token, { id: token.text, delimited: token.text },
+                        '$(ID) is a reserved name here - write $(DELIMITED) instead if you want to use it' );
+    token.type = identType;     // make next ANTLR decision assume identifier
+    return;
+  }
+
   if (recognizer._ctx._sync === 'nop')
     return;
   switch (s.stateType) {
-    case ATNState.BLOCK_START:
-    case ATNState.STAR_BLOCK_START:
-    case ATNState.PLUS_BLOCK_START:
-    case ATNState.STAR_LOOP_ENTRY:
+    case ATNState.BLOCK_START:  // 3
+    case ATNState.STAR_BLOCK_START: // 5
+    case ATNState.PLUS_BLOCK_START: // 4
+    case ATNState.STAR_LOOP_ENTRY:  // 10
       // report error and recover if possible
       if ( token.text !== '}' &&                          // do not just delete a '}'
           this.singleTokenDeletion(recognizer) !== null) { // also calls reportUnwantedToken
@@ -145,8 +162,8 @@ function sync( recognizer ) {
       }
       throw new InputMismatchException(recognizer);
 
-    case ATNState.PLUS_LOOP_BACK:
-    case ATNState.STAR_LOOP_BACK: {
+    case ATNState.PLUS_LOOP_BACK: // 11
+    case ATNState.STAR_LOOP_BACK: { // 9
       // TODO: do not delete a '}'
       this.reportUnwantedToken(recognizer);
       const expecting = new IntervalSet.IntervalSet();
@@ -425,7 +442,8 @@ function getExpectedTokensForMessage( recognizer, offendingToken, deadEnds ) {
   }
   else if (offendingToken && recognizer.$nextTokensContext &&
            offendingToken === recognizer.$nextTokensToken) {
-    // We have a state (via sync())  with more "expecting" for the same token
+    // Before exiting a rule, we had a state (via sync()) with a bigger
+    // "expecting set" for the same token
     ll1._LOOK( atn.states[recognizer.$nextTokensState], null,
                predictionContext( atn, recognizer.$nextTokensContext ),
                expected, lookBusy, calledRules, true, true );

package/lib/language/genericAntlrParser.js

@@ -11,14 +11,17 @@ const { ATNState } = require('antlr4/atn/ATNState');
 const { dictAdd, dictAddArray } = require('../base/dictionaries');
 const locUtils = require('../base/location');
 const { parseDocComment } = require('./docCommentParser');
+const { parseMultiLineStringLiteral } = require('./multiLineStringParser');
 const { functionsWithoutParens, specialFunctions } = require('../compiler/builtins');
 
 
 // Push message `msg` with location `loc` to array of errors:
 function _message( parser, severity, id, loc, ...args ) {
   const msg = parser.$messageFunctions[severity]; // set in antlrParser.js
-  return msg( id,
-              (loc instanceof antlr4.CommonToken) ? parser.tokenLocation(loc) : loc, ...args );
+  if (loc instanceof antlr4.CommonToken) {
+    loc = parser.tokenLocation(loc);
+  }
+  return msg( id, loc, ...args );
 }
 
 // Class which is to be used as grammar option with
@@ -49,6 +52,8 @@ GenericAntlrParser.prototype = Object.assign(
     attachLocation,
     startLocation,
     tokenLocation,
+    valueWithTokenLocation,
+    previousTokenAtLocation,
     combinedLocation,
     surroundByParens,
     unaryOpForParens,
@@ -82,6 +87,7 @@ GenericAntlrParser.prototype = Object.assign(
     isStraightBefore,
     meltKeywordToIdentifier,
     prepareGenericKeywords,
+    parseMultiLineStringLiteral,
     constructor: GenericAntlrParser, // keep this last
   }
 );
@@ -245,9 +251,12 @@ function prepareGenericKeywords( pathItem ) {
   // TODO: If not just at the beginning, we need a stack for $genericKeywords,
   // as we can have nested special functions
   this.$genericKeywords.argFull = Object.keys( spec );
+  // @ts-ignore
   const token = this.getCurrentToken() || { text: '' };
-  if (spec[token.text.toUpperCase()] === 'argFull')
+  if (spec[token.text.toUpperCase()] === 'argFull') {
+    // @ts-ignore
     token.type = this.constructor.GenericArgFull;
+  }
 }
 
 // Attach location matched by current rule to node `art`.  If a location is
@@ -285,25 +294,75 @@ function startLocation( token = this._ctx.start ) {
  *
  * @param {object} token
  * @param {object} endToken
- * @param {any} val
+ * @return {CSN.Location}
  */
-function tokenLocation( token, endToken, val ) {
+function tokenLocation( token, endToken = null ) {
   if (!token)
     return undefined;
   if (!endToken)                // including null
     endToken = token;
+
   /** @type {CSN.Location} */
-  const r = {
+  const loc = {
     file: this.filename,
     line: token.line,
     col: token.column + 1,
-    // we only have single-line tokens
+    // Default for single line tokens
     endLine: endToken.line,
     endCol: endToken.stop - endToken.start + endToken.column + 2, // after the last char (special for EOF?)
-  };
-  if (val !== undefined)
-    return { location: r, val };
-  return r;
+  }
+
+  // This check is done for performance reason. No need to access a token's
+  // data if we know that it spans only one single line.
+  const isMultiLineToken = (
+    endToken.type === this.constructor.DocComment ||
+    endToken.type === this.constructor.String ||
+    endToken.type === this.constructor.UnterminatedLiteral
+  );
+  if (isMultiLineToken) {
+    // Count the number of newlines in the token.
+    const source = endToken.source[1].data;
+    let newLineCount = 0;
+    let lastNewlineIndex = endToken.start;
+    for (let i = endToken.start; i < endToken.stop; i++) {
+      // Note: We do NOT check for CR, LS, and PS (/[\r\u2028\u2029]/)
+      //       because ANTLR only uses LF for line break detection.
+      if (source[i] === 10) { // code point of '\n'
+        newLineCount++;
+        lastNewlineIndex = i;
+      }
+    }
+    if (newLineCount > 0) {
+      loc.endLine = endToken.line + newLineCount;
+      loc.endCol = endToken.stop - lastNewlineIndex + 1;
+    }
+  }
+
+  return loc;
+}
+
+/**
+ * Return `val` with the location of `token`.  If `endToken` is provided, use its end
+ * location as end location in the result.
+ *
+ * @param {object} startToken
+ * @param {object} endToken
+ * @param {any} val
+ */
+function valueWithTokenLocation( val, startToken, endToken = null ) {
+  if (!startToken)
+    return undefined;
+  const loc = this.tokenLocation( startToken, endToken );
+  return { location: loc, val };
+}
+
+function previousTokenAtLocation( location ) {
+  let k = -1;
+  let token = this._input.LT(k);
+  while (token.line > location.line ||
+         token.line === location.line && token.column >= location.col)
+    token = this._input.LT(--k);
+  return (token.line === location.line && token.column + 1 === location.col) && token;
 }
 
 // Create a location with location properties `filename` and `start` from
@@ -342,16 +401,20 @@ function unaryOpForParens( query, val ) {
 //  - would influence the prediction, probably even induce adaptivePredict() calls,
 //  - is only slightly "more declarative" in the grammar.
 function docComment( node ) {
-  if (!this.options.docComment)
-    return;
   const token = this._input.getHiddenTokenToLeft( this.constructor.DocComment );
   if (!token)
     return;
+
+  // This token is actually used by / assigned to an artifact.
+  token.isUsed = true;
+
+  if (!this.options.docComment)
+    return;
   if (node.doc) {
     this.warning( 'syntax-duplicate-doc-comment', token, {},
                   'Repeated doc comment - previous doc is replaced' );
   }
-  node.doc = this.tokenLocation( token, token, parseDocComment( token.text ) );
+  node.doc = this.valueWithTokenLocation( parseDocComment( token.text ), token );
 }
 
 // Classify token (identifier category) for implicit names,
@@ -435,20 +498,24 @@ function valuePathAst( ref ) {
     path.length = 1;
   }
   const { args, id, location } = path[0];
-  if (args) {
-    if (path[0].$syntax !== ':')
-      return { op: { location, val: 'call' }, func: ref, location: ref.location, args };
-  }
-  else if (!path[0].$delimited && functionsWithoutParens.includes( id.toUpperCase() )) {
-    return { op: { location, val: 'call' }, func: ref, location: ref.location };
-  }
-  return ref;
+  if (args
+      ? path[0].$syntax === ':'
+      : path[0].$delimited || !functionsWithoutParens.includes( id.toUpperCase() ))
+    return ref;
+
+  const implicit = this.previousTokenAtLocation( location );
+  if (implicit && implicit.isIdentifier)
+    implicit.isIdentifier = 'func';
+  const op = { location, val: 'call' };
+  return (args)
+    ? { op, func: ref, location: ref.location, args }
+    : { op, func: ref, location: ref.location };
 }
 
 // If a '-' is directly before an unsigned number, consider it part of the number;
 // otherwise (including for '+'), represent it as extra unary prefix operator.
 function signedExpression( signToken, expr ) {
-  const sign = this.tokenLocation( signToken, undefined, signToken.text );
+  const sign = this.valueWithTokenLocation( signToken.text, signToken );
   const nval =
         (signToken.text === '-' &&
         expr && // expr may be null if `-` rule can't be parsed
@@ -498,8 +565,16 @@ function numberLiteral( token, sign, text = token.text ) {
 function quotedLiteral( token, literal ) {
   /** @type {CSN.Location} */
   const location = this.tokenLocation( token );
-  const pos = token.text.search( '\'' ) + 1; // pos of char after quote
-  const val = token.text.slice( pos, -1 ).replace( /''/g, '\'' );
+  let pos;
+  let val;
+
+  if (token.text.startsWith('`')) {
+    val = this.parseMultiLineStringLiteral(token);
+    literal = 'string';
+  } else {
+    pos = token.text.search( '\'' ) + 1; // pos of char after quote
+    val = token.text.slice( pos, -1 ).replace( /''/g, '\'' );
+  }
 
   if (!literal)
     literal = token.text.slice( 0, pos - 1 ).toLowerCase();
@@ -529,6 +604,7 @@ function quotedLiteral( token, literal ) {
   };
 
   function atChar(i) {
+    // Is only used with single-line strings.
     return location.col + pos + i;
   }
 }
@@ -601,7 +677,7 @@ function addDef( parent, env, kind, name, annos, props, location ) {
     // which could be tested in name search (then no undefined-ref error)
     return art;
   }
-  else if (env === 'artifacts') {
+  else if (env === 'artifacts' || env === 'vocabularies') {
     dictAddArray( parent[env], art.name.id, art );
   }
   else if (kind || this.options.parseOnly) {
@@ -671,7 +747,7 @@ function assignProps( target, annos = [], props = null, location = null) {
   for (const key in props) {
     let val = props[key];
     if (val instanceof antlr4.CommonToken)
-      val = this.tokenLocation( val, undefined, true);
+      val = this.valueWithTokenLocation( true, val);
     // only copy properties which are not undefined, null, {} or []
     if (val != null &&
         (typeof val !== 'object' ||
@@ -685,15 +761,15 @@ function assignProps( target, annos = [], props = null, location = null) {
 
 // Create AST node for prefix operator `op` and arguments `args`
 function createPrefixOp( token, args ) {
-  const op = this.tokenLocation( token, undefined, token.text.toLowerCase() );
+  const op = this.valueWithTokenLocation( token.text.toLowerCase(), token );
   return { op, args, location: this.combinedLocation( op, args[args.length - 1] ) };
 }
 
 // Create AST node for binary operator `op` and arguments `args`
 function leftAssocBinaryOp( left, opToken, eToken, right, extraProp = 'quantifier' ) {
-  const op = this.tokenLocation( opToken, undefined, opToken.text.toLowerCase() );
+  const op = this.valueWithTokenLocation( opToken.text.toLowerCase() , opToken);
   const extra = eToken
-        ? this.tokenLocation( eToken, undefined, eToken.text.toLowerCase() )
+        ? this.valueWithTokenLocation( eToken.text.toLowerCase(), eToken )
         : undefined;
   if (!left.$parens &&
       (left.op && left.op.val) === (op && op.val) &&