@sap/cds-compiler 6.3.4 → 6.4.2

package/lib/gen/BaseParser.js

@@ -1,4 +1,4 @@
-// Base class for generated parser, for redepage v0.3.0
+// Base class for generated parser, for redepage v0.3.1
 
 'use strict';
 
@@ -18,9 +18,10 @@ class BaseParser {
 
   tokens = undefined;
   tokenIdx = 0;
-  recoverTokenIdx = -1;
-  conditionTokenIdx = -1; // TODO: can we use recoverTokenIdx ?
-  errorTokenIdx = -1;
+  conditionTokenIdx = -1;
+  errorTokenIdx = -1;           // token index where error is detected
+  recoverTokenIdx = -1;         // token index where error recovery resumes
+  reuseErrorTokenIdx = null;    // tmp special for error recovery “token-reuse”
   fixKeywordTokenIdx = -1;
   conditionStackLength = -1;
   nextTokenAsId = false;
@@ -46,6 +47,17 @@ class BaseParser {
     return this;
   }
 
+  _runTransparently( callback ) {
+    const { tokenIdx } = this;
+    const saved = this._saveForWalk();
+    const { length } = this.stack;
+    const r = callback();
+    this.stack.length = length;
+    Object.assign( this, saved );
+    this.tokenIdx = tokenIdx;
+    return r;
+  }
+
   _saveForWalk() {
     return {
       s: this.s,
@@ -108,8 +120,9 @@ class BaseParser {
 
   e() {                         // error: report and recover
     const la = this.la();
-    this._trace( 'detect parsing error' );
-    if (this.errorTokenIdx === this.tokenIdx)
+    this._trace( 'detect parsing error' ); // TODO: write reason (la, no tokens matched, …)
+    if (this.errorTokenIdx === this.tokenIdx &&
+        (this.reuseErrorTokenIdx == null && this.reuseErrorTokenIdx < this.tokenIdx))
       throw Error( `Already reported error for ${ tokenFullName( la ) } at ${ la.location }`);
 
     la.parsedAs = '';           // current token is erroneous
@@ -120,23 +133,13 @@ class BaseParser {
     while (--length && this.tokenIdx === this.stack[length].tokenIdx)
       this.stack[length].followState = null;
     if (++length === this.stack.length) // last good state in current rule
-      return this._reportAndRecover();
+      return this._reportAndRecover();  // otherwise report+recovery after unwind in exit_()
 
     this.stack[length].followState = this.errorState;
     this.s = null;
     return false;
   }
 
-  _reportAndRecover() {
-    this.s = this.errorState;
-    this.reportUnexpectedToken_();
-    const syncSet = this._calculateTokenSet( 'Y' );
-    const recoverDepth = this._findSyncToken( syncSet );
-    this._trace( 'recover from error' );
-    this._recoverFromError( recoverDepth );
-    return false;
-  }
-
   // instead of e() in default if lk() had been used and 'Id' is in a non-default case
   ei() {                // error (after trying to test again as identifier)
     if (!this.la().keyword)     // lk() had directly returned the type
@@ -150,6 +153,7 @@ class BaseParser {
   // go to end of the rule, in tracing parser: g(0)
   gr( follow ) {                // intersection follow set for fast exit
     if (this.stack[this.stack.length - 1].tokenIdx === this.tokenIdx)
+      //this.tokenIdx >= this.errorTokenIdx) not necessary
       return this.e();  // match at least one token
     this.s = 0;
     // TODO: also have recursive flag in stack: was rule was called recursively?
@@ -249,9 +253,19 @@ class BaseParser {
     const la = this.tokens[this.tokenIdx];
     // TODO: consider this like a failed condition? Will be relevant if we try
     // different error recovery possibilities.
-    if (this.keywords[la.keyword])
+    if (this.keywords[la.keyword]) {
+      this._tracePush( [ 'R' ] );
+      if (this._runTransparently( () => {
+        ++this.tokenIdx;
+        this.s = state;
+        const { type, keyword } = this.la();
+        return !(this._pred_next( type, keyword, 'R' ) ??
+                 this._matchesInFollow( type, keyword, 'R' ));
+      } ))
+        return this.e();
       this.reportReservedWord_();
-    // with error recovery: use that (consider this having a good score)
+      // with error recovery: use that (consider this having a good score)
+    }
     return this.c( state, ident )
   }
 
@@ -280,6 +294,10 @@ class BaseParser {
     ++this.tokenIdx;
   }
 
+  reuseToken_() {
+    --this.tokenIdx;
+  }
+
   // condition and precedence handling ------------------------------------------
 
   // state must match the goto-state of the default (there must be no default
@@ -340,13 +358,14 @@ class BaseParser {
       --this.trace.at(-1).length; // do not show guard name in trace
       if (!keyword || this.keywords[keyword] == null)
         return false;             // ok
-      const r = this._matchesInFollow( 'Id', keyword, 'R' );
+      const r = this._matchesInFollow( 'Id', keyword, 'F' );
       --this.trace.at(-1).length; // this.gc() also traces result
       return r;
     }
     if (!keyword || this.keywords[keyword] == null)
       return false;             // ok
-    return this._matchesInFollow( 'Id', keyword, 'R' );
+    return this._matchesInFollow( 'Id', keyword, 'F' );
+    // TODO: still extra tests for 'F' in _pred_next()?
   }
 
   // rule start, end and call: --------------------------------------------------
@@ -370,32 +389,28 @@ class BaseParser {
       throw Error( `this.s === ${ this.s } // illegally set by action, or runtime/generator bug` );
     this.dynamic_ = Object.getPrototypeOf( this.dynamic_ );
     const caller = this.stack.pop();
-    const immediately = this.tokenIdx === caller.tokenIdx;
+    const immediately = this.tokenIdx === caller.tokenIdx && this.tokenIdx >= this.errorTokenIdx;
     if (this.constructor.tracingParser) {
       const post = this.s == null &&
             (immediately
-             ? ' immediately'
+             ? ' immediately'   // no token matched (further unwind or to last good state)
              : caller.followState == null
-             ? ' unsuccessfully'
-             : ' prematurely');
+             ? ' unsuccessfully' // further error rewind
+             : ' prematurely');  // continue at sync state after exit
       const text = immediately ? '⚠ exit rule' : '⏎ exit rule';
       this.s = caller.followState; // for trace
       this._trace( [ text, caller.ruleState, post, 'back to' ] )
       if (immediately && this.stack.at(-1)?.followState != null)
         this.trace = [ this.errorState ]; // show last good state in trace
     }
+    this.prec_ = caller.prec;
     this.s = caller.followState;
-    if (immediately)
+    if (immediately)            // this.gr() has already called this.e() if no token is matched
       return this.s != null && this._reportAndRecover();
 
-    this.prec_ = caller.prec;
-    if (this.s)
-      this._skipErrorTokens();  // TODO: re-think - directly with _reportAndRecover() ?
-    else if (this.s == null)
-      return true;  // attached actions are executed even with "unsuccessful exit"
-
-    this.errorState = this.s;
-    return true;
+    if (this.s != null)
+      this.errorState = this.s; // last good state is now after rule call
+    return true;                // attached actions are executed even with "unsuccessful exit"
   }
 
   // predicate used before rule call (and called by `ckP` and `gP`) on keyword
@@ -438,8 +453,11 @@ class BaseParser {
       // TODO: also not with lean condition
       let match1 = this._pred_next( 'Id', lk1, 'P' ); // first step of `K`/`I` prediction
       if (!match1) {
-        if (lk1 || match1 === false) // assert for correct code generation
-          throw Error( `Cannot match first prediction token in rule at state ${ saved.s }` );
+        if (lk1) {              // assert for correct code generation
+          // Remark: this._pred_next() returns false also if rule has not matched any token
+          const { location } = this.la();
+          throw Error( `Cannot match first prediction token at ${ location.line }:${ location.col } in rule at state ${ saved.s }` );
+        }
         if (match1 == null) {      // TODO: just return true, rule exit prediction will do it
           this._traceSubPush( 0 );  // TODO: make _pred_next push this
           match1 = this._matchesInFollow( 'Id', lk1, 'I' );
@@ -513,25 +531,25 @@ class BaseParser {
         case 'c': case 'ck': case 'ckA': // TODO: re-check ckA
           return true;
         case 'ciA':             // TODO: fixKeywordTokenIdx ?
-          return mode !== 'R';
+          return mode !== 'F';
           // in the R prediction for optional `Id<weak>` at rule end, only
           // alternative keyword matches are preferred, not identifier matches
           // TODO: delete this prediction
         case 'ci':
           if (!keyword ||
               !this.keywords[keyword] && this.fixKeywordTokenIdx !== this.tokenIdx)
-            return mode !== 'R';
+            return mode !== 'F';
           cmd = this.table[state]['']; // is currently always 'g' or 'e'
           this.s = cmd[1];
           break;
         case 'm':
           return type === cmd[2];
         case 'mi':
-          return type === 'Id' && mode !== 'R' &&
+          return type === 'Id' && mode !== 'F' &&
             (!keyword ||
              !this.keywords[keyword] && this.fixKeywordTokenIdx !== this.tokenIdx);
         case 'miA':
-          return type === 'Id' && mode !== 'R';
+          return type === 'Id' && mode !== 'F';
         case 'mk':
           return keyword === cmd[2];
         case 'g': case 'e':
@@ -573,7 +591,7 @@ class BaseParser {
     // prediction, the tool can normally omit the prediction (and output a
     // message), no so with `ruleStartingWithUnreserved`.  We will rather look
     // behind the current rule _after_ having decided that the token is to be
-      // matched as identifier.
+    // matched as identifier.
     return (hasMatchedToken ?? this.tokenIdx > this.stack.at( -1 ).tokenIdx)
       && null; // let caller decide how to interpret this
   }
@@ -609,7 +627,7 @@ class BaseParser {
       this.s = this.stack[depth].followState;
       // TODO: this.prec_ ?
       match = this._pred_next( type, keyword, mode );
-      this._traceSubPush( match == null ? 0 : match === (mode !== 'R') );
+      this._traceSubPush( match == null ? 0 : match === (mode !== 'F') );
       // successfully matching a keyword in giR() means unsuccessful match as
       // reserved identifer
       // TODO: this.stack ?
@@ -667,13 +685,13 @@ class BaseParser {
     else {                      // or always when calculating the sync-set
       let val = this.stack.length + 1;
       while (this.stack.length) {
-        if (!this._tokenSetInRule( set, val ))
-          val = this.stack.length;
+        this._tokenSetInRule( set, val ); // TODO: use if Y-M unification
+        val = this.stack.length;
         // TODO: use new _tracePush if `val` changes, probably also use Y‹val›(…)
         this.dynamic_ = Object.getPrototypeOf( this.dynamic_ );
         this.s = this.stack.pop().followState;
       }
-      set.EOF ??= 0;
+      set.EOF ??= 0;            // TODO: really necessary, see also _findSyncToken()
     }
     this.stack = savedStack;
     this.s = savedState;        // should be the errorState anyway - TODO: confirm
@@ -706,7 +724,7 @@ class BaseParser {
             collectKeywordsAndIdOnly = true;
             cmd = dict.Id;
           }
-          else {                // Id branch never leads to rule exit:
+          else {                // Id branch never leads to rule exit (really?):
             this._tracePush( [ '[' ] );
             this._tokenSetInRule( expecting, val, dict.Id, true );
             this._tracePush( [ ']' ] );
@@ -722,7 +740,7 @@ class BaseParser {
           this.addTokenToSet_( expecting, 'Id', val, false );
           // TODO: should we do s/th special, such that a reserved word is a sync
           // token for Id<greedy>?  Probably not, see also comment in
-           // _findSyncToken()
+          // _findSyncToken()
           break loop;
         case 'g': case 'gi': case 'e':
           break;
@@ -781,62 +799,77 @@ class BaseParser {
     return expecting;
   }
 
-  _findSyncToken( syncSet ) {
+  _findSyncToken( syncSet ) {   // only called from _reportAndRecover()
     const rewindDepth = this.stack.length
-    this.recoverTokenIdx = this.tokenIdx;
+    this.recoverTokenIdx = this.tokenIdx; // TODO: make it part of the return value?
     while (this.recoverTokenIdx < this.tokens.length) {
       const { keyword, type } = this.tokens[this.recoverTokenIdx];
       let recoverDepth = keyword ? syncSet[keyword] : null;
       if (recoverDepth != null)
         return recoverDepth;
       recoverDepth = syncSet[type];
-      // sync to Id only if in expected set of last good state or if after ';'
+      // sync to Id only if in intra-rule expected set of last good state or if after ';'/`}`
       if (recoverDepth != null &&
           (type !== 'Id' || (!keyword || !this.keywords[keyword]) &&
-           // reserved words do not match Id in expected-set
-           (recoverDepth > rewindDepth || this.tokens[this.recoverTokenIdx - 1].type === ';')))
+           // reserved words do not match Id in expected-set → as method
+           (recoverDepth > rewindDepth ||
+            [ ';', '}' ].includes( this.tokens[this.recoverTokenIdx - 1].type ))))
         // if (recoverDepth != null &&
         //     (this.recoverTokenIdx > this.tokenIdx || 
         return recoverDepth;
       ++this.recoverTokenIdx;
     }
-    throw Error( 'EOF must be last in `tokens`' );
+    throw Error( 'EOF must be last in `tokens`' ); // TODO: really necessary?
   }
 
-  _recoverFromError( recoverDepth ) {
+  _reportAndRecover() {         // called from this.e() and this.exit_() after rewind
+    this.s = this.errorState;
+    const syncSet = this._calculateTokenSet( 'Y' );
+    // TODO: use (extended) syncSet also for “expecting” in messages
+    this._trace( 'investigate how to recover' );
+    this.recoverTokenIdx = this.tokenIdx;;
+    const prev = this.lb();
+    const reuseRecoverDepth = this.reuseErrorTokenIdx != null && // only if specially switched on
+          prev?.keyword && prev.parsedAs !== 'keyword' &&
+          this.reuseErrorTokenIdx < this.tokenIdx && syncSet[prev.keyword];
+    // TODO: this reuse-token will only properly work if we can check that the token at
+    // this.la() will definitely match after having re-used this.lb()
+    // → if we have done that, we can again remove this.reuseErrorTokenIdx
+    this.reportUnexpectedToken_( reuseRecoverDepth ? 'reuse' : null );
+
+    if (this.reuseErrorTokenIdx != null)
+      this.reuseErrorTokenIdx = (reuseRecoverDepth) ? this.tokenIdx : -1;
+    this.fixKeywordTokenIdx = (reuseRecoverDepth) ? --this.recoverTokenIdx : -1;
+
+    const recoverDepth = reuseRecoverDepth || this._findSyncToken( syncSet );
     this.s = null;
     let depth = this.stack.length;
     if (recoverDepth > depth) { // no rewind, no rule exit
       this.trace = [ this.errorState ]; // show last good state in trace
       this.s = this.errorState;
-      if (this.s)
-        this._skipErrorTokens();
     }
     while (depth > recoverDepth)
       this.stack[--depth].followState = null;
     // TODO: when the error is due to failed rule exit prediction, try to keep
     // existing followState (if that reaches RuleEnd_)
     // Continue parsing: ignore next predicate (TODO: except some specified ones?)
-    this.conditionTokenIdx = this.tokenIdx;
-    this.conditionStackLength = null;
-    this.fixKeywordTokenIdx = -1; // was set when collecting expecting-set
 
     // TODO: re-check for rule calls which are at the optional rule end:
     //   x: 'x not'; b: 'b'? x {console.log('x→b')} 'b'?; a: b {console.log('b→a')} 'a'
     // with start rule `a` and input `x a`: output should be x→b + b→a
     // with start rule `a` and input `b a`: output should be b→a
-    //
-    // → the rule is: if a rule can continue at the specified state and has
-    // matched at least one token, then its action is executed, otherwise not
-  }
 
-  _skipErrorTokens() {
-    if (this.constructor.tracingParser && this.tokenIdx <= this.recoverTokenIdx) {
-      this._trace( `skip ${ this.recoverTokenIdx - this.tokenIdx } tokens to recover from error`,
+    if (this.constructor.tracingParser) {
+      this._trace( `skipped ${ this.recoverTokenIdx - this.tokenIdx } tokens to recover from error`,
                    this.tokens[this.recoverTokenIdx] );
-      }
-    while (this.tokenIdx < this.recoverTokenIdx)
+    }
+    if (this.tokenIdx > this.recoverTokenIdx)
+      this.reuseToken_();
+    else while (this.tokenIdx < this.recoverTokenIdx)
       this.skipToken_();
+    this.conditionTokenIdx = this.tokenIdx;
+    this.conditionStackLength = null;
+    return false;
   }
 
   // small methods --------------------------------------------------------------
@@ -851,11 +884,12 @@ class BaseParser {
   }
 
   reportUnexpectedToken_( msg ) {
-    const token = this.la();
+    const token = (msg === 'reuse') ? this.lb() : this.la();
     msg ??= `Unexpected token ${ tokenFullName( token, ': ' ) }`;
-    this.reportError_(
-      token.location, msg + ' - expecting: ' +
-        this.expectingArray_().map( tokenName ).sort().join( ', ' ) );
+    msg = (msg === 'reuse')
+      ? `Missing input before keyword ${ tokenFullName( token, ': ' ) }`
+      : msg + ' - expecting: ' + this.expectingArray_().map( tokenName ).sort().join( ', ' );
+    this.reportError_( token.location, msg );
   }
 
   reportReservedWord_() {

package/lib/gen/CdlGrammar.checksum

@@ -1 +1 @@
-bffacbf5acb61179807cb657191c5114
+2f488eae5a0a809b97737edac6e2a42d