@sap/cds-compiler 5.2.0 → 5.3.2

package/lib/gen/BaseParser.js

@@ -1,16 +1,17 @@
-// Base class for generated parser, for redepage v0.1.7
+// Base class for generated parser, for redepage v0.1.12
 
 'use strict';
 
 class BaseParser {
   constructor( lexer, keywords, table ) {
     this.keywords = keywords;
-    this.table = table;
+    this.table = compileTable( table );
     this.lexer = lexer;
     this.tokens = undefined;
     this.eofIndex = undefined;
     this.tokenIdx = 0;
     this.conditionTokenIdx = -1;
+    this.fixKeywordTokenIdx = -1;
     this.conditionStackLength = -1;
     this.nextTokenAsId = false;
 
@@ -63,7 +64,7 @@ class BaseParser {
     if (this.trace.length > 1)
       this._trace( 'detected parsing error,' );
     this.reportUnexpectedToken_( la );
-    la.parsed = 0;
+    la.parsedAs = 0;
 
     if (this.conditionTokenIdx === this.tokenIdx &&
         this.conditionStackLength === this.stack.length &&
@@ -73,7 +74,8 @@ class BaseParser {
       this.s = (this.tokenIdx > tokenIdx) ? this.errorState : ruleState;
       return false;             // error recovery: ignore condition/precedence
     }
-    if (++this.tokenIdx > this.eofIndex)
+
+    if (this.tokenIdx >= this.eofIndex)
       return this._stopParsing( this.stack.length );
     // TODO: also sync to what comes next in current rule, at least after rule call,
     // this way we do not have to do the check of g(0) in re() as we did before 2023-12-07
@@ -87,7 +89,6 @@ class BaseParser {
   ei() {                // error (after trying to test again as identifier)
     if (!this.tokens[this.tokenIdx].keyword) // lk() had directly returned the type
       return this.e();
-    this._traceIdOrPred( '-Id' );
     this.nextTokenAsId = true;
     return false;               // do not execute action after it
   }
@@ -104,25 +105,27 @@ class BaseParser {
     const { type: lt, keyword: lk } = this.tokens[this.tokenIdx];
     if (lk &&            // Id also for unreserved, except after condition failure
         follow?.[0] === 'Id' && this.keywords[lk] !== false &&
-        this.conditionTokenIdx !== this.tokenIdx ||
-        follow?.includes( lk || lt ))
-      return true;
-
-    // Do we have possibilities to stay in rule with error recovery?
-    const expecting = this._expecting( 0 ); // dynamic follow-set
-    // TODO: improve performance: no check needed for a rule-end directly after
-    // a rule end: the second is definitely successful if the first was.
-    // TODO: do not calculate the complete dynamic follow-set, provide dedicated
-    // function to test whether the next token is valid
-    // we might also cache the result in the stack
-    // ok: lk or lt -> lk=e or (lt=e && (not cond || not keyw)
-    if (expecting[lk] ||
-        // if at failed condition, do not make Id in follow end the rule
-        // (assuming that there is no condition for `Id` at optional rule end):
-        expecting[lt] && !(lk && this.conditionTokenIdx === this.tokenIdx))
+        this.fixKeywordTokenIdx !== this.tokenIdx ||
+        follow?.includes( lk || lt )) {
+      this._tracePush( [ 'E', true ] );
       return true;
-
-    return this.e();
+    }
+    this._tracePush( [ 'E', 0 ] );
+    // TODO: caching
+    const { dynamic_ } = this;
+    let match;
+    let depth = this.stack.length;
+    while (match == null && --depth) {
+      this.dynamic_ = Object.getPrototypeOf( this.dynamic_ );
+      const { followState } = this.stack[depth];
+      match = this._pred_next( followState, lt, lk, 'E' );
+      this._traceSubPush( match ?? 0 );
+    }
+    this.dynamic_ = dynamic_;
+    // If the parser reaches this point with match = null, even the top-level rule
+    // does not have a required token (typically `EOF`) at the end → the parser
+    // must accept any token → rule exit possible (but no output '✔' in trace).
+    return (match ?? true) || this.e();
   }
 
   // go to state; non-tracing parser: `this.s=‹state›` or `this.gr()`
@@ -138,7 +141,6 @@ class BaseParser {
   giA( state, follow ) { // go to state (after trying to test again as identifier)
     if (!this.tokens[this.tokenIdx].keyword) // lk() had directly returned the type
       return this.g( state, follow );
-    this._traceIdOrPred( '-Id' );
     this.nextTokenAsId = true;
     return false;               // do not execute action after it
   }
@@ -148,9 +150,8 @@ class BaseParser {
     const lk = this.tokens[this.tokenIdx].keyword;
     // As opposed to ei(), we also check for reserved keywords here; this way, we
     // do not have to add reserved keywords from the follow-set to the `switch`.
-    if (!lk || this.keywords[lk] === false)
+    if (!lk || this.keywords[lk] === false) // TODO: consider fixKeywordTokenIdx ?
       return this.g( state, follow );
-    this._traceIdOrPred( '-Id' );
     this.nextTokenAsId = true;
     return false;               // do not execute action after it
   }
@@ -160,7 +161,6 @@ class BaseParser {
     const lk = this.tokens[this.tokenIdx].keyword;
     if (!lk || this.keywords[lk] === false || this._keyword_after_rule( lk ))
       return this.g( state, follow );
-    this._traceIdOrPred( '-Id' );
     this.nextTokenAsId = true;
     return false;               // do not execute action after it
   }
@@ -199,9 +199,9 @@ class BaseParser {
       : this.e();
   }
 
-  c( state, parsed = 'token' ) { // consume token
+  c( state, parsedAs = 'token' ) { // consume token
     const la = this.tokens[this.tokenIdx];
-    la.parsed = parsed;
+    la.parsedAs = parsedAs;
     if (this.tokenIdx < this.eofIndex) ++this.tokenIdx;
     // TODO: handle identifier-including-reserved-words later (e.g. for id after a `.`)
     this.s = state;
@@ -235,12 +235,16 @@ class BaseParser {
     return this.lP( first2 ) && this.ck( state );
   }
 
-  // for parser token
+  // for parser token or token set via `/`
   ckA( state ) {
-    // if it really should be considered an Id, `set this.la().parsed` yourself
+    // if it really should be considered an Id, `set this.la().parsedAs` yourself
     return this.c( state, (this.l() === 'Id' ? 'keyword' : 'token') );
   }
 
+  skipToken_() {
+    ++this.tokenIdx;
+  }
+
   // condition and precedence handling ------------------------------------------
 
   // state must match the goto-state of the default (there must be no default
@@ -250,14 +254,25 @@ class BaseParser {
   // “go if user condition fails”
   gc( state, cond ) {
     if (this.conditionTokenIdx === this.tokenIdx &&
-        this.conditionStackLength === this.stack.length)
+        this.conditionStackLength === this.stack.length) {
+      this._tracePush( [ 'C' ] );
       return true;              // error recovery: ignore condition
+    }
     this.conditionTokenIdx = this.tokenIdx;
     this.conditionStackLength = this.stack.length;
     // TODO: let this[cond]( true ) return recovery badness in error case
     const fail = !this[cond]( true );
     if (this.constructor.tracingParser)
-      this._tracePush( `${ fail ? '¬' : '✓' } ${ cond }` );
+      this._tracePush( [ 'C', cond, !fail ] );
+    // TODO TOOL: in this case, the default case must not have actions (tool must
+    // add state if it does)
+    if (fail) {                 // TODO: extra gcK() method instead of check below
+      // TODO: extra method necessary for academic case
+      // ( 'unreserved' 'foo' | <cond> Id 'bar' )` with input `unreserved bar`
+      const { keyword } = this.la();
+      if (keyword && this.table[keyword])
+        this.fixKeywordTokenIdx = this.tokenIdx;
+    }
     return !fail || this.g( state ) && false;
   }
 
@@ -269,7 +284,7 @@ class BaseParser {
   gp( state, prec, mode ) {
     if (this.conditionTokenIdx === this.tokenIdx &&
         this.conditionStackLength === this.stack.length) {
-      this._tracePush( `(${ this._prec })!` );
+      this._tracePush( [ 'C' ] );
       return true;              // error recovery: ignore condition
     }
     this.conditionTokenIdx = this.tokenIdx;
@@ -282,11 +297,17 @@ class BaseParser {
     if (this.constructor.tracingParser) {
       const pp = (parentPrec === -Infinity) ? '-∞' : parentPrec;
       const tp = (this.prec_ == null) ? '∞' : this.prec_;
-      const suffix = mode === 'post' && ` ≤ ${ tp }` || mode === 'none' && ` < ${ tp }`;
-      this._tracePush( `${ fail ? '¬' : '✓' }(${ pp } < ${ prec }${ suffix || '' })` );
+      const suffix = mode === 'post' && `≤${ tp }` || mode === 'none' && `<${ tp }`;
+      this._tracePush( [ 'C', `${ pp }<${ prec }${ suffix || '' }`, !fail ] );
     }
-    if (fail)
+    if (fail) {                 // TODO: extra gcK() method instead of check below
+      // TODO: extra method necessary for academic case
+      // ( 'unreserved' 'foo' | <cond> Id 'bar' )` with input `unreserved bar`
+      const { keyword } = this.la();
+      if (keyword && this.table[this.s][keyword])
+        this.fixKeywordTokenIdx = this.tokenIdx;
       return this.g( state ) && false;                // TODO: reset this.prec_ ?
+    }
     this.prec_ = (mode === 'right') ? prec - 1 : prec; // -1: <…,assoc=right>, <…,prefix>
     return true;
   }
@@ -329,75 +350,60 @@ class BaseParser {
 
   // predicate used before rule call if with LL(1) conflict, 'Id' in other case
   lP( first2 ) {             // only start rule if this predicate returns true
-    const { type: lt2, keyword: lk2 } = this.tokens[this.tokenIdx + 1];
-    // Argument first2 is just a performance hint with ckP():
-    if (lk2 && first2?.[0] === 'Id' && this.keywords[lk2] !== false ||
-        first2?.includes( lk2 || lt2 ))
-      return true;
-
     // nothing to check if not a non-reserved keyword:
     const { keyword: lk1 } = this.tokens[this.tokenIdx];
     if (!lk1 || !this.keywords[lk1])
       return true;
 
+    const { type: lt2, keyword: lk2 } = this.tokens[this.tokenIdx + 1];
+    // Argument first2 is just a performance hint with ckP():
+    if (lk2 && first2?.[0] === 'Id' && this.keywords[lk2] !== false ||
+        first2?.includes( lk2 || lt2 )) {
+      this._tracePush( [ 'P', true ] );
+      return true;
+    }
+    this._tracePush( [ 'P' ] );
     // now check it dynamically:
     let cmd = this.table[this.s][lk1];
-    if (typeof cmd === 'string')
-      cmd = this.table[this.s][cmd];
-    if (!Array.isArray( cmd ) || cmd[2] !== 1)
+    if (cmd[2] !== 1)
       throw Error( `Unexpected command '${ cmd?.[0] }' without prediction at state ${ this.s } for ‘${ lk1 }’` );
 
-    this._traceIdOrPred( '-P1' );
+    // if not the keyword match, the command is “goto” or “rule call”
     const nextState = (cmd[0] === 'ck') ? cmd[1] : this._pred_keyword( cmd[1], lk1 );
 
-    if (this._pred( nextState, lt2, lk2 ))
-      return true;
-    if (lt2 === 'IllegalToken') // TODO: keep?
+    ++this.tokenIdx;            // for user lookahead fns and conditions
+    const match = this._pred_next( nextState, lt2, lk2, 'P' );
+    --this.tokenIdx;
+
+    const r = match ?? true;
+    if (match == null)
+      this._traceSubPush( 0 );
+    if (lt2 === 'IllegalToken')
       return true
-    // TODO: instead of this IllegalToken test, set tokenIndex+nextState for extra
-    // expected calculation if parser fails after Id - we would then also add the
-    // expected tokens after keyword-interpretation
-    this._traceIdOrPred( '-Id' );
-    this.nextTokenAsId = true;
-    return false;               // do not execute action after it
+    // TODO: instead of this IllegalToken test, implement a “confirm unreserved
+    // keyword as Id” prediction which tests whether the token after the then-Id
+    // matches.
+    this._traceSubPush( r );
+    if (!r)
+      this.nextTokenAsId = true;
+    return r;
   }
 
   // Now the helper methods =====================================================
 
   // Standard weak-conflict predicate -------------------------------------------
-  // Weak (and fast) single-step walk and test (no rule exit, start is fine): for
-  // pg(), pr(). The main point is that we do not (again) consider predicates.
-  // Currently just tests against the token _type_ of the next token, not its
-  // specific keyword; see comments below for details.
-
-  _pred( nextState, lt2, lk2 ) {
-    if (nextState) {
-      // return this._pred_test( nextState, lt2 );
-      const r = this._pred_next( nextState, lt2, lk2 );
-      this._tracePush( this.s );
-      return r;
-    }
-    // dubious weak conflict at end of rule:
-    this._traceIdOrPred( '-P0' );
-    this._tracePush( this.s );
-    return true;                // dubious
-  }
 
   _pred_keyword( state, keyword ) {
-    // returns next state for first token as keyword, for lP()
+    // returns state after matching the first token as keyword, for lP()
     while (state) {
-      this._tracePush( `${ state }-P1` );
+      this._traceSubPush( state );
       let cmd = this.table[state];
-      if (!Array.isArray( cmd )) {
-        const alt = cmd[keyword] || cmd.Id; // Id to cover optimized rule call
-        cmd = (typeof alt === 'string')
-          ? cmd[alt]
-          : typeof alt === 'number' && [ 'g', alt ] || alt || [ 'g', cmd[''] ];
-      }
+      if (!Array.isArray( cmd ))
+        cmd = cmd[keyword] || cmd.Id || cmd[''];
       switch (cmd[0]) {
         case 'ck': case 'mk':
           return cmd[1];        // state after token consumption
-        case 'g':
+        case 'g':               // TODO: another rule call?
           break;
         default:
           if (typeof cmd[0] !== 'number')
@@ -409,43 +415,70 @@ class BaseParser {
     throw Error( 'Not supported: option for unreserved keywords in follow set' );
   }
 
-  _pred_next( state, type, keyword ) {
+  _pred_next( state, type, keyword, mode ) {
+    let hasEnteredRule = false;
     while (state) {
-      this._tracePush( `${ state }-P2` );
+      this._traceSubPush( state );
       let cmd = this.table[state];
       if (!Array.isArray( cmd )) {
-        const alt = keyword && cmd[keyword] || cmd[type];
-        cmd = (typeof alt === 'string')
-          ? cmd[alt]
-          : typeof alt === 'number' && [ 'g', alt ] || alt || [ 'default', cmd[''] ];
+        const lookahead = cmd[' lookahead'];
+        cmd = lookahead
+          ? cmd[this[lookahead]( mode )] || cmd['']
+          : keyword && cmd[keyword] || cmd[type] || cmd[''];
       }
       switch (cmd[0]) {
-        case 'c': case 'ck': case 'ciA':
+        case 'c': case 'ck': case 'ciA': case 'ckA': // TODO: re-check ckA
           return true;
+        case 'ci':
+          if (!keyword ||
+              this.keywords[keyword] !== false && this.fixKeywordTokenIdx !== this.tokenIdx)
+            return true;
+          cmd = this.table[state]['']; // is currently always 'g' or 'e'
+          break;
         case 'm':
           return type === cmd[2];
-        case 'mi': case 'ci':
-          return type === 'Id' && (!keyword || this.keywords[keyword] !== false);
+        case 'mi':
+          return type === 'Id' &&
+            (!keyword ||
+             this.keywords[keyword] !== false && this.fixKeywordTokenIdx !== this.tokenIdx);
         case 'miA':
           return type === 'Id';
         case 'mk':
           return keyword === cmd[2];
+        case 'g': case 'e':
+          break;
+        default:
+          if (typeof cmd[0] !== 'number')
+            throw Error( `Unexpected command ${ cmd[0] } at state ${ this.s }` );
+          // If the parser enters a rule, reaching the rule end (can happen with
+          // option `minTokensMatched`) means "no match".
+          hasEnteredRule = true;
+          // If we want to support conditions before matching the first token in a
+          // rule, we would have to handle `this.stack` and `this.dynamically_`.
       }
       // We could optimize with rule call - only 'Id' must be further investigated
       state = cmd[1];
     }
-    this._traceIdOrPred( 'f' );
-    this._tracePush( this.s );
-    // TODO: really false, not true?
-    // `false` means that la1 is not considered an unreserved keyword.  This is
-    // correct (consider `e: Association @Anno`), but probably not optimal for
-    // error reporting (consider `e: Association +`).  Improving that is more
-    // costly, as we really need to consider rule exits → stack.
-    return false;
+    // If invalid state, the second token does not match, e.g. for `VIRTUAL +`
+    // or `VIRTUAL §` (with IllegalToken):
+    if (state == null)
+      return false;
+
+    // Otherwise, the parser could end the rule after having matched the keyword
+    // with prediction.  TODO: as we do not look behind the current rule for the
+    // prediction, the tool can normally omit the prediction (and output a
+    // message), no so with `ruleStartingWithUnreserved`.  We will rather look
+    // behind the current rule _after_ having decided that the token is to be
+    // matched as identifier.
+    return !hasEnteredRule && null; // let caller decide how to interpret this
   }
 
   _keyword_after_rule( keyword ) {
     // TODO: this is a slow implementation - do dedicated traversal later
+    // It is used in giR() only and this is currently used just once.
+    // TODO: using mode = 'R' and tracing R(…)
+    // TODO: investigate why this was not written before adding
+    // `<default=fallback>` in rule `fromRefWithOptAlias`.
     return this._expecting()[keyword];
   }
 
@@ -477,16 +510,13 @@ class BaseParser {
   // tokens at the follow state of the current rule.  Argument `prop` is the token
   // name for `cmd` in a decision.
   _exp_collect( expecting, cmd, prop ) {
-    if (prop != null) {
-      cmd = cmd[(typeof cmd[prop] === 'string') ? cmd[prop] : prop];
-    }
-    if (typeof cmd === 'number') // ‹followState› = short form for this.g(‹followState›)
-      cmd = [ 'g', cmd ];
-
+    if (prop != null)
+      cmd = cmd[prop];
     if (!Array.isArray( cmd )) {
       let reachedRuleEnd = false;
       for (const tok in cmd) {
-        if (Object.hasOwn( cmd, tok ) && this._exp_collect( expecting, cmd, tok ))
+        if (Object.hasOwn( cmd, tok ) && tok.charAt(0) !== ' ' &&
+            this._exp_collect( expecting, cmd, tok ))
           reachedRuleEnd = true;
       }
       return reachedRuleEnd;
@@ -496,7 +526,7 @@ class BaseParser {
         expecting[prop] = true;
         return false;
       case 'ckA':
-        for (const tok of this.translateParserToken_( prop ) || [ prop ])
+        for (const tok of this.translateParserToken_( prop ))
           expecting[tok] = true;
         return false;
       case 'm': case 'mk':
@@ -524,24 +554,23 @@ class BaseParser {
     }
   }
 
-  translateParserToken_( _token ) {
-    return null;
+  translateParserToken_( token ) {
+    return [ token ];
   }
 
   // Error recovery -------------------------------------------------------------
 
   _recoverInline( expecting ) {
-    // Inline error recovery - single token deletion (TODO later: also try more !)
-    // token position has been advanced before calling this function
-    if (!expecting[this.lk()] && !expecting[this.l()])
+    const { type: lt2, keyword: lk2 } = this.tokens[this.tokenIdx + 1];
+    if (!(lk2 && expecting[lk2] || expecting[lt2]))
       return false;
 
     // Immediately exit rules (except start) when no tokens have yet been consumed:
     let { length } = this.stack;
     while (--length > 0) {
       const caller = this.stack[length];
-      // matched tokens (other than the one skipped one) in rule: found rule
-      if (this.tokenIdx - 1 > caller.tokenIdx)
+      // matched tokens in rule: found rule
+      if (this.tokenIdx > caller.tokenIdx)
         break;
       caller.followState = null;
     }
@@ -557,13 +586,13 @@ class BaseParser {
       this.s = this.errorState;
     }
 
+    this.skipToken_();
     if (this.constructor.tracingParser)
       this._trace( [ this.stack[length - 1].ruleState, 'recover inside rule' ] );
     return true;              // to be re-checked with actions
   }
 
   _recoverPanicMode() {
-    --this.tokenIdx
     const { length } = this.stack;
     // Panic mode: resume at token in then-expecting set:
     const followSets = { EOF: 0 };
@@ -584,7 +613,7 @@ class BaseParser {
       // TODO: handle Id here
       if (depth != null)
         return this._error_panic( depth, length, tokenIdx );
-      ++this.tokenIdx;
+      this.skipToken_();
     }
     throw Error( 'EOF was added...' );
   }
@@ -614,11 +643,12 @@ class BaseParser {
   }
 
   _stopParsing( idx ) {
-    --this.tokenIdx;
     if (this.constructor.tracingParser) {
       this.log( this.la().location.toString() + ':', 'Info:',
                 `leave all active ${ idx } rules prematurely, stop parsing` );
     }
+    // TODO: run this.skipToken_() on all remaining tokens?  Does ANTLR consumes
+    // those in error recovery mode?  Probably not.
     for (const c of this.stack)
       c.followState = null;
     this.errorState = null;
@@ -663,9 +693,9 @@ class BaseParser {
     if (this.constructor.tracingParser)
       this.trace.push( state ?? '⚠' );
   }
-  _traceIdOrPred( suffix ) {
+  _traceSubPush( state ) {
     if (this.constructor.tracingParser)
-      this.trace[this.trace.length - 1] += suffix;
+      this.trace.at(-1).push( state );
   }
   traceAction( location ) {     // will be put into tracing parser
     this._trace( 'execute action,', { location } );
@@ -678,7 +708,7 @@ class BaseParser {
       msg = this._rule( ...msg );
     this.trace.push( this.s ?? '⚠' );
     this.log( (la || this.la()).location.toString() + ':',
-              'Info:', msg, 'states:', this.trace.join( ' → ' ) );
+              'Info:', msg, 'states:', this.trace.map( traceStep ).join( ' → ' ) );
     this.trace = [ this.s ?? '⚠' ];
   }
 
@@ -704,17 +734,45 @@ class BaseParser {
 
 }
 
+function traceStep( step ) {
+  if (!Array.isArray( step ))
+    return step;
+  const result = { true: '✔', false: '✖' }[step.at( -1 )] ?? '';
+  const intro = (typeof step[1] === 'number') ? '→' : '';
+  const arg = step.slice( 1, result ? -1 : undefined ).join( '→' );
+  return `${ step[0] }(${ intro }${ arg })${ result }`;
+}
+
 function tokenName( type ) {
   if (typeof type !== 'string')
-    type = (!type.parsed || type.parsed === 'keyword') && type.keyword || type.type;
+    type = (!type.parsedAs || type.parsedAs === 'keyword') && type.keyword || type.type;
   return (/^[A-Z]+/.test( type )) ? `‹${ type }›` : `‘${ type }’`;
 }
 
 function tokenFullName( token, sep ) {
-  return (token.parsed && token.parsed !== 'keyword' && token.parsed !== 'token' ||
+  return (token.parsedAs && token.parsedAs !== 'keyword' && token.parsedAs !== 'token' ||
           token.type !== 'Id' && token.type !== token.text && token.text)
     ? `‘${ token.text }’${ sep }${ tokenName( token ) }`
     : tokenName( token );
 }
 
+function compileTable( table ) {
+  if (table.$compiled)
+    return table;
+  for (const line of table) {
+    if (typeof line !== 'object' || Array.isArray( line ))
+      continue;
+    const cache = Object.create( null ); // very sparse array
+    for (const prop of Object.keys( line )) {
+      const alt = line[prop];
+      if (!Array.isArray( alt ) && prop.charAt(0) !== ' ') // string or number
+        line[prop] = (typeof alt === 'string') ? line[alt] : (cache[alt] ??= [ 'g', alt ]);
+    }
+    if (!line[''])
+      line[''] = [ 'e' ];
+  }
+  table.$compiled = true;
+  return table;
+}
+
 module.exports = BaseParser;