@sap/cds-compiler 5.3.0 → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +30 -2
  2. package/bin/cdsc.js +1 -1
  3. package/doc/CHANGELOG_BETA.md +2 -2
  4. package/lib/api/options.js +4 -2
  5. package/lib/base/builtins.js +0 -10
  6. package/lib/base/keywords.js +3 -31
  7. package/lib/base/message-registry.js +23 -5
  8. package/lib/base/messages.js +1 -1
  9. package/lib/checks/existsMustEndInAssoc.js +7 -2
  10. package/lib/checks/foreignKeys.js +12 -7
  11. package/lib/compiler/assert-consistency.js +11 -3
  12. package/lib/compiler/builtins.js +2 -0
  13. package/lib/compiler/checks.js +88 -38
  14. package/lib/compiler/define.js +2 -2
  15. package/lib/compiler/shared.js +9 -10
  16. package/lib/compiler/xpr-rewrite.js +11 -0
  17. package/lib/compiler/xsn-model.js +1 -1
  18. package/lib/edm/csn2edm.js +2 -0
  19. package/lib/edm/edm.js +2 -1
  20. package/lib/edm/edmPreprocessor.js +14 -1
  21. package/lib/edm/edmUtils.js +17 -2
  22. package/lib/gen/BaseParser.js +291 -197
  23. package/lib/gen/CdlParser.js +1631 -1605
  24. package/lib/gen/Dictionary.json +74 -6
  25. package/lib/gen/language.checksum +1 -1
  26. package/lib/gen/language.interp +1 -1
  27. package/lib/gen/languageParser.js +1808 -1804
  28. package/lib/language/antlrParser.js +8 -4
  29. package/lib/language/genericAntlrParser.js +3 -3
  30. package/lib/model/csnUtils.js +6 -1
  31. package/lib/optionProcessor.js +4 -0
  32. package/lib/parsers/AstBuildingParser.js +172 -108
  33. package/lib/parsers/CdlGrammar.g4 +154 -134
  34. package/lib/parsers/Lexer.js +3 -3
  35. package/lib/parsers/identifiers.js +59 -0
  36. package/lib/render/toCdl.js +5 -5
  37. package/lib/render/utils/common.js +5 -0
  38. package/lib/render/utils/delta.js +23 -5
  39. package/lib/transform/db/expansion.js +2 -1
  40. package/lib/transform/db/rewriteCalculatedElements.js +11 -5
  41. package/lib/transform/db/transformExists.js +52 -26
  42. package/lib/transform/effective/annotations.js +147 -0
  43. package/lib/transform/effective/main.js +17 -3
  44. package/lib/transform/forOdata.js +53 -10
  45. package/lib/transform/forRelationalDB.js +8 -1
  46. package/lib/transform/odata/createForeignKeys.js +180 -0
  47. package/lib/transform/odata/flattening.js +135 -19
  48. package/lib/transform/odata/typesExposure.js +4 -3
  49. package/lib/transform/transformUtils.js +6 -6
  50. package/package.json +1 -1
@@ -1,7 +1,16 @@
1
- // Base class for generated parser, for redepage v0.1.12
1
+ // Base class for generated parser, for redepage v0.1.16
2
2
 
3
3
  'use strict';
4
4
 
5
+ // TODO: instance method
6
+ // name → true, list of predicates which are tested for rule exit
7
+ // const ruleExitPredicates = {};
8
+
9
+ // list of predicates which are tested when continue parsing after error starts,
10
+ // i.e. there is a predicate on the first token to match after recover example
11
+ // `afterBrace` or just method which by default just sets this.conditionTokenIdx
12
+ // and this.conditionStackLength and returns true?
13
+
5
14
  class BaseParser {
6
15
  constructor( lexer, keywords, table ) {
7
16
  this.keywords = keywords;
@@ -10,7 +19,9 @@ class BaseParser {
10
19
  this.tokens = undefined;
11
20
  this.eofIndex = undefined;
12
21
  this.tokenIdx = 0;
13
- this.conditionTokenIdx = -1;
22
+ this.recoverTokenIdx = -1;
23
+ this.conditionTokenIdx = -1; // TODO: can we use recoverTokenIdx ?
24
+ this.errorTokenIdx = -1;
14
25
  this.fixKeywordTokenIdx = -1;
15
26
  this.conditionStackLength = -1;
16
27
  this.nextTokenAsId = false;
@@ -22,7 +33,7 @@ class BaseParser {
22
33
  this.prec_ = null;
23
34
  this.$hasErrors = null;
24
35
  // trace:
25
- this.trace = [ -1 ];
36
+ this.trace = [];
26
37
  }
27
38
 
28
39
  init() {
@@ -60,28 +71,23 @@ class BaseParser {
60
71
 
61
72
  e() { // error: report and recover
62
73
  const la = this.tokens[this.tokenIdx];
63
- const expecting = this._expecting();
64
74
  if (this.trace.length > 1)
65
- this._trace( 'detected parsing error,' );
75
+ this._trace( 'detected parsing error,', la );
66
76
  this.reportUnexpectedToken_( la );
67
- la.parsedAs = 0;
68
-
69
- if (this.conditionTokenIdx === this.tokenIdx &&
70
- this.conditionStackLength === this.stack.length &&
71
- (la.keyword && expecting[la.keyword] || expecting[la.type])) {
72
- // called with/after gc()/gp(), and the token would actually match
73
- const { tokenIdx, ruleState } = this.stack.at( -1 );
74
- this.s = (this.tokenIdx > tokenIdx) ? this.errorState : ruleState;
75
- return false; // error recovery: ignore condition/precedence
77
+ la.parsedAs = ''; // current token is erroneous
78
+
79
+ if (this.errorTokenIdx === this.tokenIdx) {
80
+ // TODO: investigate why this is not handled otherwise
81
+ this.reportInternalError_( la );
82
+ this.skipToken_();
83
+ return false;
76
84
  }
85
+ this.errorTokenIdx = this.tokenIdx;
86
+ this.conditionStackLength = null;
77
87
 
78
- if (this.tokenIdx >= this.eofIndex)
79
- return this._stopParsing( this.stack.length );
80
- // TODO: also sync to what comes next in current rule, at least after rule call,
81
- // this way we do not have to do the check of g(0) in re() as we did before 2023-12-07
82
- // (not sure yet whether to make it part of recoverInline or recoverPanicMode),
83
- if (!this._recoverInline( expecting ))
84
- this._recoverPanicMode();
88
+ const { rewindDepth, syncSet } = this._calculateSyncSet();
89
+ const recoverDepth = this._findSyncToken( syncSet, rewindDepth );
90
+ this._recoverFromError( rewindDepth, recoverDepth );
85
91
  return false;
86
92
  }
87
93
 
@@ -166,8 +172,8 @@ class BaseParser {
166
172
  }
167
173
 
168
174
  // instead of g() in a non-default case if there is a LL1 conflict
169
- gP( state ) { // goto state with standard weak-conflict prediction
170
- return this.lP() && this.g( state );
175
+ gP( state, follow ) { // goto state with standard weak-conflict prediction
176
+ return this.lP( follow ) && this.g( state );
171
177
  }
172
178
 
173
179
  // match and consume token: ---------------------------------------------------
@@ -252,73 +258,52 @@ class BaseParser {
252
258
  // “or Id” behavior other than via gpP()
253
259
 
254
260
  // “go if user condition fails”
255
- gc( state, cond ) {
256
- if (this.conditionTokenIdx === this.tokenIdx &&
257
- this.conditionStackLength === this.stack.length) {
261
+ gc( state, cond, arg ) {
262
+ if (this.conditionTokenIdx === this.tokenIdx && // tested on same
263
+ this.conditionStackLength == null) { // after error recovery
258
264
  this._tracePush( [ 'C' ] );
259
- return true; // error recovery: ignore condition
265
+ return true;
260
266
  }
261
- this.conditionTokenIdx = this.tokenIdx;
262
- this.conditionStackLength = this.stack.length;
263
267
  // TODO: let this[cond]( true ) return recovery badness in error case
264
- const fail = !this[cond]( true );
265
- if (this.constructor.tracingParser)
266
- this._tracePush( [ 'C', cond, !fail ] );
267
- // TODO TOOL: in this case, the default case must not have actions (tool must
268
- // add state if it does)
269
- if (fail) { // TODO: extra gcK() method instead of check below
270
- // TODO: extra method necessary for academic case
271
- // ( 'unreserved' 'foo' | <cond> Id 'bar' )` with input `unreserved bar`
272
- const { keyword } = this.la();
273
- if (keyword && this.table[keyword])
274
- this.fixKeywordTokenIdx = this.tokenIdx;
268
+ if (this.constructor.tracingParser) {
269
+ const { traceName } = this[cond];
270
+ this._tracePush( [ 'C', traceName?.call( this, arg ) ?? cond ] );
275
271
  }
276
- return !fail || this.g( state ) && false;
277
- }
272
+ // calling the condition might have side effects (precendence conditions have)
273
+ // → call tracing “name” before
274
+ const fail = !this[cond]( true, arg );
275
+ if (this.constructor.tracingParser)
276
+ this._traceSubPush( !fail );
277
+ // The default case must not have actions. If written in grammar with action,
278
+ // the default must have <default=fallback>
278
279
 
279
- ec( cond ) {
280
- return this.gc( null, cond );
281
- }
282
280
 
283
- // “go if precedence condition fails”
284
- gp( state, prec, mode ) {
285
- if (this.conditionTokenIdx === this.tokenIdx &&
286
- this.conditionStackLength === this.stack.length) {
287
- this._tracePush( [ 'C' ] );
288
- return true; // error recovery: ignore condition
289
- }
290
- this.conditionTokenIdx = this.tokenIdx;
291
- this.conditionStackLength = this.stack.length;
292
- const parentPrec = this.stack.at( -1 ).prec ?? -Infinity;
293
- const fail = prec <= parentPrec ||
294
- this.prec_ != null && // previous op parsed by current rule
295
- // <…,postfix> || <…,assoc=none>, <…,postfix=once>:
296
- (mode === 'post' && prec > this.prec_ || mode === 'none' && prec >= this.prec_);
297
- if (this.constructor.tracingParser) {
298
- const pp = (parentPrec === -Infinity) ? '-∞' : parentPrec;
299
- const tp = (this.prec_ == null) ? '∞' : this.prec_;
300
- const suffix = mode === 'post' && `≤${ tp }` || mode === 'none' && `<${ tp }`;
301
- this._tracePush( [ 'C', `${ pp }<${ prec }${ suffix || '' }`, !fail ] );
302
- }
303
281
  if (fail) { // TODO: extra gcK() method instead of check below
282
+ // TODO: probably remove the following (and `conditionStackLength` tests)
283
+ // altogether, error with gr() should be enough
284
+ // if (this.conditionTokenIdx === this.tokenIdx &&
285
+ // this.conditionStackLength == this.stack.length)
286
+ // return this.e(); // already failed on same token in same rule
304
287
  // TODO: extra method necessary for academic case
305
288
  // ( 'unreserved' 'foo' | <cond> Id 'bar' )` with input `unreserved bar`
306
289
  const { keyword } = this.la();
307
290
  if (keyword && this.table[this.s][keyword])
308
291
  this.fixKeywordTokenIdx = this.tokenIdx;
309
- return this.g( state ) && false; // TODO: reset this.prec_ ?
292
+ this.conditionTokenIdx = this.tokenIdx;
293
+ this.conditionStackLength = this.stack.length;
310
294
  }
311
- this.prec_ = (mode === 'right') ? prec - 1 : prec; // -1: <…,assoc=right>, <…,prefix>
312
- return true;
295
+ return !fail || this.g( state ) && false;
313
296
  }
314
297
 
315
- ep( prec, mode ) {
316
- return this.gp( null, prec, mode );
298
+ ec( cond, arg ) {
299
+ return this.gc( null, cond, arg );
317
300
  }
318
301
 
319
302
  // rule start, end and call: --------------------------------------------------
320
303
 
321
304
  rule_( state, followState = -1 ) { // start rule
305
+ this.s = state;
306
+ this._trace( [ 'call rule', state, ' at alt start' ], this.la() );
322
307
  this.stack.push( {
323
308
  ruleState: state,
324
309
  followState,
@@ -326,26 +311,42 @@ class BaseParser {
326
311
  prec: this.prec_,
327
312
  } );
328
313
  this.dynamic_ = Object.create( this.dynamic_ );
329
- this.s = state;
330
314
  this.prec_ = null;
331
- this.conditionTokenIdx = -1;
332
315
  this.errorState ??= state;
333
- this._trace( [ state, 'call rule', '', ' at alt start', -1 ] );
334
316
  }
335
317
 
336
- exit_( rulePrecMethod ) { // exit rule
318
+ exit_() { // exit rule
337
319
  if (this.s)
338
320
  throw Error( `this.s === ${ this.s } // illegally set by action, or runtime/generator bug` );
339
321
  this.dynamic_ = Object.getPrototypeOf( this.dynamic_ );
340
322
  const caller = this.stack.pop();
323
+ const immediately = this.tokenIdx === caller.tokenIdx;
324
+ if (this.constructor.tracingParser) {
325
+ const post = this.s == null &&
326
+ (immediately
327
+ ? ' immediately'
328
+ : caller.followState == null
329
+ ? ' unsuccessfully'
330
+ : ' prematurely');
331
+ const text = immediately ? '⚠ exit rule' : '⏎ exit rule';
332
+ this.s = caller.followState; // for trace
333
+ this._trace( [ text, caller.ruleState, post, this.stack.length + 1 ],
334
+ this.la() );
335
+ if (this.tokenIdx === caller.tokenIdx &&
336
+ this.stack.at(-1)?.followState != null)
337
+ this.trace = [ this.errorState ]; // show last good state in trace
338
+ }
341
339
  this.s = caller.followState;
342
- this.prec_ = (rulePrecMethod) ? this[rulePrecMethod]( caller ) : caller.prec;
343
- this._trace( [ caller.ruleState, 'exit rule', '', '', 1 ] );
344
- //if (this.errorState == 0 || this.s != null)
340
+ this.prec_ = caller.prec;
341
+ if (this.s)
342
+ this._skipErrorTokens();
343
+ else if (this.s == null)
344
+ return !immediately; // attached actions are executed even with "unsuccessful exit"
345
+
346
+ if (immediately)
347
+ return false;
345
348
  this.errorState = this.s;
346
- // execute actions if not in error recovery (pass-through) and at least one
347
- // token has been matched in rule:
348
- return this.s != null && this.tokenIdx > caller.tokenIdx;
349
+ return true;
349
350
  }
350
351
 
351
352
  // predicate used before rule call if with LL(1) conflict, 'Id' in other case
@@ -359,10 +360,10 @@ class BaseParser {
359
360
  // Argument first2 is just a performance hint with ckP():
360
361
  if (lk2 && first2?.[0] === 'Id' && this.keywords[lk2] !== false ||
361
362
  first2?.includes( lk2 || lt2 )) {
362
- this._tracePush( [ 'P', true ] );
363
+ this._tracePush( [ 'K', true ] );
363
364
  return true;
364
365
  }
365
- this._tracePush( [ 'P' ] );
366
+ this._tracePush( [ 'K' ] );
366
367
  // now check it dynamically:
367
368
  let cmd = this.table[this.s][lk1];
368
369
  if (cmd[2] !== 1)
@@ -372,7 +373,7 @@ class BaseParser {
372
373
  const nextState = (cmd[0] === 'ck') ? cmd[1] : this._pred_keyword( cmd[1], lk1 );
373
374
 
374
375
  ++this.tokenIdx; // for user lookahead fns and conditions
375
- const match = this._pred_next( nextState, lt2, lk2, 'P' );
376
+ const match = this._pred_next( nextState, lt2, lk2, 'K' );
376
377
  --this.tokenIdx;
377
378
 
378
379
  const r = match ?? true;
@@ -484,10 +485,13 @@ class BaseParser {
484
485
 
485
486
  // Set of expected tokens: for error reporting and recovery -------------------
486
487
 
488
+ // method like _exp_collect - conditions in called rules are evaluated with
489
+ // unchanged stack and dynamic & no site-effects (are run with extra mode)
490
+
487
491
  // Calculate array of expected tokens
488
- _expecting( errorState, length ) {
492
+ _expecting( token ) {
489
493
  // Remark: rules must not have been exited too early, see _expecting call in re()
490
- const stack = this.stack.slice( 0, length || this.stack.length );
494
+ const stack = this.stack.slice( 0, this.stack.length );
491
495
  // Immediately exit rules when no tokens have yet been consumed:
492
496
  let caller = stack.at( -1 );
493
497
  while (stack.length && this.tokenIdx === caller.tokenIdx) {
@@ -496,11 +500,20 @@ class BaseParser {
496
500
  }
497
501
  // Now calculate dictionary of expected tokens:
498
502
  const expecting = Object.create(null);
499
- let state = errorState ?? this.errorState;
503
+ let state = this.errorState;
500
504
  // At potential rule end, we must add follow sets of outer rules
501
505
  // TODO: we also need to unravel this.dynamic_ for translateParserToken_()
502
506
  while ((!state || this._exp_collect( expecting, this.table[state] )) && stack.length)
503
507
  state = stack.pop().followState;
508
+
509
+ // Remove token (TODO later: instead, use conditions when collecting tokens):
510
+ if (token) {
511
+ const { keyword, type } = token;
512
+ if (keyword && expecting[keyword] === true)
513
+ delete expecting[keyword];
514
+ else if (expecting[type] === true)
515
+ delete expecting[type];
516
+ }
504
517
  return expecting;
505
518
  }
506
519
 
@@ -509,31 +522,41 @@ class BaseParser {
509
522
  // Return true if the rule end is reached, i.e. we also need to add the expected
510
523
  // tokens at the follow state of the current rule. Argument `prop` is the token
511
524
  // name for `cmd` in a decision.
512
- _exp_collect( expecting, cmd, prop ) {
525
+
526
+ // translateParserToken must work, i.e. this.stack and this.dynamic_ must be
527
+ // according to stack level
528
+ _exp_collect( expecting, cmd, prop, val = true ) {
513
529
  if (prop != null)
514
530
  cmd = cmd[prop];
531
+ else if (!cmd) // called on follow state of start rule
532
+ return false;
533
+
515
534
  if (!Array.isArray( cmd )) {
516
535
  let reachedRuleEnd = false;
517
536
  for (const tok in cmd) {
537
+ // TODO: except for `Id`, we can directly continue if `tok` is already in
538
+ // `expecting`
518
539
  if (Object.hasOwn( cmd, tok ) && tok.charAt(0) !== ' ' &&
519
- this._exp_collect( expecting, cmd, tok ))
540
+ this._exp_collect( expecting, cmd, tok, val ))
520
541
  reachedRuleEnd = true;
521
542
  }
522
543
  return reachedRuleEnd;
523
544
  }
524
545
  switch (cmd[0]) {
525
546
  case 'c': case 'ck':
526
- expecting[prop] = true;
547
+ expecting[prop] ??= val;
527
548
  return false;
528
549
  case 'ckA':
529
550
  for (const tok of this.translateParserToken_( prop ))
530
- expecting[tok] = true;
551
+ expecting[tok] ??= val;
531
552
  return false;
532
553
  case 'm': case 'mk':
533
- expecting[cmd[2]] = true;
554
+ expecting[cmd[2]] ??= val;
534
555
  return false;
535
556
  case 'ci': case 'ciA': case 'mi': case 'miA':
536
- expecting['Id'] = true;
557
+ expecting['Id'] ??= val;
558
+ // TODO: should we do s/th special, such that a reserved word is a sync
559
+ // token for Id<all>? Probably not, see also comment in _findSyncToken()
537
560
  return false;
538
561
  case 'g': case 'gi':
539
562
  if (!cmd[1])
@@ -544,12 +567,12 @@ class BaseParser {
544
567
  // UPDATE: no, there will be at least gP()s
545
568
  // TOOD: do properly for (...)+ - currently, the token for directly
546
569
  // exiting the rule is also collected
547
- return this._exp_collect( expecting, this.table[cmd[1]] );
570
+ return this._exp_collect( expecting, this.table[cmd[1]], undefined, val );
548
571
  default:
549
572
  // a called rule must match at least one token → after having called a
550
573
  // rule, do not collect expecting tokens after exiting the rule
551
574
  if (typeof cmd[0] === 'number')
552
- this._exp_collect( expecting, this.table[cmd[1]] );
575
+ this._exp_collect( expecting, this.table[cmd[1]], undefined, val );
553
576
  return false;
554
577
  }
555
578
  }
@@ -560,72 +583,72 @@ class BaseParser {
560
583
 
561
584
  // Error recovery -------------------------------------------------------------
562
585
 
563
- _recoverInline( expecting ) {
564
- const { type: lt2, keyword: lk2 } = this.tokens[this.tokenIdx + 1];
565
- if (!(lk2 && expecting[lk2] || expecting[lt2]))
566
- return false;
567
-
568
- // Immediately exit rules (except start) when no tokens have yet been consumed:
569
- let { length } = this.stack;
570
- while (--length > 0) {
571
- const caller = this.stack[length];
572
- // matched tokens in rule: found rule
573
- if (this.tokenIdx > caller.tokenIdx)
574
- break;
575
- caller.followState = null;
576
- }
577
-
578
- if (++length < this.stack.length) {
579
- this.s = null;
580
- this.stack[length].followState = this.errorState;
581
- // assume the erroneous token to be skipped before having called the rule:
582
- ++this.stack[length].tokenIdx
583
- this.errorState = null;
586
+ _calculateSyncSet() {
587
+ const { stack, dynamic_ } = this;
588
+ let { length } = stack;
589
+ while (stack[--length].tokenIdx === this.tokenIdx && length)
590
+ this.dynamic_ = Object.getPrototypeOf( this.dynamic_ );
591
+ this.stack = stack.slice( 0, ++length );
592
+
593
+ // needs (copy of) "real stack"
594
+ const syncSet = {};
595
+ let depth = length + 1;
596
+ if (!this._exp_collect( syncSet, this.table[this.errorState], undefined, depth ))
597
+ --depth;
598
+ while (this.stack.length) {
599
+ this.dynamic_ = Object.getPrototypeOf( this.dynamic_ );
600
+ const caller = this.stack.pop();
601
+ // this.stack and this.dynamic_ must be changed for parser token
602
+ // translation:
603
+ if (caller.followState > 0 &&
604
+ !this._exp_collect( syncSet, this.table[caller.followState], undefined, depth ))
605
+ depth = this.stack.length;
584
606
  }
585
- else { // no rule to leave immediately
586
- this.s = this.errorState;
607
+ syncSet.EOF ??= 0;
608
+ this.stack = stack;
609
+ this.dynamic_ = dynamic_;
610
+ return { rewindDepth: length, syncSet };
611
+ }
612
+
613
+ _findSyncToken( syncSet, rewindDepth ) {
614
+ this.recoverTokenIdx = this.tokenIdx;
615
+ while (this.recoverTokenIdx <= this.eofIndex) {
616
+ const { keyword, type } = this.tokens[this.recoverTokenIdx];
617
+ const tryKeyw = keyword ? syncSet[keyword] : null;
618
+ if (tryKeyw != null)
619
+ return tryKeyw;
620
+ const tryType = syncSet[type];
621
+ // sync to Id only if in expected set of last good state or if after ';'
622
+ if (tryType != null &&
623
+ (type !== 'Id' || (!keyword || this.keywords[keyword] !== false) &&
624
+ // reserved words do not match Id in expected-set, see _exp_collect()
625
+ (tryType > rewindDepth || this.tokens[this.recoverTokenIdx - 1].type === ';')))
626
+ return tryType;
627
+ ++this.recoverTokenIdx;
587
628
  }
629
+ throw Error( 'EOF must be last in `tokens`' );
630
+ }
588
631
 
589
- this.skipToken_();
590
- if (this.constructor.tracingParser)
591
- this._trace( [ this.stack[length - 1].ruleState, 'recover inside rule' ] );
592
- return true; // to be re-checked with actions
593
- }
594
-
595
- _recoverPanicMode() {
596
- const { length } = this.stack;
597
- // Panic mode: resume at token in then-expecting set:
598
- const followSets = { EOF: 0 };
599
- for (let idx = 0; idx < length; ++idx) {
600
- const caller = this.stack[idx];
601
- const exp = this._expecting( caller.followState, length );
602
- for (const t of Object.keys( exp )) {
603
- // no sync to 'Id' - TODO: provide grammar and rule options
604
- if (t !== 'Id') // TODO: see below
605
- followSets[t] = idx;
606
- }
632
+ _recoverFromError( rewindDepth, recoverDepth ) {
633
+ this.s = null;
634
+ let depth = this.stack.length;
635
+ if (recoverDepth > depth) { // no rewind, no rule exit
636
+ this.trace = [ this.errorState ]; // show last good state in trace
637
+ this.s = this.errorState;
638
+ if (this.s)
639
+ this._skipErrorTokens();
607
640
  }
608
- const tokenIdx = this.tokenIdx;
609
- // console.log( this.la().location.toString(), followSets )
610
- while (this.tokenIdx <= this.eofIndex) {
611
- // TODO: exclude reserved words for test with this.l()
612
- const depth = followSets[this.lk()] || followSets[this.l()];
613
- // TODO: handle Id here
614
- if (depth != null)
615
- return this._error_panic( depth, length, tokenIdx );
616
- this.skipToken_();
641
+ else if (recoverDepth > rewindDepth) { // rewind, no rule exit
642
+ this.stack[rewindDepth].followState = this.errorState;
617
643
  }
618
- throw Error( 'EOF was added...' );
619
- }
644
+ while (depth > recoverDepth)
645
+ this.stack[--depth].followState = null;
646
+ // TODO: when the error is due to failed rule exit prediction, try to keep
647
+ // existing followState (if that reaches RuleEnd_)
648
+ // Continue parsing: ignore next predicate (TODO: except some specified ones?)
649
+ this.conditionTokenIdx = this.tokenIdx;
650
+ this.conditionStackLength = null;
620
651
 
621
- _error_panic( low, high, tokenIdx ) {
622
- this.s = null; // mark current rule for exit
623
- if (this.constructor.tracingParser) {
624
- this._trace( this.stack.length - 1 > low
625
- ? `recover by exiting ${ this.stack.length - low} rules prematurely,`
626
- : 'recover by exiting current rule prematurely,' );
627
- }
628
- // eventually mark outer rules for exit:
629
652
  // TODO: re-check for rule calls which are at the optional rule end:
630
653
  // x: 'x not'; b: 'b'? x {console.log('x→b')} 'b'?; a: b {console.log('b→a')} 'a'
631
654
  // with start rule `a` and input `x a`: output should be x→b + b→a
@@ -633,27 +656,15 @@ class BaseParser {
633
656
  //
634
657
  // → the rule is: if a rule can continue at the specified state and has
635
658
  // matched at least one token, then its action is executed, otherwise not
636
- for (let idx = low + 1; idx < high; ++idx) {
637
- this.stack[idx].followState = null;
638
- }
639
- const resume = this.stack[low];
640
- if (tokenIdx === resume.tokenIdx) // no tokens matched other than those by skipping
641
- resume.tokenIdx = this.tokenIdx; // make exit_() return false
642
- this.errorState = null;
643
659
  }
644
660
 
645
- _stopParsing( idx ) {
646
- if (this.constructor.tracingParser) {
647
- this.log( this.la().location.toString() + ':', 'Info:',
648
- `leave all active ${ idx } rules prematurely, stop parsing` );
649
- }
650
- // TODO: run this.skipToken_() on all remaining tokens? Does ANTLR consumes
651
- // those in error recovery mode? Probably not.
652
- for (const c of this.stack)
653
- c.followState = null;
654
- this.errorState = null;
655
- this.s = null;
656
- return false;
661
+ _skipErrorTokens() {
662
+ if (this.constructor.tracingParser && this.tokenIdx <= this.recoverTokenIdx) {
663
+ this._trace( `skipped ${ this.recoverTokenIdx - this.tokenIdx } tokens to recover from error,`,
664
+ this.tokens[this.recoverTokenIdx] );
665
+ }
666
+ while (this.tokenIdx < this.recoverTokenIdx)
667
+ this.skipToken_();
657
668
  }
658
669
 
659
670
  // small methods --------------------------------------------------------------
@@ -662,31 +673,39 @@ class BaseParser {
662
673
  console.log( ...args );
663
674
  }
664
675
 
665
- expectingForMessage_( sep = ',' ) {
666
- return Object.keys( this._expecting() ).map( tokenName ).sort().join( sep );
676
+ expectingForMessage_( token ) {
677
+ return Object.keys( this._expecting( token ) ).map( tokenName ).sort().join( ',' );
667
678
  }
668
679
 
669
680
  reportError_( location, text ) {
670
681
  this.$hasErrors = true;
671
- this.log( `${ location }: Error:`, text );
682
+ this.log( `${ location }:`, text );
672
683
  }
673
684
 
674
685
  reportUnexpectedToken_( token ) {
675
686
  this.reportError_( token.location,
676
- `unexpected token ${ tokenFullName( token, ': ' ) } - expecting: ` +
677
- this.expectingForMessage_() );
687
+ `Unexpected token ${ tokenFullName( token, ': ' ) } - expecting: ` +
688
+ this.expectingForMessage_( token ) );
689
+ }
690
+
691
+ reportInternalError_( token ) {
692
+ this.reportError_( token.location,
693
+ `Unexpected token at ${ tokenFullName( token, ': ' ) } - skipped one token` );
678
694
  }
679
695
 
680
696
  reportReservedWord_( token ) {
681
697
  this.reportError_( token.location,
682
- `unexpected reserved word ‘${ token.text }’ - expecting: ` +
698
+ `Unexpected reserved word ‘${ token.text }’ - expecting: ` +
683
699
  this.expectingForMessage_() );
684
700
  }
685
701
 
686
702
  errorAndRecoverOutside( token, text ) { // TODO: re-check
703
+ // TODO: TMP
687
704
  this.reportError_( token.location, text );
688
- ++this.tokenIdx;
689
- return this._recoverPanicMode( this.stack.length );
705
+ while (this.l() !== ';')
706
+ this.skipToken_();
707
+ this.s = null;
708
+ return false;
690
709
  }
691
710
 
692
711
  _tracePush( state ) {
@@ -697,31 +716,55 @@ class BaseParser {
697
716
  if (this.constructor.tracingParser)
698
717
  this.trace.at(-1).push( state );
699
718
  }
700
- traceAction( location ) { // will be put into tracing parser
701
- this._trace( 'execute action,', { location } );
719
+ traceAction( location ) { // TODO: remove
720
+ this._trace( location );
702
721
  }
703
722
 
704
723
  _trace( msg, la ) {
705
724
  if (!this.constructor.tracingParser)
706
725
  return;
707
- if (Array.isArray( msg ))
708
- msg = this._rule( ...msg );
726
+ // indentation according to rule call depth is nice, but only if without
727
+ // excessive spaces truncate:
728
+ const indent = ' '.repeat( this.stack.length % 32 );
729
+ if (!la) {
730
+ let line = ' execute action'; // align with non-action messages
731
+ if (this.trace.length > 1) { // i.e. with some 'g' command
732
+ line += ', states: ' + this.trace.map( traceStep ).join( ' → ' );
733
+ this.trace = [ this.s ?? '⚠' ];
734
+ }
735
+ this.log( indent, line, `(${ msg })` );
736
+ return;
737
+ }
738
+ const { location } = la;
739
+ if (!this.trace.length) {
740
+ this.log( `In ${ location.file }:` );
741
+ this.trace = [ -1 ];
742
+ }
709
743
  this.trace.push( this.s ?? '⚠' );
710
- this.log( (la || this.la()).location.toString() + ':',
711
- 'Info:', msg, 'states:', this.trace.map( traceStep ).join( ' → ' ) );
744
+ if (Array.isArray( msg )) {
745
+ const [ intro, state, finale, exitLength ] = msg;
746
+ let depth = (exitLength) ? exitLength - 1 : this.stack.length + 1;
747
+ let length = this.trace.length - 1;
748
+ this.trace[length] = `${ this.trace[length] }(${ depth })`;
749
+ depth = exitLength || this.stack.length;
750
+ while (length && typeof this.trace[--length] !== 'number')
751
+ ;
752
+ this.trace[length] = `${ this.trace[length] }(${ depth })`;
753
+
754
+ let start = state;
755
+ while (typeof this.table[--start] !== 'string')
756
+ ;
757
+ const post = (exitLength || start + 1 < state) && finale;
758
+ msg = `${ intro } “${ this.table[start] }”${ post || '' },`;
759
+ }
760
+ // Yes, I know util.format, but do not want to have a `require` in this file
761
+ const line = location.line < 1e5 ? ` ${ location.line }`.slice(-5) : `${ location.line }`;
762
+ const col = location.col < 1e4 ? `:${ location.col } `.slice(0,5) : `:${location.col }`;
763
+ this.log( line + col + indent + msg,
764
+ 'states:', this.trace.map( traceStep ).join( ' → ' ) );
712
765
  this.trace = [ this.s ?? '⚠' ];
713
766
  }
714
767
 
715
- // TODO: rename to ruleName_, leaving out the msg stuff
716
- _rule( state, msg, post = '', postOther = post, depthDiff ) {
717
- const start = --state;
718
- while (typeof this.table[state] !== 'string')
719
- --state;
720
- const { length } = this.stack;
721
- const depth = depthDiff ? `, depth ${ length + depthDiff } → ${ length }` : '';
722
- return `${ msg } “${ this.table[state] }”${ state < start ? postOther : post }${ depth },`;
723
- }
724
-
725
768
  inSameRule_( lowState, highState ) {
726
769
  if (lowState > highState)
727
770
  [ lowState, highState ] = [ highState, lowState ];
@@ -732,6 +775,57 @@ class BaseParser {
732
775
  return true;
733
776
  }
734
777
 
778
+ // Predefined conditions with extra option names:
779
+
780
+ precLeft_( _test, prec ) { // <prec=…>, <…,assoc=left>, <…,prefix=once>
781
+ const parentPrec = this.stack.at( -1 ).prec;
782
+ if (parentPrec != null && parentPrec >= prec)
783
+ return false;
784
+ this.prec_ = prec;
785
+ return true;
786
+ }
787
+ precRight_( _test, prec ) { // <…,assoc=right>, <…,prefix>
788
+ const parentPrec = this.stack.at( -1 ).prec;
789
+ if (parentPrec != null && parentPrec >= prec)
790
+ return false;
791
+ this.prec_ = prec - 1;
792
+ return true;
793
+ }
794
+ precNone_( _test, prec ) { // <…,assoc=none>, <…,postfix=once>
795
+ const parentPrec = this.stack.at( -1 ).prec;
796
+ if (parentPrec != null && parentPrec >= prec ||
797
+ this.prec_ != null && this.prec_ <= prec)
798
+ return false;
799
+ this.prec_ = prec;
800
+ return true;
801
+ }
802
+ precPost_( _test, prec ) { // <…,postfix>
803
+ const parentPrec = this.stack.at( -1 ).prec;
804
+ if (parentPrec != null && parentPrec >= prec ||
805
+ this.prec_ != null && this.prec_ < prec)
806
+ return false;
807
+ this.prec_ = prec;
808
+ return true;
809
+ }
810
+ }
811
+ const members = BaseParser.prototype;
812
+ // functions below are to be called with `call` to set `this`
813
+
814
+ members.precLeft_.traceName = function( prec ) {
815
+ const parentPrec = this.stack.at( -1 ).prec;
816
+ return `${ parentPrec ?? '-∞' }<${ prec }`;
817
+ }
818
+ members.precRight_.traceName = function( prec ) {
819
+ const left = this.precLeft_.traceName.call( this, prec );
820
+ return `${ left },↓`;
821
+ }
822
+ members.precNone_.traceName = function( prec ) {
823
+ const left = this.precLeft_.traceName.call( this, prec );
824
+ return `${ left }<${ this.prec_ == null ? '∞' : this.prec_ }`;
825
+ }
826
+ members.precPost_.traceName = function( prec ) {
827
+ const left = this.precLeft_.traceName.call( this, prec );
828
+ return `${ left }≤${ this.prec_ == null ? '∞' : this.prec_ }`;
735
829
  }
736
830
 
737
831
  function traceStep( step ) {