@sap/cds-compiler 2.15.8 → 3.1.0

package/lib/language/genericAntlrParser.js

@@ -7,12 +7,13 @@
 'use strict';
 
 const antlr4 = require('antlr4');
-const { ATNState } = require('antlr4/atn/ATNState');
+const { ATNState } = require('antlr4/src/antlr4/atn/ATNState');
 const { dictAdd, dictAddArray } = require('../base/dictionaries');
 const locUtils = require('../base/location');
 const { parseDocComment } = require('./docCommentParser');
 const { parseMultiLineStringLiteral } = require('./multiLineStringParser');
-const { functionsWithoutParens, specialFunctions } = require('../compiler/builtins');
+const { functionsWithoutParens, specialFunctions, quotedLiteralPatterns } = require('../compiler/builtins');
+const { pathName } = require("../compiler/utils");
 
 const $location = Symbol.for('cds.$location');
 
@@ -32,118 +33,91 @@ function _message( parser, severity, id, loc, ...args ) {
 //   this.<function>(...)
 // in the actions inside the grammar.
 //
-function GenericAntlrParser( ...args ) {
-  // ANTLR restriction: we cannot add parameters to the constructor.
-  antlr4.Parser.call( this, ...args );
-  this.buildParseTrees = false;
-
-  // Common properties.
-  // We set them here so that they are available in the prototype.
-  // This improved performance by 25% for certain scenario tests.
-  // Probably because there was no need to look up the prototype chain anymore.
-  this.$adaptExpectedToken = null;
-  this.$adaptExpectedExcludes = [ ];
-  this.$nextTokensToken = null;
-  this.$nextTokensContext = null;
-
-  this.prepareGenericKeywords();
-  this.options = {};
-
-  return this;
-}
-
-// When we define this class with the ES6 `class` syntax, we get
-//   TypeError: Class constructors cannot be invoked without 'new'
-// Reason: the generated ANTLR constructor calls its super constructor via
-// old-style `<super>.call(this,...)`, not via `super(...)`.
-
-GenericAntlrParser.prototype = Object.assign(
-  Object.create( antlr4.Parser.prototype ), {
-    message: function(...args) { return _message( this, 'message', ...args ); },
-    error: function(...args) { return _message( this, 'error', ...args ); },
-    warning: function(...args) { return _message( this, 'warning', ...args ); },
-    info: function(...args) { return _message( this, 'info', ...args ); },
-    attachLocation,
-    startLocation,
-    tokenLocation,
-    valueWithTokenLocation,
-    previousTokenAtLocation,
-    combinedLocation,
-    createDict,
-    setDictEndLocation,
-    surroundByParens,
-    unaryOpForParens,
-    leftAssocBinaryOp,
-    classifyImplicitName,
-    fragileAlias,
-    identAst,
-    functionAst,
-    valuePathAst,
-    signedExpression,
-    numberLiteral,
-    quotedLiteral,
-    pathName,
-    docComment,
-    addDef,
-    addItem,
-    artifactForElementAnnotateOrExtend,
-    assignProps,
-    createPrefixOp,
-    setOnce,
-    setMaxCardinality,
-    pushIdent,
-    handleComposition,
-    associationInSelectItem,
-    reportExpandInline,
-    checkTypeFacet,
-    notSupportedYet,
-    csnParseOnly,
-    disallowElementExtension,
-    noAssignmentInSameLine,
-    noSemicolonHere,
-    setLocalToken,
-    setLocalTokenIfBefore,
-    excludeExpected,
-    isStraightBefore,
-    meltKeywordToIdentifier,
-    prepareGenericKeywords,
-    parseMultiLineStringLiteral,
-    constructor: GenericAntlrParser, // keep this last
-  }
-);
-
-// Patterns for literal token tests and creation.  The value is a map from the
-// `prefix` argument of function `quotedliteral` to the following properties:
-//  - `test_msg`: error message which is issued if `test_fn` or `test_re` fail.
-//  - `test_fn`: function called with argument `value`, fails falsy return value
-//  - `test_re`: regular expression, fails if it does not match argument `value`
-//  - `unexpected_msg`: error message which is issued if `unexpected_char` matches
-//  - `unexpected_char`: regular expression matching an illegal character in `value`,
-//    the error location is only correct for a literal <prefix>'<value>'
-//  - `literal`: the value which is used instead of `prefix` in the AST
-//  - `normalized`: function called with argument `value`, return value is used
-//    instead of `value` in the AST
-// TODO: think about laxer regexp for date/time/timestamp - normalization?
-const quotedLiteralPatterns = {
-  x: {
-    test_msg: 'A binary literal must have an even number of characters',
-    test_fn: (str => Number.isInteger(str.length / 2)),
-    unexpected_msg: 'A binary literal must only contain characters 0-9, a-f and A-F',
-    unexpected_char: /[^0-9a-f]/i,
-  },
-  time: {
-    test_msg: 'Expected time\'HH:MM:SS\' where H, M and S are numbers and \':SS\' is optional',
-    test_re: /^[0-9]{1,2}:[0-9]{1,2}(:[0-9]{1,2})?$/,
-  },
-  date: {
-    test_msg: 'Expected date\'YYYY-MM-DD\' where Y, M and D are numbers',
-    test_re: /^[0-9]{1,4}-[0-9]{1,2}-[0-9]{1,2}$/,
-  },
-  timestamp: {
-    test_msg: 'Expected timestamp\'YYYY-MM-DD HH:MM:SS.u…u\' where Y, M, D, H, S and u are numbers (optional 1-7×u)',
-    test_re: /^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}(:[0-9]{2}(\.[0-9]{1,7})?)?$/,
-  },
-};
+class GenericAntlrParser extends antlr4.Parser {
+  constructor( ...args ) {
+    // ANTLR restriction: we cannot add parameters to the constructor.
+    super( ...args );
+    this.buildParseTrees = false;
+
+    // Common properties.
+    // We set them here so that they are available in the prototype.
+    // This improved performance by 25% for certain scenario tests.
+    // Probably because there was no need to look up the prototype chain anymore.
+    this.$adaptExpectedToken = null;
+    this.$adaptExpectedExcludes = [ ];
+    this.$nextTokensToken = null;
+    this.$nextTokensContext = null;
+
+    this.options = {};
+
+    this.genericFunctionsStack = [];
+    this.$genericKeywords = specialFunctions[''][1];
+  }
+}
+
+// TODO: Use actual methods.
+Object.assign(GenericAntlrParser.prototype, {
+  message: function(...args) { return _message( this, 'message', ...args ); },
+  error: function(...args) { return _message( this, 'error', ...args ); },
+  warning: function(...args) { return _message( this, 'warning', ...args ); },
+  info: function(...args) { return _message( this, 'info', ...args ); },
+  attachLocation,
+  assignAnnotation,
+  addAnnotation,
+  checkExtensionDict,
+  handleDuplicateExtension,
+  startLocation,
+  tokenLocation,
+  isMultiLineToken,
+  fixMultiLineTokenEndLocation,
+  valueWithTokenLocation,
+  previousTokenAtLocation,
+  combinedLocation,
+  surroundByParens,
+  unaryOpForParens,
+  leftAssocBinaryOp,
+  classifyImplicitName,
+  fragileAlias,
+  identAst,
+  functionAst,
+  setLastAsXpr,
+  xprToken,
+  valuePathAst,
+  signedExpression,
+  numberLiteral,
+  quotedLiteral,
+  pathName,
+  docComment,
+  addDef,
+  addItem,
+  addExtension,
+  createSource,
+  createDict,
+  createArray,
+  finalizeDictOrArray,
+  createPrefixOp,
+  setOnce,
+  setMaxCardinality,
+  pushIdent,
+  handleComposition,
+  associationInSelectItem,
+  reportExpandInline,
+  checkTypeFacet,
+  notSupportedYet,
+  csnParseOnly,
+  disallowElementExtension,
+  noAssignmentInSameLine,
+  noSemicolonHere,
+  setLocalToken,
+  setLocalTokenIfBefore,
+  setLocalTokenForId,
+  excludeExpected,
+  isStraightBefore,
+  meltKeywordToIdentifier,
+  prepareGenericKeywords,
+  reportErrorForGenericKeyword,
+  parseMultiLineStringLiteral,
+});
 
 // Use the following function for language constructs which we (currently)
 // just being able to parse, in able to run tests from HANA CDS.  As soon as we
@@ -163,8 +137,8 @@ function notSupportedYet( text, ...tokens ) {
 }
 
 // Use the following function for language constructs which we (currently) do
-// not really compile, just use to produce a CSN for functions parseToCqn() and
-// parseToExpr().
+// not really compile, just use to produce a CSN for functions parse.cql() and
+// parse.expr().
 function csnParseOnly( text, ...tokens ) {
   if (!text || this.options.parseOnly)
     return;
@@ -218,6 +192,15 @@ function setLocalTokenIfBefore( string, tokenName, before, inSameLine ) {
     ll1.type = this.constructor[tokenName];
 }
 
+function setLocalTokenForId( tokenNameMap ) {
+  const tokenName = tokenNameMap[ this._input.LT(2).text || '' ];
+  const ll1 = this.getCurrentToken();
+  if (tokenName &&
+      (ll1.type === this.constructor.Identifier || /^[a-zA-Z_]+$/.test( ll1.text )))
+    ll1.type = this.constructor[tokenName];
+  return !!tokenName;
+}
+
 // // Special function for rule `requiredSemi` before return $ctx
 // function braceForSemi() {
 //   if (RBRACE == null)
@@ -275,24 +258,47 @@ function meltKeywordToIdentifier( exceptTrueFalseNull = false ) {
     token.type = Identifier;
 }
 
-function prepareGenericKeywords( pathItem ) {
-  this.$genericKeywords = { argFull: [] };
-  if (!pathItem)
-    return;
-  const func = pathItem.id && specialFunctions[pathItem.id.toUpperCase()];
-  const spec = func && func[pathItem.args ? pathItem.args.length : 0];
-  if (!spec)
-    return;
-  // currently, we only have 'argFull', i.e. a keyword which is alternative to expression
-  // TODO: If not just at the beginning, we need a stack for $genericKeywords,
-  // as we can have nested special functions
-  this.$genericKeywords.argFull = Object.keys( spec );
+const genericTokenTypes = {
+  expr: 'GenericExpr',
+  separator: 'GenericSeparator',
+  intro: 'GenericIntro',
+};
+
+function prepareGenericKeywords( pathItem, expected = null) {
+  const length = pathItem?.args?.length || 0;
+  const argPos = (expected ? length -1 : length);
+  const func = pathItem?.id && specialFunctions[pathItem.id.toUpperCase()];
+  const spec = func && func[argPos] || specialFunctions[''][argPos ? 1 : 0];
+  this.$genericKeywords = spec;
   // @ts-ignore
   const token = this.getCurrentToken() || { text: '' };
-  if (spec[token.text.toUpperCase()] === 'argFull') {
-    // @ts-ignore
-    token.type = this.constructor.GenericArgFull;
+  const text = token.text.toUpperCase();
+  let generic = spec[text];
+  // console.log('PGK:',token.text,generic,expected,spec,func,argPos)
+  if (expected) {               // 'separator' or 'expr' (after 'separator')
+    if (generic !== expected)
+      return;
+  }
+  else if (!generic || generic === 'separator') {
+    // Mismatch at beginning (or just an expression): keep token type
+    // (if not expression, issue error and consider the token to be an
+    // expression replacement, like ALL)
+    return;
+  }
+  else if (generic === 'expr' && spec.intro && spec.intro.includes( text )) {
+    // token is both an intro and an expression, like LEADING for TRIM
+    const next = this._input.LT(2).text;
+    if (!next || // followed by EOF -> consider it to be 'intro', better for CC
+        next !== ',' && next !== ')' && spec[next.toUpperCase()] !== 'separator')
+      generic = 'intro';        // is intro if next token is not separator, not ',', ')'
   }
+  // @ts-ignore
+  token.type = this.constructor[genericTokenTypes[generic]];
+}
+// To be called before having matched ( HideAlternatives | … )
+function reportErrorForGenericKeyword() {
+  this._errHandler.reportUnwantedToken( this );
+  //this._errHandler.reportInputMismatch( this, { offending: this._input.LT(1) }, null );
 }
 
 // Attach location matched by current rule to node `art`.  If a location is
@@ -302,14 +308,133 @@ function prepareGenericKeywords( pathItem ) {
 function attachLocation( art ) {
   if (!art || art.$parens)
     return art;
-  if (!art.location)
-    art.location = this.startLocation();
-  const { stop } = this._ctx;
-  art.location.endLine = stop.line;
-  art.location.endCol = stop.stop - stop.start + stop.column + 2; // after the last char (special for EOF?)
+  if (!art.location) {
+    art.location = this.tokenLocation(this._ctx.start, this._ctx.stop);
+    return art;
+  }
+
+  // The last token (this._ctx.stop) may be a multi-line string literal, in which
+  // case we can't rely on `this._ctx.stop.line`.
+  if (this.isMultiLineToken(this._ctx.stop)) {
+    this.fixMultiLineTokenEndLocation(this._ctx.stop, art.location);
+
+  } else {
+    const { stop } = this._ctx;
+    art.location.endLine = stop.line;
+    art.location.endCol = stop.stop - stop.start + stop.column + 2; // after the last char (special for EOF?)
+  }
+
   return art;
 }
 
+function assignAnnotation( art, anno, prefix = '', iHaveVariant = false ) {
+  const { name, $flatten } = anno;
+  const { path } = name;
+  if (path.broken || !path[path.length - 1].id)
+    return;
+  const pathname = pathName( path );
+  let absolute = '';
+  if (name.variant) {
+    const variant = pathName( name.variant.path );
+    absolute = `${ prefix }${ pathname }#${ variant }`;
+    if (iHaveVariant) {       // TODO: do we really care in the parser / core compiler?
+      this.error( 'anno-duplicate-variant', [ name.variant.location ],
+                  {},      // TODO: params
+                  'Annotation variant has been already provided' );
+    }
+  }
+  else if (!prefix || pathname !== '$value') {
+    absolute = `${ prefix }${ pathname }`;
+  }
+  else {
+    absolute = prefix.slice( 0, -1 );
+  }
+  if ($flatten) {
+    for (const a of $flatten)
+      this.assignAnnotation( art, a, `${ absolute }.`, iHaveVariant || name.variant);
+  }
+  else {
+    name.absolute = absolute;
+    this.addAnnotation( art, '@' + absolute, anno );
+  }
+  if (!prefix) {                // set deprecated $annotations for cds-lsp
+    if (!art.$annotations)
+      art.$annotations = [];
+    const location = locUtils.combinedLocation( anno.name, anno );
+    art.$annotations.push( { value: anno, location } );
+  }
+}
+
+function addAnnotation( art, prop, anno ) {
+  dictAddArray( art, prop, anno, (n, location, a) => {
+    // if we would make it a warning, we would still need to keep it an error
+    // with '...'; otherwise parse.cdl would have to split annotate statements
+    this.error( 'syntax-duplicate-anno', [ location ], { anno: n },
+                'Duplicate assignment with $(ANNO)' );
+    a.$errorReported = 'syntax-duplicate-anno';
+    // do not report again later as anno-duplicate-xyz
+  } );
+}
+
+const extensionDicts = { elements: true, enum: true, params: true, returns: true };
+
+function checkExtensionDict( dict ) {
+  for (const name in dict) {
+    const def = dict[name];
+    if (!def.$duplicates)
+      continue;
+
+    if (def.kind !== 'annotate') {
+      const numDefines =
+            def.$duplicates.reduce( addOneForDefinition, addOneForDefinition( 0, def ) );
+      this.handleDuplicateExtension( def, name, numDefines );
+      for (const dup of def.$duplicates)
+        this.handleDuplicateExtension( dup, name, numDefines );
+      continue;
+    }
+    // move annotations, 'doc' and 'elements' etc to main member
+    for (const dup of def.$duplicates) {
+      for (const prop of Object.keys( dup )) {
+        if (prop.charAt(0) === '@') {
+          this.addAnnotation( def, prop, dup[prop] )
+        }
+        else if (prop === 'doc') {
+          if (def.doc)
+            this.warning( 'syntax-duplicate-doc-comment', def.doc.location, {},
+                          'Doc comment is overwritten by another one below' );
+          def.doc = dup.doc;
+        }
+        else if (extensionDicts[prop]) {
+          if (def[prop])
+            this.message( 'syntax-duplicate-annotate', [ def.name.location ], { name, prop } );
+          def[prop] = dup[prop]; // continuation semantics: last wins
+        }
+      }
+    }
+    def.$duplicates = null;
+  }
+}
+
+function addOneForDefinition( count, ext ) {
+  return (ext.kind === 'extend') ? count : count + 1;
+}
+
+/**
+ * Handle duplicate extensions.  Does not handle `annotate`.
+ *
+ * @param {XSN.Extension} ext
+ * @param {string} name
+ * @param {number} numDefines
+ */
+function handleDuplicateExtension( ext, name, numDefines ) {
+  if (ext.kind === 'extend')
+    this.error( 'syntax-duplicate-extend', [ ext.name.location ],
+                { name, '#': (numDefines ? 'define' : 'extend') } );
+  else if (numDefines === 1)
+    ext.$errorReported = 'syntax-duplicate-extend'; // a definition, but not duplicate
+}
+
+
 /**
  * Return start location of `token`, or the first token matched by the current
  * rule if `token` is undefined
@@ -350,31 +475,49 @@ function tokenLocation( token, endToken = null ) {
 
   // This check is done for performance reason. No need to access a token's
   // data if we know that it spans only one single line.
-  const isMultiLineToken = (
-    endToken.type === this.constructor.DocComment ||
-    endToken.type === this.constructor.String ||
-    endToken.type === this.constructor.UnterminatedLiteral
+  if (this.isMultiLineToken(token))
+    this.fixMultiLineTokenEndLocation(token, loc);
+
+  return loc;
+}
+
+function isMultiLineToken(token) {
+  return (
+    token.type === this.constructor.DocComment ||
+    token.type === this.constructor.String ||
+    token.type === this.constructor.UnterminatedLiteral
   );
-  if (isMultiLineToken) {
-    // Count the number of newlines in the token.
-    const source = endToken.source[1].data;
-    let newLineCount = 0;
-    let lastNewlineIndex = endToken.start;
-    for (let i = endToken.start; i < endToken.stop; i++) {
-      // Note: We do NOT check for CR, LS, and PS (/[\r\u2028\u2029]/)
-      //       because ANTLR only uses LF for line break detection.
-      if (source[i] === 10) { // code point of '\n'
-        newLineCount++;
-        lastNewlineIndex = i;
-      }
-    }
-    if (newLineCount > 0) {
-      loc.endLine = endToken.line + newLineCount;
-      loc.endCol = endToken.stop - lastNewlineIndex + 1;
+}
+
+/**
+ * Adapt end location of `location` according to `token`, assuming that `token` is a multi-line
+ * token such as a multi-line string or doc comment.
+ *
+ * Sets `endLine`/`endCol`, respecting newline characters in the token.
+ *
+ * @param token
+ * @param {CSN.Location} location
+ */
+function fixMultiLineTokenEndLocation( token, location ) {
+  // Count the number of newlines in the token.
+  const source = token.source[1].data;
+  let newLineCount = 0;
+  let lastNewlineIndex = token.start;
+  for (let i = token.start; i < token.stop; i++) {
+    // Note: We do NOT check for CR, LS, and PS (/[\r\u2028\u2029]/)
+    //       because ANTLR only uses LF for line break detection.
+    if (source[i] === 10) { // code point of '\n'
+      newLineCount++;
+      lastNewlineIndex = i;
     }
   }
-
-  return loc;
+  if (newLineCount > 0) {
+    location.endLine = token.line + newLineCount;
+    location.endCol = token.stop - lastNewlineIndex + 1;
+  } else {
+    location.endLine = token.line;
+    location.endCol = token.stop - token.start + token.column + 2; // after the last char (special for EOF?)
+  }
 }
 
 /**
@@ -409,20 +552,6 @@ function combinedLocation( start, end ) {
   return locUtils.combinedLocation( start, end );
 }
 
-function createDict( location = null ) {
-  const dict = Object.create(null);
-  dict[$location] = location || this.startLocation( this._input.LT(-1) );
-  return dict;
-}
-
-function setDictEndLocation( dict ) {
-  const stop = this._input.LT(-1);
-  Object.assign( dict[$location], {
-    endLine: stop.line,
-    endCol: stop.stop - stop.start + stop.column + 2,
-  } );
-}
-
 function surroundByParens( expr, open, close, asQuery = false ) {
   if (!expr)
     return expr;
@@ -435,7 +564,7 @@ function surroundByParens( expr, open, close, asQuery = false ) {
 }
 
 function unaryOpForParens( query, val ) {
-  const parens = query.$parens;
+  const parens = query?.$parens;
   if (!parens)
     return query;
   const location = parens[parens.length - 1];
@@ -461,8 +590,8 @@ function docComment( node ) {
   if (!this.options.docComment)
     return;
   if (node.doc) {
-    this.warning( 'syntax-duplicate-doc-comment', token, {},
-                  'Repeated doc comment - previous doc is replaced' );
+    this.warning( 'syntax-duplicate-doc-comment', node.doc.location, {},
+                  'Doc comment is overwritten by another one below' );
   }
   node.doc = this.valueWithTokenLocation( parseDocComment( token.text ), token );
 }
@@ -488,10 +617,11 @@ function fragileAlias( ast, safe = false ) {
 }
 
 // Return AST for identifier token `token`.  Also check that identifer is not empty.
-function identAst( token, category ) {
+function identAst( token, category, noTokenTypeCheck = false ) {
   token.isIdentifier = category;
   let id = token.text;
-  if (token.type !== this.constructor.Identifier && !/^[a-zA-Z]+$/.test( id ))
+  if (!noTokenTypeCheck &&
+      token.type !== this.constructor.Identifier && !/^[a-zA-Z_]+$/.test( id ))
     id = '';
   if (token.text[0] === '!') {
     id = id.slice( 2, -1 ).replace( /]]/g, ']' );
@@ -518,11 +648,11 @@ function identAst( token, category ) {
   return { id, $delimited: true, location: this.tokenLocation( token ) };
 }
 
-function functionAst( token, xprToken ) {
+function functionAst( token, xpr ) { // only for special function TRIM and EXTRACT
   // TODO: XSN func cleanup
   const location = this.tokenLocation( token );
-  const args = xprToken
-    ? [ { op: { location, val: 'xpr' }, args: [], location: this.tokenLocation( xprToken ) } ]
+  const args = xpr
+    ? [ { op: { location, val: 'ixpr' }, args: [], location: this.tokenLocation( xpr ) } ]
     : [];
   return {
     op: { location, val: 'call' },
@@ -532,6 +662,21 @@ function functionAst( token, xprToken ) {
   };
 }
 
+function setLastAsXpr( args ) {
+  const pos = args.length - 1;
+  const last = args[pos];
+  if (!last || last?.op?.val === 'ixpr') // actuall an internal xpr, which is always flattened
+    return last;
+  const location = { ...last.location };
+  args[pos] = { op: { location, val: 'ixpr' }, args: [ last ], location };
+  return args[pos].args;
+}
+
+function xprToken() {
+  const token = this._input.LT(-1);
+  return { location: this.tokenLocation( token ), val: token.text, literal: 'token' }
+}
+
 function valuePathAst( ref ) {
   // TODO: XSN representation of functions is a bit strange - rework if methods
   // are introduced
@@ -630,21 +775,19 @@ function quotedLiteral( token, literal ) {
     literal = token.text.slice( 0, pos - 1 ).toLowerCase();
   const p = quotedLiteralPatterns[literal] || {};
 
-  // TODO: make tests available for CSN parser
-  if ((p.test_fn && !p.test_fn(val) || p.test_re && !p.test_re.test(val)) &&
-      !this.options.parseOnly)
-    this.error( null, location, p.test_msg ); // TODO: message id
+  if (p.test_fn && !p.test_fn(val) && !this.options.parseOnly)
+    this.warning( 'syntax-invalid-literal', location, { '#': p.test_variant } );
 
   if (p.unexpected_char) {
     const idx = val.search(p.unexpected_char);
     if (~idx) {
-      this.error( null, {     // TODO: message id
+      this.warning( 'syntax-invalid-literal', {
         file: location.file,
         line: location.line,
         endLine: location.line,
         col: atChar(idx),
         endCol: atChar( idx + (val[idx] === '\'' ? 2 : 1) ),
-      }, p.unexpected_msg );
+      }, { '#': p.unexpected_variant } );
     }
   }
   return {
@@ -659,10 +802,6 @@ function quotedLiteral( token, literal ) {
   }
 }
 
-function pathName( path, brokenName ) {
-  return (path && !path.broken) ? path.map( id => id.id ).join('.') : brokenName;
-}
-
 function pushIdent( path, ident, prefix ) {
   if (!ident) {
     path.broken = true;
@@ -691,61 +830,62 @@ function pushIdent( path, ident, prefix ) {
   }
 }
 
-// Add new definition to dictionary property `env` of node `parent` and return
-// that definition.  Also attach the following properties to the new definition:
-//  - `name`: argument `name`, which is used as key in the dictionary
-//  - `kind`: argument `kind` if that is truthy
-//  - `location`: argument `location` or the start location of source matched by
-//    current rule
-//  - properties in argument `props` which are no empty (undefined, null, {},
-//    []), ANTLR tokens are replaced by their locations
+// Add new definition `art` to dictionary property `env` of node `parent`.
+// Return `art`.
 //
-// Hack: if argument `location` is exactly `true`, do not set `location`
-// (except if part of `props`), but also include the empty properties of
-// `props`.
-function addDef( parent, env, kind, name, annos, props, location ) {
+// If argument `kind` is provided, set `art.kind` to that value.
+// If argument `name` is provided, set `art.name`:
+//  - if `name` is an array, the name consist of the ID of the last path item
+//    (for elements via columns, foreign keys, table aliases)
+//  - if `name` is an object, the name consist of the IDs of all path items
+//    (for main artifact definitions)
+function addDef( art, parent, env, kind, name ) {
   if (Array.isArray(name)) {
     // XSN TODO: clearly say: definitions have name.path, members have name.id
     const last = name.length && name[name.length - 1];
     if (last && last.id) {       // // A.B.C -> 'C'
-      name = {
+      art.name = {
         id: last.id, location: last.location, $inferred: 'as',
       };
     }
   }
-  else if (name && name.id == null) {
-    name.id = pathName( name.path ); // A.B.C -> 'A.B.C'
+  else if (name) {
+    art.name = name;
+    if (name.id == null)
+      name.id = pathName( name.path ); // A.B.C -> 'A.B.C'
+    // TODO: get rid of setting `id`, only use for named values in structs
   }
-  const art = this.assignProps( { name }, annos, props, location );
+
   if (kind)
     art.kind = kind;
-  if (!parent[env])      // TODO: dump with --test-mode, env !== 'artifacts'
-    parent[env] = env === 'args' ? Object.create(null) : this.createDict( { ...location } );
   if (!art.name || art.name.id == null) {
     // no id was parsed, but with error recovery: no further error
     // TODO: add to  parent[env]['']
     // which could be tested in name search (then no undefined-ref error)
+    // console.log( kind+': !!', art, parent, env, name )
     return art;
   }
-  else if (env === 'artifacts' || env === 'vocabularies') {
+  // console.log( kind+':', art, parent, env, name )
+
+  if (env === 'artifacts' || env === 'vocabularies') {
     dictAddArray( parent[env], art.name.id, art );
   }
   else if (kind || this.options.parseOnly) {
     dictAdd( parent[env], art.name.id, art );
   }
   else {
-    dictAdd( parent[env], art.name.id, art, ( name, loc ) => {
+    dictAdd( parent[env], art.name.id, art, ( duplicateName, loc ) => {
       // do not use function(), otherwise `this` is wrong:
       if (kind === 0) {
-        this.error( 'duplicate-argument', loc, { name },
+        this.error( 'duplicate-argument', loc, { name: duplicateName },
                     'Duplicate value for parameter $(NAME)' );
       }
       else if (kind === '') {
-        this.error( 'duplicate-excluding', loc, { name, keyword: 'excluding' },
+        this.error( 'duplicate-excluding', loc, { name: duplicateName, keyword: 'excluding' },
                     'Duplicate $(NAME) in the $(KEYWORD) clause' );
       }
       else {
-        this.error( 'duplicate-prop', loc, { name },
+        this.error( 'duplicate-prop', loc, { name: duplicateName },
                     'Duplicate value for structure property $(NAME)' );
       }
     } );
@@ -753,85 +893,78 @@ function addDef( parent, env, kind, name, annos, props, location ) {
   return art;
 }
 
-// Add new definition to array property `env` of node `parent` and return
-// that definition.  Also attach the following properties to the new definition:
-//  - `kind`: argument `kind` if that is truthy
-//  - `location`: argument `location` or the start location of source matched by
-//    current rule
-//  - properties in argument `props` which are no empty (undefined, null, {},
-//    []); ANTLR tokens are replaced by their locations
-//
-// Hack: if argument `location` is exactly `true`, do not set `location`
-// (except if part of `props`), but also include the empty properties of
-// `props`.
-function addItem( parent, env, kind, annos, props, location ) {
-  const art = this.assignProps( {}, annos, props, location );
-  if (kind)
-    art.kind = kind;
-  if (!env)
-    parent.push( art );
-  else if (!parent[env])
-    parent[env] = [ art ];
-  else
-    parent[env].push( art );
+// Add new definition `art` to array property `env` of node `parent`.
+// Also set `kind`.  Returns `art`.
+function addItem( art, parent, env, kind ) {
+  art.kind = kind;
+  parent[env].push( art );
   return art;
 }
-
 /**
- * For `annotate/extend E:elem.sub`, create the `elements` structure
- * that can be used by the core compiler to annotate/extend elements.
+ * Add `annotate/extend Main.Artifact:elem.sub` to `‹xsn›.extensions`:
+ * - the array item is an extend/annotate for `Main.Artifact`,
+ * - for each path item in `elem.sub`, we add an `elements` property containing
+ *   one extend/annotate for the corresponding element
+ * - The deepest extend/annotate is the object which is to be extended
  *
- * @param {string} kind Either `annotate` or `extend`
- * @param {object} artifact Main artifact that shall have `elements`.
- * @param {XSN.Path} elementPath Path as returned by `simplePath` token.
- * @param {object[]} annos  Existing annotations that shall be added to the _last_ path step.
- * @param {XSN.Location} artifactLocation Start location of the `annotate` statement.
- * @returns {object} Deepest element
- */
-function artifactForElementAnnotateOrExtend(kind, artifact, elementPath, annos, artifactLocation ) {
-  if (!Array.isArray(elementPath) || elementPath.broken || elementPath.length < 1)
-    return artifact;
-
-  for (const seg of elementPath.slice(0, -1)) {
-    artifact = this.addDef( artifact, 'elements', kind,
-      { path: [seg], location: seg.location }, null, {}, artifactLocation );
+ * @param {object} ext The object containing the location and annotations for the extension.
+ * @param {object} parent The parent containing the `extensions` property, i.e. the source.
+ * @param {string} kind Either `annotate` or `extend`.
+ * @param {object} artName The "name object" for `Main.Artifact`.
+ * @param {XSN.Path} elemPath Path as returned by `simplePath` rule.
+*/
+function addExtension( ext, parent, kind, artName, elemPath ) {
+  const { location } = ext;
+  if (!Array.isArray( elemPath ) || !elemPath.length || elemPath.broken) {
+    ext.name = artName;
+    this.addItem( ext, parent, 'extensions', kind );
+    return;
+  }
+  // Note: the element extensions share a common `location`, also with the
+  // extension of the main artifact; its end location will usually set later
+  parent = this.addItem( { name: artName, location }, parent, 'extensions', kind );
+
+  const last = elemPath[elemPath.length - 1];
+  for (const seg of elemPath) {
+    parent.elements = Object.create(null); // no dict location → no createDict()
+    parent = this.addDef( (seg === last ? ext : { location }),
+                          parent, 'elements', kind, seg );
   }
-  const last = elementPath[elementPath.length - 1];
-  artifact = this.addDef( artifact, 'elements', kind,
-    { path: [ last ], location: last.location }, annos, {}, artifactLocation );
-  return artifact;
 }
 
-/** Assign all non-empty (undefined, null, {}, []) properties in argument
- * `props` and argument `annos` as property `$annotations` to `target`
- * and return it.  Hack: if argument `annos` is exactly `true`, return
- * `Object.assign( target, props )`, for rule `namedValue`.  ANTLR tokens are
- * replaced by their locations.
- *
- * @param {any} target
- * @param {any[]|true} [annos=[]]
- * @param {any} [props]
- * @param {any} [location]
- */
-function assignProps( target, annos = [], props = null, location = null) {
-  if (annos === true)
-    return Object.assign( target, props );
-  target.location = location || this.startLocation( this._ctx.start );
-  // Object.assign without "empty" elements/properties and with mappings:
-  //  - token instanceof antlr4.CommonToken => location of token
-  for (const key in props) {
-    let val = props[key];
-    if (val instanceof antlr4.CommonToken)
-      val = this.valueWithTokenLocation( true, val);
-    // only copy properties which are not undefined, null, {} or []
-    if (val != null &&
-        (typeof val !== 'object' ||
-         (Array.isArray(val) ? val.length : Object.getOwnPropertyNames(val).length) ) )
-      target[key] = val;
-  }
-  if (annos)
-    target.$annotations = annos;
-  return target;
+// must be in action directly after having parsed '{' or '(`
+function createDict() {
+  const dict = Object.create(null);
+  dict[$location] = this.startLocation( this._input.LT(-1) );
+  return dict;
+}
+
+// must be in action directly after having parsed '[' or '(` or `{`
+function createArray() {
+  const array = [];
+  array[$location] = this.startLocation( this._input.LT(-1) );
+  return array;
+}
+
+// must be in action directly after having parsed '}' or ')`
+function finalizeDictOrArray( dict ) {
+  const loc = dict[$location];
+  if (!loc)
+    return;
+  const stop = this._input.LT(-1);
+  loc.endLine = stop.line;
+  loc.endCol = stop.stop - stop.start + stop.column + 2;
+}
+
+function createSource() {
+  return {
+    kind: 'source',
+    usings: [],
+    dependencies: [],
+    artifacts: Object.create(null),
+    // vocabularies: Object.create(null),
+    extensions: [],
+  };
 }
 
 // Create AST node for prefix operator `op` and arguments `args`
@@ -879,14 +1012,12 @@ function setOnce( target, prop, value, ...tokens ) {
   target[prop] = value;
 }
 
-function setMaxCardinality( art, token, max, inferred ) {
+function setMaxCardinality( art, token, max ) {
   const location = this.tokenLocation( token );
   if (!art.cardinality) {
     art.cardinality = { targetMax: Object.assign( { location }, max ), location };
-    if (inferred)
-      art.cardinality.$inferred = inferred;
   }
-  else if (!inferred) {
+  else {
     this.warning( 'syntax-repeated-cardinality', location, { keyword: token.text },
                   'The target cardinality has already been specified - ignored $(KEYWORD)' );
   }
@@ -905,6 +1036,9 @@ function handleComposition( cardinality, isComposition ) {
 }
 
 function associationInSelectItem( art ) {
+  if (!art.value) // e.g. `expand` without value (for new structures)
+    return;
+
   const isPath = art.value.path && art.value.path.length
   const isIdentifier = isPath && art.value.path.length === 1;
   if (isIdentifier) {
@@ -958,6 +1092,4 @@ function checkTypeFacet( art, argIdent ) {
   }
 }
 
-module.exports = {
-  genericAntlrParser: GenericAntlrParser,
-};
+module.exports = GenericAntlrParser;