@sap/cds-compiler 2.4.4 → 2.10.2

package/lib/json/to-csn.js

@@ -12,14 +12,21 @@
 'use strict';
 
 const { locationString } = require('../base/messages');
-const { isDeprecatedEnabled } = require('../base/model');
+const { isBetaEnabled, isDeprecatedEnabled } = require('../base/model');
 
 const compilerVersion = require('../../package.json').version;
 const creator = `CDS Compiler v${ compilerVersion }`;
 const csnVersion = '2.0';
 
+const normalizedKind = {
+  param: 'param',
+  action: 'action',
+  function: 'action',
+};
+
 /** @type {boolean|string} */
 let gensrcFlavor = true;       // good enough here...
+let universalCsn = false;
 let strictMode = false;        // whether to dump with unknown properties (in standard)
 let parensAsStrings = false;
 let projectionAsQuery = false;
@@ -31,6 +38,7 @@ let dictionaryPrototype = null;
 const transformers = {
   // early and modifiers (without null / not null) ---------------------------
   kind,
+  _outer: ( _, csn, node ) => addOrigin( csn, node ),
   id: n => n,                   // in path item
   doc: value,
   '@': value,
@@ -50,7 +58,7 @@ const transformers = {
   all: ignore,                  // XSN TODO use quantifier
   // type properties (without 'elements') ------------------------------------
   localized: value,
-  type: t => artifactRef( t, !t.$extra ),
+  type,
   length: value,
   precision: value,
   scale: value,
@@ -67,11 +75,12 @@ const transformers = {
   columns,
   expand: ignore,               // do not list for select items as elements
   inline: ignore,               // do not list for select items as elements
-  excludingDict: renameTo( 'excluding', Object.keys ),
+  excludingDict,
   groupBy: arrayOf( expression ),
   where: condition,             // also pathItem after 'cardinality' before 'args'
   having: condition,
   args,                        // also pathItem after 'where', before 'on'/'orderBy'
+  suffix: node => [].concat( ...node.suffix.map( xprArg ) ),
   orderBy: arrayOf( orderBy ), // TODO XSN: make `sort` and `nulls` sibling properties
   sort: value,
   nulls: value,
@@ -80,14 +89,14 @@ const transformers = {
   offset: expression,
   on: onCondition,
   // definitions, extensions, members ----------------------------------------
-  returns: standard,            // storing the return type of actions
+  returns: definition,          // storing the return type of actions
   notNull: value,
   default: expression,
   // targetElement: ignore,     // special display of foreign key, renameTo: select
   value: enumValue,             // do not list for select items as elements
   query,
   elements,
-  actions: nonEmptyDict,        // TODO: just normal dictionary
+  actions,                      // TODO: just normal dictionary
   // special: top-level, cardinality -----------------------------------------
   sources,
   definitions: sortedDict,
@@ -128,7 +137,7 @@ const transformers = {
 // which should appear at that place in order.
 const csnPropertyNames = {
   virtual: [ 'abstract' ],      // abstract is compiler v1 CSN property
-  kind: [ 'annotate', 'extend' ],
+  kind: [ 'annotate', 'extend', '$origin' ],
   op: [ 'join', 'func', 'xpr' ],    // TODO: 'func','xpr' into 'quantifier'?  TODO: 'global'(scope)?
   quantifier: [
     'some', 'any', 'distinct',  // 'all' explicitly listed
@@ -148,7 +157,7 @@ const csnPropertyNames = {
   name: [ 'as', 'cast' ],
   location: [ '$env', '$location' ], // --enrich-csn
   expectedKind: [
-    '_type', '_targetAspect', '_target', '_includes', '_links', '_art', '_scope',
+    '_origin', '_type', '_targetAspect', '_target', '_includes', '_links', '_art', '_scope',
   ],                            // --enrich-csn
 };
 
@@ -183,6 +192,13 @@ const operators = {
   notLike: ternary( [ 'not', 'like' ], [ 'escape' ] ),
   when: exprs => [ 'when', ...exprs[0], 'then', ...exprs[1] ],
   case: exprs => [ 'case' ].concat( ...exprs, [ 'end' ] ),
+  over: exprs => [ 'over', { xpr: [].concat( ...exprs ) } ],
+  orderBy: exprs => [
+    'order', 'by', ...exprs[0].concat( ...exprs.slice(1).map( e => [ ',', ...e ] ) ),
+  ],
+  partitionBy: exprs => [
+    'partition', 'by', ...exprs[0].concat( ...exprs.slice(1).map( e => [ ',', ...e ] ) ),
+  ],
   // xpr: (exprs) => [].concat( ...exprs ), see below - handled extra
 };
 
@@ -194,7 +210,7 @@ const csnDirectValues = [ 'val' ]; // + all starting with '@' - TODO: still rele
 /**
  * Sort property names of CSN according to sequence which is also used by the compactModel function
  * Only returns enumerable properties, except for certain hidden properties
- * if requested (cloneOptions != false): $location, $env, elements.
+ * if requested (cloneOptions != false): $location, elements.
  *
  * If cloneOptions is false or if either cloneOptions.testMode or cloneOptions.testSortCsn
  * are set, definitions are also sorted.
@@ -224,24 +240,36 @@ function sortCsn( csn, cloneOptions = false ) {
       r[n] = sortCsn(val, cloneOptions);
   }
   if (cloneOptions && typeof csn === 'object') {
-    if (csn.$sources && !r.$sources)
-      setHidden(r, '$sources', csn.$sources);
-    if (csn.$location && !r.$location)
-      setHidden(r, '$location', csn.$location);
-    if (csn.$env)
-      setHidden(r, '$env', csn.$env);
-    if (csn.$path) // used in generic reference flattener
-      setHidden(r, '$path', csn.$path);
-    if (csn.$paths) // used in generic reference flattener
-      setHidden(r, '$paths', csn.$paths);
-    if (csn.elements && !r.elements) // non-enumerable 'elements'
-      setHidden(r, 'elements', csnDictionary( csn.elements, false, cloneOptions ) );
-    if (csn.$tableConstraints && !r.$tableConstraints)
-      setHidden(r, '$tableConstraints', csn.$tableConstraints);
+    if ({}.hasOwnProperty.call( csn, '$sources' ) && !r.$sources)
+      setHidden( r, '$sources', csn.$sources );
+    if ({}.hasOwnProperty.call( csn, '$location' ) && !r.$location)
+      setHidden( r, '$location', csn.$location );
+    if ({}.hasOwnProperty.call( csn, '$path' )) // used in generic reference flattener
+      setHidden( r, '$path', csn.$path );
+    if ({}.hasOwnProperty.call( csn, '$paths' )) // used in generic reference flattener
+      setHidden( r, '$paths', csn.$paths );
+    if (hasNonEnumerable( csn, 'elements' ) && !r.elements) // non-enumerable 'elements'
+      setHidden( r, 'elements', csnDictionary( csn.elements, false, cloneOptions ) );
+    if (hasNonEnumerable( csn, '$tableConstraints' ) && !r.$tableConstraints)
+      setHidden( r, '$tableConstraints', csn.$tableConstraints );
   }
   return r;
 }
 
+/**
+ * Check wether the given object has non enumerable property.
+ * Ensure that we don't take it from the prototype, only "directly" - we accidentally
+ * cloned elements with a cds.linked input otherwise.
+ *
+ * @param {object} object
+ * @param {string} property
+ * @returns
+ */
+function hasNonEnumerable(object, property) {
+  return {}.hasOwnProperty.call( object, property ) &&
+    !{}.propertyIsEnumerable.call( object, property );
+}
+
 /**
  * @param {object} csn
  * @param {boolean} sort
@@ -283,6 +311,7 @@ function compactModel( model, options = model.options || {} ) {
       csn.requires = using;
   }
   // 'namespace' for complete model is 'namespace' of first source
+  // (not a really useful property at all, avoids XSN inspection by Umbrella)
   for (const first in srcDict) {
     const { namespace } = srcDict[first];
     if (namespace && namespace.path)
@@ -352,10 +381,12 @@ function usings( srcDict ) {
  * @param {object} csn
  * @param {object} model
  */
+
+
 function extensions( node, csn, model ) {
   if (model.kind && model.kind !== 'source')
     return undefined;
-  const exts = node.map( standard );
+  const exts = node.map( definition );
 
   // builtins are non-enumerable for smaller display
   for (const name of Object.getOwnPropertyNames( model.definitions || {} ).sort()) {
@@ -370,17 +401,21 @@ function extensions( node, csn, model ) {
     }
     else if (gensrcFlavor) {
       // From definitions (without redefinitions) with potential inferred elements:
-      if (!Array.isArray(art) && art.elements &&
-          (art.query || art.includes || art.$inferred)) {
-        const annos = art.$inferred && annotations( art, true );
-        const elems = inferred( art.elements, art.$inferred );
-        /** @type {object} */
-        const annotate = Object.assign( { annotate: name }, annos );
-        if (Object.keys( elems ).length)
-          annotate.elements = elems;
-        if (Object.keys( annotate ).length > 1)
-          exts.push( annotate );
+      const annotate = { annotate: name };
+      if (art.$inferred)
+        Object.assign( annotate, annotationsAndDocComment( art, true ) );
+      if (art.$expand === 'annotate') {
+        if (art.actions)
+          attachAnnotations( annotate, 'actions', art.actions, art.$inferred );
+        else if (art.params)
+          attachAnnotations( annotate, 'params', art.params, art.$inferred );
+        const obj = art.returns || art;
+        const elems = (obj.items || obj).elements; // no targetAspect here
+        if (elems)
+          attachAnnotations( annotate, 'elements', elems, art.$inferred, art.returns );
       }
+      if (Object.keys( annotate ).length > 1)
+        exts.push( annotate );
     }
   }
 
@@ -388,12 +423,64 @@ function extensions( node, csn, model ) {
     (a, b) => (a.annotate || a.extend).localeCompare( b.annotate || b.extend )
   );
 
+  /*
+  function attachElementAnnos( annotate, art ) {
+    while (art.items)
+      art = art.items;
+    if (art.elements) {
+      const elems = inferred( art.elements, art.$inferred );
+      if (Object.keys( elems ).length)
+        annotate.elements = elems;
+    }
+  }
+
+  function attachParamAnnos( annotate, art ) {
+    const inferredParent = art.$inferred;
+    if (art.params) {
+      const ext = Object.create( dictionaryPrototype );
+      for (const name in art.params) {
+        const par = art.params[name];
+        if (!inferredParent && !par.$inferred && par.$expand !== 'annotate')
+          continue;
+        const render = annotationsAndDocComment( par, true );
+        const subElems = par.$expand !== 'origin' && (par.items || par).elements;
+        if (subElems) {
+          const sub = inferred( subElems, par.$inferred );
+          if (Object.keys( sub ).length)
+            render.elements = sub;
+        }
+        if (Object.keys(render).length)
+          ext[name] = render;
+      }
+      if (obj.keys( ext ))
+        annotate.params = ext;
+    }
+    if (art.returns) {
+      const par = art.returns;
+      if (!inferredParent && !par.$inferred && par.$expand !== 'annotate')
+        return;
+      const render = annotationsAndDocComment( par, true );
+      const subElems = par.$expand !== 'origin' && (par.items || par).elements;
+      if (subElems) {
+        const sub = inferred( subElems, par.$inferred );
+        if (Object.keys( sub ).length)
+          render.elements = sub;
+      }
+      if (Object.keys(render).length)
+        const sub = inferred( subElems, par.$inferred );
+      if (Object.keys( sub ).length)
+        render.elements = sub;
+    }
+  }
+  return ext;
+  */
+
   // extract namespace/builtin annotations
   function extractAnnotationsToExtension( art ) {
     const name = art.name.absolute;
     // 'true' because annotations on namespaces and builtins can only
     // happen through extensions.
-    const annos = annotations( art, true );
+    const annos = annotationsAndDocComment( art, true );
     const annotate = Object.assign( { annotate: name }, annos );
     if (Object.keys( annotate ).length > 1) {
       const loc = locationForAnnotationExtension();
@@ -433,8 +520,13 @@ function i18n( i18nNode ) {
 }
 
 function sources( srcDict, csn ) {
+  const names = Object.keys( srcDict );
+  const $sources = names.length && srcDict[names[0]].$sources;
+  if ($sources) {
+    setHidden( csn, '$sources', $sources );
+    return undefined;
+  }
   // TODO: sort according to some layering order, see #6368
-  const names = Object.keys( srcDict);
   setHidden( csn, '$sources', (!strictMode) ? names : names.map( relativeName ) );
   return undefined;
 
@@ -444,17 +536,29 @@ function sources( srcDict, csn ) {
   }
 }
 
-function inferred( elems, inferredParent ) {
-  const ext = Object.create( dictionaryPrototype );
-  for (const name in elems) {
-    const elem = elems[name];
-    if (Array.isArray(elem) || !inferredParent && !elem.$inferred)
-      continue;
-    const csn = annotations( elem, true );
-    if (Object.keys(csn).length)
-      ext[name] = csn;
+function attachAnnotations( annotate, prop, dict, inferred, returns = false ) {
+  const annoDict = Object.create( dictionaryPrototype );
+  for (const name in dict) {
+    const elem = dict[name];
+    const inf = inferred || elem.$inferred; // is probably always inferred if parent was
+    const sub = (inf) ? annotationsAndDocComment( elem, true ) : {};
+    if (elem.$expand === 'annotate') {
+      if (elem.params)
+        attachAnnotations( sub, 'params', elem.params, inf );
+      const obj = elem.returns || elem;
+      const elems = (obj.items || obj.targetAspect || obj).elements;
+      if (elems)
+        attachAnnotations( sub, 'elements', elems, inf, elem.returns );
+    }
+    if (Object.keys( sub ).length)
+      annoDict[name] = sub;
+  }
+  if (Object.keys( annoDict ).length) {
+    if (returns)
+      annotate.returns = { elements: annoDict };
+    else
+      annotate[prop] = annoDict;
   }
-  return ext;
 }
 
 function standard( node ) {
@@ -493,54 +597,79 @@ function set( prop, csn, node ) {
 }
 
 function targetAspect( val, csn, node ) {
+  const ta = (val.elements)
+    ? addLocation( val.location, standard( val ) )
+    : artifactRef( val, true );
   if (!gensrcFlavor || node.target && !node.target.$inferred)
-    return (val.elements) ? standard( val ) : artifactRef( val, true );
+    return ta;
   // For compatibility, put aspect in 'target' with parse.cdl and csn flavor 'gensrc'
-  csn.target = (val.elements) ? standard( val ) : artifactRef( val, true );
+  csn.target = ta;
   return undefined;
 }
 
 function target( val, _csn, node ) {
   if (gensrcFlavor && node._origin && node._origin.$inferred === 'REDIRECTED')
     val = node._origin.target;
-  if (!val.elements)
+  if (val.elements)
+    return standard( val );       // elements in target (parse-cdl)
+  if (!universalCsn || node.on)
     return artifactRef( val, true );
-  return standard( val );       // elements in target (parse-cdl)
+  const tref = artifactRef( val, true );
+  const proto = node.type && !node.type.$inferred ? node.type._artifact : node._origin;
+  return (proto && proto.target && artifactRef( proto.target, true ) === tref)
+    ? undefined
+    : tref;
 }
 
 function items( obj, csn, node ) {
-  if (!keepElements( node ) && node.type && node.kind !== 'type')
-    // no 'elements' with SELECT or inferred elements with gensrc;
-    // hidden 'elements' will be set in query()
+  if (!keepElements( node ))
     return undefined;
-  return standard( obj );
+  return standard( obj );   // no 'elements' with inferred elements with gensrc
 }
 
 function elements( dict, csn, node ) {
-  if (csn.from ||
+  if (node.from ||              // do not directly show query elements here
       gensrcFlavor && (node.query || node.type) ||
-      node.type && node.kind !== 'type' && !keepElements( node ))
+      !keepElements( node ))
     // no 'elements' with SELECT or inferred elements with gensrc;
-    // hidden 'elements' will be set in query()
+    // hidden or visible 'elements' will be set in query()
     return undefined;
   return insertOrderDict( dict );
 }
 
-// We do not optimize away elements which are potentially adapted during their
-// way from the original structure type definition to the current usage
+function enumerableQueryElements( select ) {
+  if (!universalCsn || select === select._main._leadingQuery)
+    return false;
+  if (select.orderBy || select.$orderBy)
+    return true;
+  const alias = select._parent;
+  return alias.query && (alias.query._leadingQuery || alias.query) === select;
+}
+
+// Should we render the elements?  (and items?)
 function keepElements( node ) {
+  if (universalCsn)
+    // $expand = null/undefined: not elements not via expansion
+    // $expand = 'target'/'annotate': with redirections / individual annotations
+    return node.$expand !== 'origin';
+  if (!node.type || node.kind === 'type')
+    return true;
+  // even if expanded elements have no new target or direct annotation,
+  // they might have got one via propagation – any new target/annos during their
+  // way from the original structure type definition to the current usage
   while (node) {
     if (node.$expand !== 'origin')
       return true;
     node = node._origin;
   }
-  return false;
+  // all in _origin chain only have expanded elements with 'origin':
+  return false;                 // no need to render elements
 }
 
 // for gensrcFlavor and namespace/builtin annotation extraction:
 // return annotations from definition (annotated==false)
 // or annotations (annotated==true)
-function annotations( node, annotated ) {
+function annotationsAndDocComment( node, annotated ) {
   const csn = {};
   const transformer = transformers['@'];
   const keys = Object.keys( node ).filter( a => a.charAt(0) === '@' ).sort();
@@ -557,6 +686,8 @@ function annotations( node, annotated ) {
         csn[prop] = sub;
     }
   }
+  if (node.doc)
+    csn.doc = transformers.doc(node.doc);
   return csn;
 }
 
@@ -618,17 +749,17 @@ function sortedDict( dict ) {
   return dictionary( dict, keys );
 }
 
-function nonEmptyDict( dict ) {
+function actions( dict ) {
   const keys = Object.keys( dict );
   return (keys.length)
-    ? dictionary( dict, keys )
+    ? dictionary( dict, keys, 'actions' )
     : undefined;
 }
 
-function dictionary( dict, keys ) {
+function dictionary( dict, keys, prop ) {
   const csn = Object.create( dictionaryPrototype );
   for (const name of keys) {
-    const def = definition( dict[name] );
+    const def = definition( dict[name], null, null, prop );
     if (def !== undefined)
       csn[name] = def;
   }
@@ -636,6 +767,8 @@ function dictionary( dict, keys ) {
 }
 
 function foreignKeys( dict, csn, node ) {
+  if (universalCsn && !target( node.target, csn, node ))
+    return;
   if (gensrcFlavor && node._origin && node._origin.$inferred === 'REDIRECTED')
     dict = node._origin.foreignKeys;
   const keys = [];
@@ -649,7 +782,7 @@ function foreignKeys( dict, csn, node ) {
   csn.keys = keys;
 }
 
-function definition( art ) {
+function definition( art, _csn, _node, prop ) {
   if (!art || typeof art !== 'object')
     return undefined;           // TODO: complain with strict
   // Do not include namespace definitions or inferred construct (in gensrc):
@@ -661,22 +794,79 @@ function definition( art ) {
     addLocation( art.targetElement.location, key );
     return extra( key, art );
   }
-  return standard( art );
+  const c = standard( art );
+  // The XSN of actions in extensions do not contain a returns yet - TODO?
+  const elems = c.elements;
+  if (elems && (prop === 'actions' || art.$syntax === 'returns')) {
+    delete c.elements;
+    c.returns = { elements: elems };
+  }
+  return c;
+}
+
+function addOrigin( csn, xsn ) {
+  if (!universalCsn)
+    return csn;
+  if (xsn._from) {
+    csn.$origin = originRef( xsn._from[0]._origin );
+  }
+  else if (xsn.includes && xsn.includes.length > 1) {
+    csn.$origin = { $origin: originRef( xsn.includes[0]._artifact ) };
+  }
+  else if (xsn._origin && !hasExplicitProp( xsn.type ) && xsn._origin.kind !== 'builtin') {
+    let origin = xsn._origin;
+    while (origin._parent && origin._parent.$expand === 'origin')
+      origin = origin._origin || origin.type._artifact;
+    csn.$origin = originRef( origin );
+  }
+  return csn;
+}
+
+function hasExplicitProp( ref ) {
+  return ref && !ref.$inferred;
+}
+
+function originRef( art ) {
+  const r = [];
+  // do not use name.element, as we allow `.`s in name
+  let main = art;
+  while (main._main && main.kind !== 'select') {
+    const nkind = normalizedKind[main.kind];
+    if (main.name.id || !r.length) // { param: "" } only for return, not elements inside
+      r.push( nkind ? { [nkind]: main.name.id } : main.name.id );
+    main = main._parent;
+  }
+  if (main._main)    // well, an element of an query in FROM
+    return definition( art );   // use $origin: {}
+  // for sub query in FROM in sub query in FROM, we could condense the info
+  r.push( art.name.absolute );
+  r.reverse();
+  return r;
 }
 
 function kind( k, csn, node ) {
-  if (!node._main && [ 'annotate', 'extend' ].includes( k )) {
+  if (k === 'annotate' || k === 'extend') {
     // We just use `name.absolute` because it is very likely a "constructed"
     // extensions.  The CSN parser must produce name.path like for other refs.
-    csn[k] = node.name.absolute || artifactRef( node.name, true );
-    return undefined;
+    if (!node._main)
+      csn[k] = node.name.absolute || artifactRef( node.name, true );
+    else if (k === 'extend')
+      csn.kind = k;
   }
-  if ([
-    'element', 'key', 'param', 'enum', 'annotate', 'select', '$join',
-    '$tableAlias', 'annotation', 'mixin',
-  ].includes(k))
+  else {
+    if (![
+      'element', 'key', 'param', 'enum', 'select', '$join',
+      '$tableAlias', 'annotation', 'mixin',
+    ].includes(k))
+      csn.kind = k;
+    addOrigin( csn, node );
+  }
+}
+
+function type( node, csn, xsn ) {
+  if (universalCsn && node.$inferred && xsn._origin)
     return undefined;
-  return k;
+  return artifactRef( node, !node.$extra );
 }
 
 function artifactRef( node, terse ) {
@@ -778,10 +968,12 @@ function args( node ) {
   return dict;
 }
 
-// "Short" value form, e.g. for annotation assignments
 function value( node ) {
+// "Short" value form, e.g. for annotation assignments
   if (!node)
     return true;                // `@aBool` short for `@aBool: true`
+  if (universalCsn && node.$inferred === 'prop') // via propagator.js
+    return undefined;
   if (node.$inferred && gensrcFlavor)
     return undefined;
   if (node.path) {
@@ -792,6 +984,8 @@ function value( node ) {
     return extra( { '#': node.sym.id }, node );
   if (node.literal === 'array')
     return node.val.map( value );
+  if (node.literal === 'token' && node.val === '...')
+    return extra( { '...': true } );
   if (node.literal !== 'struct')
     // no val (undefined) as true only for annotation values (and struct elem values)
     return node.name && !('val' in node) || node.val;
@@ -821,23 +1015,8 @@ function onCondition( cond, csn, node ) {
 
 function condition( node ) {
   const expr = expression( node );
-  return !expr.cast && expr.xpr || [ expr ];
-}
-
-// TODO: quoted magic names like $now should be complained about in the compiler
-
-function pathRef( path ) {
-  const ref = { ref: path.map( pathItem ) };
-  const nav = path[0]._navigation;
-  if (nav && nav.kind !== '$self' && nav.kind !== 'element' && nav.name.select != null) {
-    setHidden( ref, '$env', (nav.kind === '$navElement')
-      ? nav.name.alias
-      : nav.name.select );
-  }
-  else if ( path[0]._artifact && path[0]._artifact.query ) {
-    setHidden( ref, '$env', true );
-  }
-  return ref;
+  // we do not set a hidden $parens on array - we could still do it if requested
+  return !expr.cast && !expr.func && expr.xpr || [ expr ];
 }
 
 function expression( node, dollarExtra ) {
@@ -852,8 +1031,8 @@ function expression( node, dollarExtra ) {
     return { ref: [ node.param.val ], param: true }; // CDL rule for runtimes
   }
   if (node.path) {
-    // TODO: global
-    return extra( pathRef( node.path ), dollarExtraNode );
+    // we would need to consider node.global here if we introduce that
+    return extra( { ref: node.path.map( pathItem ) }, dollarExtraNode );
   }
   if (node.literal) {
     if (typeof node.val === node.literal || node.val === null)
@@ -877,37 +1056,32 @@ function expression( node, dollarExtra ) {
           arg0.xpr.unshift( quantifier.val );
       }
     }
+    if (node.suffix)
+      call.xpr = [].concat( ...node.suffix.map( xprArg ) );
     return extra( call, dollarExtraNode );
   }
   if (node.query)
-    return query( node.query );
+    return query( node.query, null, null, null, 1 );
   if (!node.op)                 // parse error
     return { xpr: [] };
   else if (node.op.val === 'xpr')
     // do not use xpr() for xpr, as it would flatten inner xpr's
-    return extra({ xpr: node.args.map( expression ) }, dollarExtraNode );
+    return extra({ xpr: node.args.map( expression ) }, dollarExtraNode, 1 );
   else if (node.op.val === 'cast')
     return cast( expression( node.args[0] ), dollarExtraNode );
   // from here on: CDL input (no $extra possible - but $parens)
   else if (node.op.val !== ',')
-    return extra( { xpr: xpr( node ) }, dollarExtraNode );
+    return extra( { xpr: xpr( node ) }, dollarExtraNode, (dollarExtra === 'sub-xpr' ? 1 : 0) );
   return (parensAsStrings)
     ? { xpr: [ '(', ...xpr( node ), ')' ] }
-    : extra( { list: node.args.map( expression ) } );
+    // the inner parens belong to the tuple construct, i.e. won't count as parens
+    : extra( { list: node.args.map( expression ) }, dollarExtraNode, 0 );
 }
 
 function xpr( node ) {
   // if (!node.op) console.log(node)
   const op = operators[node.op.val] || node.op.val.split(' ');
-  const exprs = node.args.map( ( sub ) => {
-    const expr = expression( sub );
-    // return !sub.$parens && !expr.cast && expr.xpr || [ expr ]; if parensAsStrings is gone
-    if (expr.cast || !expr.xpr || sub.$parens && !parensAsStrings)
-      return [ expr ];
-    else if (sub.$parens && sub.op.val !== ',')
-      return [ '(', ...expr.xpr, ')' ];
-    return expr.xpr;
-  } );
+  const exprs = node.args.map( xprArg );
   if (op instanceof Function)
     return op( exprs );
   if (node.quantifier)
@@ -917,6 +1091,27 @@ function xpr( node ) {
   return exprs[0].concat( ...exprs.slice(1).map( a => [ ...op, ...a ] ) );
 }
 
+function xprArg( sub ) {
+  const realXpr = sub.op && sub.op.val === 'xpr';
+  const expr = expression( sub, 'sub-xpr' );
+  // `sort`/`nulls` will be attached to arguments of orderBy
+  // which might be either `path`s or `xpr`s
+  const sortAndNulls = [];
+  if (sub.sort)
+    sortAndNulls.push( sub.sort.val );
+  if (sub.nulls)
+    sortAndNulls.push( ...[ 'nulls', sub.nulls.val ] );
+  // return !sub.$parens && !expr.cast && !expr.func && expr.xpr || [ expr ];
+  // if parensAsStrings is gone
+  if (realXpr || expr.cast || expr.func || !expr.xpr || sub.$parens && !parensAsStrings)
+    return [ expr, ...sortAndNulls ];
+  else if (sub.$parens && sub.op.val !== ',')
+    return [ '(', ...expr.xpr, ')' ];
+
+  expr.xpr.push( ...sortAndNulls );
+  return expr.xpr;
+}
+
 function ternary( op1, op2 ) {
   return function ternaryOp( exprs ) {
     return (exprs[2])
@@ -940,16 +1135,15 @@ function binaryRightParen( op ) {
   };
 }
 
-function query( node, csn, xsn ) {
-  while (Array.isArray(node)) // in parentheses -> remove
-    node = node[0];
+function query( node, csn, xsn, _prop, expectedParens = 0 ) {
   if (node.op.val === 'SELECT') {
     if (xsn && xsn.query === node && xsn.$syntax === 'projection' &&
        node.from && node.from.path && !projectionAsQuery) {
       csn.projection = standard( node );
       return undefined;
     }
-    const select = { SELECT: standard( node ) };
+    const select = { SELECT: extra( standard( node ), node, expectedParens ) };
+    // one paren pair is not put into XSN - TODO: change that?
     const elems = node.elements;
     if (elems && node._main && node !== node._main._leadingQuery && gensrcFlavor !== true) {
       // Set hidden 'elements' for csnRefs.js.  In select-item subqueries,
@@ -958,7 +1152,10 @@ function query( node, csn, xsn ) {
       const gensrcSaved = gensrcFlavor;
       try {
         gensrcFlavor = false;
-        setHidden( select.SELECT, 'elements', insertOrderDict( elems ) );
+        if (enumerableQueryElements( node ))
+          select.SELECT.elements = insertOrderDict( elems );
+        else
+          setHidden( select.SELECT, 'elements', insertOrderDict( elems ) );
       }
       finally {
         gensrcFlavor = gensrcSaved;
@@ -995,11 +1192,13 @@ function columns( xsnColumns, csn, xsn ) {
   return csnColumns;
 }
 
+function excludingDict( xsnDict, csn, xsn ) {
+  if (xsn.kind !== 'element')
+    csn.excluding = Object.keys( xsnDict );
+}
+
 function from( node ) {
-  while (Array.isArray(node)) // TODO: old-style parentheses - keep tmp for A2J
-    node = node[0];
-  // TODO: CSN: FROM ((SELECT...)) as -> also add 'subquery' op? - Together
-  // with []-elimination in FROM... -> normal standard()
+  // TODO: can we use the normal standard(), at least with JOIN?
   if (node.join) {
     const join = { join: node.join.val };
     set( 'cardinality', join, node );
@@ -1008,7 +1207,7 @@ function from( node ) {
     return extra( join, node );
   }
   else if (node.query) {
-    return addExplicitAs( query( node.query ), node.name ); // $extra inside SELECT/SET
+    return addExplicitAs( query( node.query, null, null, null, 1 ), node.name );
   }
   else if (!node._artifact || node._artifact._main) { // CQL or follow assoc
     return extra( addExplicitAs( artifactRef( node, false ), node.name ), node );
@@ -1024,7 +1223,7 @@ function addElementAsColumn( elem, cols ) {
   if (elem.$inferred === '*')
     return;
   // only list annotations here which are provided directly with definition
-  const col = (gensrcFlavor) ? annotations( elem, false ) : {};
+  const col = (gensrcFlavor) ? annotationsAndDocComment( elem, false ) : {};
   // with `client` flavor, assignments are available at the element
   const gensrcSaved = gensrcFlavor;
 
@@ -1034,33 +1233,29 @@ function addElementAsColumn( elem, cols ) {
     set( 'key', col, elem );
     const expr = expression( elem.value, true );
     Object.assign( col, (expr.cast ? { xpr: [ expr ] } : expr) );
+    gensrcFlavor = gensrcSaved; // for not having annotations in inline etc
     if (elem.expand)
       col.expand = columns( elem.expand );
     if (elem.inline)
       col.inline = columns( elem.inline );
+    gensrcFlavor = gensrcFlavor || 'column';
+    if (elem.excludingDict)
+      col.excluding = Object.keys( elem.excludingDict );
     // yes, the AS comes after the EXPAND
     addExplicitAs( col, elem.name, neqPath( elem.value ) );
-    // $env and elements (sub queries) in expr are hidden (not set via Object.assign):
-    if (!expr.cast) {
-      if (expr.$env)
-        setHidden( col, '$env', expr.$env );
-      if (expr.elements)
-        setHidden( col, 'elements', expr.elements );
-    }
+    // elements of sub queries (in expr) are hidden (not set via Object.assign):
+    if (!expr.cast && expr.elements)
+      setHidden( col, 'elements', expr.elements );
     if (elem.type && !elem.type.$inferred || elem.target && !elem.target.$inferred)
       cast( col, elem );
   }
   finally {
     gensrcFlavor = gensrcSaved;
   }
-  // FIXME: Currently toHana requires that an '_ignore' property on the elem is
-  // also visible on the column.  Don't ignore virtual columns, let the
-  // renderer decide how to render that column.
-  if (!elem.virtual && elem._ignore)
-    col._ignore = true;
-
   if (elem.value && !elem.$inferred) {
     const parens = elem.value.$parens;
+    if (parens)
+      setHidden( col, '$parens', parens.length );
     addLocation( (parens ? parens[parens.length - 1] : elem.value.location), col );
   }
   cols.push( extra( col, elem ) );
@@ -1075,12 +1270,13 @@ function orderBy( node ) {
   return extra( expr, node );   // extra properties after sort/nulls
 }
 
-function extra( csn, node ) {
+function extra( csn, node, expectedParens = 0 ) {
   if (node) {
     if (node.$extra)
       Object.assign( csn, node.$extra );
-    if (node.$parens)
-      setHidden( csn, '$parens', node.$parens.length );
+    const parens = (node.$parens ? node.$parens.length : 0);
+    if (parens !== expectedParens)
+      setHidden( csn, '$parens', parens );
   }
   return csn;
 }
@@ -1142,6 +1338,9 @@ function compactExpr( e ) {     // TODO: options
 function initModuleVars( options = { csnFlavor: 'gensrc' } ) {
   gensrcFlavor = options.parseCdl || options.csnFlavor === 'gensrc' ||
                  options.toCsn && options.toCsn.flavor === 'gensrc';
+  universalCsn = (options.csnFlavor === 'universal' ||
+                  options.toCsn && options.toCsn.flavor === 'universal' ) &&
+    isBetaEnabled( options, 'enableUniversalCsn' ) && !options.parseCdl;
   strictMode = options.testMode;
   const proto = options.dictionaryPrototype;
   // eslint-disable-next-line no-nested-ternary
@@ -1154,7 +1353,7 @@ function initModuleVars( options = { csnFlavor: 'gensrc' } ) {
 }
 
 module.exports = {
-  cloneCsnDictionary: csnDictionary,
+  cloneCsnDictionary: (csn, options) => csnDictionary(csn, false, options),
   compactModel,
   compactQuery,
   compactExpr,