@sap/cds-compiler 3.6.0 → 3.7.2

package/lib/transform/db/rewriteCalculatedElements.js

@@ -0,0 +1,559 @@
+'use strict';
+
+const { isBetaEnabled, setProp } = require('../../base/model');
+const { CompilerAssertion } = require('../../base/error');
+const {
+  forEachDefinition, applyTransformationsOnNonDictionary, applyTransformationsOnDictionary, implicitAs, cloneCsnNonDict, getUtils,
+} = require('../../model/csnUtils');
+const { getBranches } = require('./flattening');
+const { getColumnMap } = require('./views');
+const cloneCsnOptions = { hiddenPropertiesToClone: [ '_art', '_links', '$env', '$scope' ] };
+/**
+ * Rewrite usage of calculated Elements into the expression itself.
+ * Delete calculated elements in entities after processing so they don't materialize on the db.
+ *
+ * @param {CSN.Model} csn
+ * @param {CSN.Options} options
+ * @param {string} pathDelimiter
+ * @param {Function} error
+ */
+function rewriteCalculatedElementsInViews( csn, options, pathDelimiter, error ) {
+  if (!isBetaEnabled(options, 'calculatedElements'))
+    return;
+
+  const { inspectRef, effectiveType } = getUtils(csn, 'init-all');
+
+  const views = [];
+  const entities = [];
+
+  // In this first pass, we rewrite all the .value things in tables into their most basic form
+  forEachDefinition(csn, (artifact, artifactName) => {
+    if (artifact.kind === 'entity') {
+      if (artifact.query || artifact.projection) {
+        views.push({ artifact, artifactName });
+      }
+      else {
+        rewriteInEntity(artifact);
+        entities.push({ artifact, artifactName });
+      }
+    }
+  });
+
+  // Replace calculated elements in filters (if the root-association element is in an entity).
+  // Depends on the first pass!
+  entities.forEach(({ artifactName }) => {
+    applyTransformationsOnNonDictionary(csn.definitions, artifactName, {
+      where: (parent, prop) => {
+        applyTransformationsOnNonDictionary(parent, prop, {
+          ref: (_parent, _prop, ref, _path, root, index) => {
+            if (_parent._art && _parent._art.value) {
+              root[index] = _parent._art.value;
+              applyTransformationsOnNonDictionary(root, index, {
+                ref: (__parent, _, _ref) => {
+                  if (_ref[0] === '$self' || _ref[0] === '$projection')
+                    __parent.ref = _ref.slice(-1);
+                },
+              });
+            }
+          },
+        });
+      },
+    }, { drillRef: true }, [ 'definitions' ]);
+  });
+
+
+  // In this third pass, we process our views, generate .columns if needed and replace usage
+  // of calculated elements with their respective `.value`.
+  // This depends on the first pass!
+  views.forEach(({ artifact, artifactName }) => {
+    applyTransformationsOnNonDictionary(csn.definitions, artifactName, {
+      SELECT: (parent, prop, SELECT, path) => {
+        rewriteInView(SELECT, SELECT.elements || artifact.elements, path);
+      },
+      projection: (parent, prop, projection, path) => {
+        parent.SELECT = projection; // Fake as SELECT so our path below will match in the applyTransformations...
+        rewriteInView(parent.SELECT, artifact.elements, path);
+        delete parent.SELECT;
+      },
+    }, {}, [ 'definitions' ]);
+  });
+
+  // Last pass, turn .value in tables into a simple val: 1 so we don't need to rewrite/flatten properly - will kill them later
+  entities.forEach(({ artifact, artifactName }) => {
+    dummifyInEntity(artifact, [ 'definitions', artifactName ]);
+  });
+
+  /**
+   * Rewrite calculated-elements-columns in views/projections and replace them
+   * with their "root"-expression.
+   *
+   * As a first step, we ensure that all views/projections have a .columns (see {@link calculateColumns}) and that
+   * all calculated elements are addressed explicitly and not via a * (see {@link makeAllCalculatedElementsExplicitColumns}).
+   *
+   * Then, we check the `art` of each ref for a `.value` and rewrite accordingly.
+   * We need to ensure that the scope of the rewritten expressions is still correct!
+   * An `id` in the `.value` needs to point to the entity containing the element,
+   * not to some random view element named `id`. See {@link absolutifyPaths} for
+   * details on that.
+   *
+   * @param {CSN.QuerySelect} SELECT
+   * @param {CSN.Elements} elements
+   * @param {CSN.Path} path
+   */
+  function rewriteInView( SELECT, elements, path ) {
+    const containsExpandInline = hasExpandInline(SELECT);
+    if (!SELECT.columns) // needs to happen for all subqueries!
+      calculateColumns(elements, SELECT);
+    else
+      makeAllCalculatedElementsExplicitColumns(elements, SELECT, containsExpandInline);
+
+    const name = SELECT.from.args ? undefined : SELECT.from.as || implicitAs(SELECT.from.ref);
+
+    if (!containsExpandInline) {
+      applyTransformationsOnNonDictionary({ SELECT }, 'SELECT', {
+        ref: (parent, prop, ref, p, root) => {
+          const {
+            art, env, links, scope,
+          } = getRefInfo(parent, p);
+
+          if (art?.value) {
+            const alias = parent.as || implicitAs(parent.ref);
+            // TODO: What about other scopes? expand/inline?
+            const value = (scope !== 'ref-target') ? absolutifyPaths(env, art, ref, links, name).value : art.value;
+
+            // Is a shallow copy enough?
+            if (art.value.cast)
+              root[p[p.length - 1]] = { xpr: [ value ] };
+            else
+              root[p[p.length - 1]] = { ...value };
+
+            if (p[p.length - 2] === 'columns')
+              root[p[p.length - 1]].as = alias;
+            else
+              delete root[p[p.length - 1]].as;
+          }
+        },
+      }, {}, path);
+    }
+  }
+
+  /**
+   *
+   * @param {CSN.QuerySelect} SELECT
+   * @returns {boolean}
+   */
+  function hasExpandInline( SELECT ) {
+    if (!SELECT.columns)
+      return false;
+
+    for (let i = 0; i < SELECT.columns.length; i++) {
+      const column = SELECT.columns[i];
+      if (column.expand || column.inline)
+        return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Replace all nested .value things (in .xpr, in .ref) with their most-direct thing:
+   * - A ref to a non-calculated element
+   * - A .val
+   * - An expression containing the above
+   *
+   * @param {CSN.Artifact} artifact The artifact currently being processed
+   */
+  function rewriteInEntity( artifact ) {
+    applyTransformationsOnDictionary(artifact.elements, {
+      value: (parent, prop, value) => {
+        replaceValuesWithBaseValue(parent, value);
+      },
+    });
+  }
+
+  /**
+   * Iteratively replace all .values with the most-basic form:
+   * - a .val thing
+   * - a .ref to a non-value thing
+   *
+   * @param {object | Array} parent
+   * @param {object} value
+   */
+  function replaceValuesWithBaseValue( parent, value ) {
+    if (value.val && parent.value === value)
+      return;
+
+    const stack = [ { parent, value } ];
+    while (stack.length > 0) {
+      const current = stack.pop();
+
+      if (current.value.xpr) {
+        applyTransformationsOnNonDictionary(current.value, 'xpr', {
+          ref: (p, prop, ref, path, root ) => {
+            stack.push({
+              parent: root,
+              value: p,
+              isInXpr: true,
+              refBase: current.refBase,
+              linksBase: current.linksBase,
+            });
+          },
+        });
+      }
+      else if (current.value.ref && current.value._art?.value) {
+        const linksBase = current.value._links;
+        const refBase = current.value.ref;
+        const parentIndex = Array.isArray(current.parent) ? current.parent.indexOf(current.value) : -1;
+
+        replaceInRef(current.parent, current.value._art.value, current.isInXpr, refBase, linksBase, parentIndex);
+
+        stack.push(Object.assign(current, {
+          value: parentIndex > -1 ? current.parent[parentIndex] : current.parent.value,
+          refBase,
+          linksBase,
+        }));
+      }
+      // No need for cloning here, as we don't rewrite this further and will later on kill all the stuff anyway
+      else if (current.value.val) { // this is the base case - or a ref to a non-calculated element
+        if (current.isInXpr) { // inside of expressions we directly need the val
+          current.parent.val = current.value.val;
+          delete current.parent.value;
+        }
+        else { // outside of expressions, i.e. as normal elements, we need it in a .value wrapper
+          current.parent.value = current.value;
+        }
+      }
+    }
+  }
+
+  /**
+   * A value referenced via a ref is replaced here
+   * - kill the ref
+   * - explicitly mention the value
+   *
+   * We either "trick" it into the correct place in an .xpr or we simply overwrite the existing .ref
+   *
+   * @param {object} parent
+   * @param {object} newValue
+   * @param {boolean} isInXpr
+   * @param {Array} refBase
+   * @param {Array} linksBase
+   * @param {number} indexInParent
+   */
+  function replaceInRef( parent, newValue, isInXpr, refBase, linksBase, indexInParent ) {
+    delete parent.ref;
+    const clone = {
+      value: cloneCsnNonDict({ value: newValue }, cloneCsnOptions).value,
+    };
+    const refPrefix = refBase.slice(0, -1);
+    const linksPrefix = linksBase.slice(0, -1);
+    if (newValue.xpr) {
+      // We need to adapt the scope of all refs in the new .xpr, as it might have been at a different "root"
+      applyTransformationsOnNonDictionary(clone, 'value', {
+        ref: (p, prop, ref) => {
+          if (ref[0] !== '$self' && ref[0] !== '$projection') {
+            p.ref = [ ...refPrefix, ...ref ];
+            if (p._links)
+              p._links = [ ...linksPrefix, ...p._links ]; // TODO: Make non-enum, increment idx
+          }
+        },
+      }, {
+        // Do not rewrite refs inside of an association-where; avoids endless loop
+        skipStandard: { where: true },
+      });
+      if (indexInParent > -1) // a .xpr in a .xpr
+        parent[indexInParent] = clone.value;
+      else
+        parent.value = clone.value;
+    }
+    else {
+      if (indexInParent > -1) // a .ref in a .xpr
+        parent[indexInParent] = clone.value;
+      else
+        parent.value = clone.value;
+      if (clone.value.ref && clone.value.ref[0] !== '$self' && clone.value.ref[0] !== '$projection' ) {
+        clone.value.ref = [ ...refPrefix, ...clone.value.ref ];
+        clone.value._links = [ ...linksPrefix, ...clone.value._links ]; // TODO: Make non-enum, increment idx
+      }
+    }
+  }
+
+  /**
+   * For a `view V as select from E;` or a `entity P as projection on E;` calculate and
+   * attach the .columns if they contain a calculated element so we can rewrite them in
+   * the later steps.
+   *
+   * @param {CSN.Elements} elements Artifact elements
+   * @param {object} carrier The thing that will "carry" the columns - .SELECT or .projection
+   */
+  function calculateColumns( elements, carrier ) {
+    carrier.columns = [ '*' ];
+    makeAllCalculatedElementsExplicitColumns(elements, carrier, false);
+    if (carrier.columns.length === 1 && carrier.columns[0] === '*')
+      delete carrier.columns;
+  }
+
+  /**
+   *
+   * @param {CSN.QuerySelect} SELECT
+   * @returns {object}
+   */
+  function getDirectlyAdressableElements( SELECT ) {
+    const { from } = SELECT;
+    if (from.ref) {
+      return from._art.elements;
+    }
+    else if (from.SELECT) {
+      return from.SELECT.elements;
+    }
+    else if (from.SET) {
+      return from.SET.elements || getDirectlyAdressableElements(from.SET.args[0].SELECT);
+    }
+    else if (from.args) {
+      const mergedElements = Object.create(null);
+      for (const arg of from.args) {
+        if (arg.ref) {
+          for (const elementName in arg._art.elements)
+            mergedElements[elementName] = arg._art.elements[elementName];
+        }
+        else if (arg.SELECT) { // TODO: UNION
+          for (const elementName in arg.SELECT.elements)
+            mergedElements[elementName] = arg.SELECT.elements[elementName];
+        }
+        else {
+          throw new CompilerAssertion(`Unhandled arg type: ${JSON.stringify(arg, null, 2)}`);
+        }
+      }
+      return mergedElements;
+    }
+    throw new CompilerAssertion(`Unhandled query type:  ${JSON.stringify(SELECT, null, 2)}`);
+  }
+
+  /**
+   * Ensure that all elements of the query that are calculated elements have an explicit column that we can rewrite.
+   * If a field originally comes in via the *, then we need to add an explicit column for it.
+   *
+   * @param {CSN.Elements} elements
+   * @param {CSN.QuerySelect} SELECT
+   * @param {boolean} containsExpandInline
+   */
+  function makeAllCalculatedElementsExplicitColumns( elements, SELECT, containsExpandInline ) {
+    const root = getDirectlyAdressableElements(SELECT);
+    const columnMap = getColumnMap( { SELECT });
+    const hasStar = SELECT.columns.includes('*');
+    const unfoldingMap = {};
+    let starContainsCalculated = false;
+    let containsCalculated = false;
+    for (const name in elements) {
+      const originalRef = columnMap[name] && columnMap[name].ref || [ name ];
+
+      if (columnMap[name] || hasStar) {
+        let element;
+        if (columnMap[name]?.expand || columnMap[name]?.inline)
+          element = elements[name]; // only the direct thing in .elements has the .excluding respected properly!
+        else
+          element = columnMap[name]?._art || columnMap[name]?._element || root[name] || elements[name];
+        const branches = getBranches(element, name, effectiveType, pathDelimiter); // TODO: is our elements[name] really the root[name]?
+        if (hasCalculatedLeaf(branches)) {
+          containsCalculated = true;
+          const columns = [];
+          for (const branchName in branches) {
+            if (columnMap[branchName]) // Existing column - don't overwrite, we need $env!
+              columns.push(columnMap[branchName]);
+            else // TODO: Hm, will we have a $env in the leaf of the thing then?
+              columns.push({ ref: [ ...originalRef, ...branches[branchName].ref.slice(1) ], as: branchName });
+          }
+          if (columnMap[name]) {
+            unfoldingMap[name] = [ false, [ ...columns ] ];
+          }
+          else if (hasStar) { // Via * - just append
+            starContainsCalculated = true;
+            unfoldingMap[name] = [ true, [ ...columns ] ];
+          }
+        }
+        else if (!columnMap[name] && hasStar) { // Via * - just append
+          unfoldingMap[name] = [ true, [ { ref: [ name ] } ] ];
+        }
+        else { // just a random column - keep
+          unfoldingMap[name] = [ false, [ columnMap[name] ] ];
+        }
+      }
+    }
+
+    if (containsExpandInline && containsCalculated) {
+      error('query-unsupported-calc', SELECT.$path, { '#': 'std' });
+    }
+    else if (containsCalculated) {
+      const newColumns = [];
+      if (hasStar && !starContainsCalculated)
+        newColumns.push('*');
+      for (const name in elements) {
+        const [ isViaStar, columns ] = unfoldingMap[name];
+        if (isViaStar && starContainsCalculated || !isViaStar)
+          newColumns.push(...columns);
+      }
+
+
+      SELECT.columns = newColumns;
+    }
+  }
+
+  /**
+   *
+   * @param {object} branches
+   * @returns {boolean}
+   */
+  function hasCalculatedLeaf( branches ) {
+    for (const branchName in branches) {
+      const branch = branches[branchName].steps;
+      const leaf = branch[branch.length - 1];
+      if (hasValue(leaf))
+        return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * A leaf can reference a column which in turn references a real element - that might have a .value.
+   * Find such cases.
+   *
+   * @param {object} baseLeaf Leaf to start at
+   * @returns {boolean}
+   */
+  function hasValue( baseLeaf ) {
+    const visited = new WeakSet();
+    const stack = [ baseLeaf ];
+    while (stack.length > 0) {
+      const leaf = stack.pop();
+      if (!visited.has(leaf)) { // Don't re-process things
+        if (leaf.value)
+          return true;
+        else if (leaf._art)
+          stack.push(leaf._art);
+        else if (leaf['@Core.Computed'] && leaf._column && leaf._column !== baseLeaf)
+          stack.push(leaf._column);
+      }
+
+      visited.add(leaf);
+    }
+
+    return false;
+  }
+
+
+  /**
+   * In order to just replace them in views, our calculated elements need to reference absolute things, i.e. have a table alias in front!
+   *
+   * @param {string | object} env
+   * @param {object} art
+   * @param {Array} artRef
+   * @param {Array} artLinks
+   * @param {string|undefined} name
+   * @todo this is probably very wonky and will break with some view hierarchy stuff etc!
+   * @returns {object}
+   */
+  function absolutifyPaths( env, art, artRef, artLinks, name ) {
+    const clone = { value: cloneCsnNonDict(art.value, cloneCsnOptions) };
+    applyTransformationsOnNonDictionary(clone, 'value', {
+      ref: (parent, prop, ref) => {
+        const artifactName = typeof env === 'string' ? env : name;
+        if (parent._links) {
+          if (parent._links[0].art.kind !== 'entity') {
+            if (artLinks[0].art.kind === 'entity' || artifactName === undefined) {
+              parent.ref = [ ...artRef.slice(0, -1), ...ref ];
+              setProp(parent, '_links', [ ...artLinks.slice(0, -1), ...parent._links ]); // TODO: increment idx
+            }
+            else {
+              parent.ref = [ artifactName, ...artRef.slice(0, -1), ...ref ];
+              setProp(parent, '_links', [ { idx: 0 }, ...artLinks.slice(0, -1), ...parent._links ]); // TODO: increment idx
+            }
+          }
+          else if (parent.$scope === '$self') {
+            if (artifactName !== undefined)
+              parent.ref[0] = artifactName;
+            else
+              parent.ref = parent.ref.slice(-1);
+          }
+        }
+      },
+    }, {
+      skipStandard: { where: true }, // Do not rewrite refs inside of an association-where
+    });
+
+    return clone;
+  }
+
+  /**
+   * Get the ref-info
+   * - either the cached _art etc.
+   * - or calculate using inspectRef
+   *
+   * @param {object} parent
+   * @param {CSN.Path} path
+   * @returns {object}
+   */
+  function getRefInfo( parent, path ) {
+    if (parent._art) {
+      return {
+        art: parent._art,
+        env: parent.$env,
+        links: parent._links,
+        scope: parent.$scope,
+      };
+    }
+
+    return inspectRef(path);
+  }
+}
+
+/**
+ *
+ * @param {CSN.Model} csn
+ * @param {CSN.Options} options
+ */
+function processCalculatedElementsInEntities( csn, options ) {
+  if (!isBetaEnabled(options, 'calculatedElements'))
+    return;
+  forEachDefinition(csn, (artifact, artifactName) => {
+    if (artifact.kind === 'entity' && !(artifact.query || artifact.projection))
+      killInEntity(artifact, [ 'definitions', artifactName ]);
+  });
+}
+
+
+/**
+ * In an entity, remove all instances of calculated elements.
+ *
+ * @param {CSN.Artifact} artifact
+ * @param {CSN.Path} path
+ * @todo calculated elements that "live" on the database?
+ * @todo error when artifact is empty afterwards? Probably better as a CSN check!
+ */
+function killInEntity( artifact, path ) {
+  applyTransformationsOnDictionary(artifact.elements, {
+    value: (parent, prop, value, p, root) => {
+      delete root[p[p.length - 1]];
+    },
+  }, {}, path);
+}
+
+/**
+ * In an entity, turn all instances of calculated elements into an = 1. This way,
+ * we don't have to rewrite any scope there and can kill them after A2J, see {@link processCalculatedElementsInEntities}.
+ *
+ * @param {CSN.Artifact} artifact
+ * @param {CSN.Path} path
+ */
+function dummifyInEntity( artifact, path ) {
+  applyTransformationsOnDictionary(artifact.elements, {
+    value: (parent) => {
+      parent.value = { val: 1 };
+    },
+  }, {}, path);
+}
+
+module.exports = {
+  rewriteCalculatedElementsInViews,
+  processCalculatedElementsInEntities,
+};

package/lib/transform/db/transformExists.js

@@ -3,7 +3,6 @@
 const { forAllQueries, forEachDefinition, walkCsnPath } = require('../../model/csnUtils');
 const { setProp } = require('../../base/model');
 const { getRealName } = require('../../render/utils/common');
-const { csnRefs } = require('../../model/csnRefs');
 const { ModelError } = require('../../base/error');
 
 /**
@@ -47,9 +46,11 @@ const { ModelError } = require('../../base/error');
  * @param {CSN.Model} csn
  * @param {CSN.Options} options
  * @param {Function} error
+ * @param {Function} inspectRef
+ * @param {Function} initDefinition
+ * @param {Function} dropDefinitionCache
  */
-function handleExists( csn, options, error ) {
-  let { inspectRef } = csnRefs(csn);
+function handleExists( csn, options, error, inspectRef, initDefinition, dropDefinitionCache ) {
   const generatedExists = new WeakMap();
   forEachDefinition(csn, (artifact, artifactName) => {
     if (artifact.projection) // do the same hack we do for the other stuff...
@@ -78,15 +79,19 @@ function handleExists( csn, options, error ) {
 
           while (toProcess.length > 0) {
             const [ queryPath, exprPath ] = toProcess.pop();
+            // Re-init caches for this artifact
+            dropDefinitionCache(artifact);
+            initDefinition(artifact);
             // leftovers can happen with nested exists - we then need to drill down into the created SELECT
             // to check for further exists
             const { result, leftovers } = processExists(queryPath, exprPath);
             walkCsnPath(csn, exprPath.slice(0, -1))[exprPath[exprPath.length - 1]] = result;
-            if (leftovers.length > 0)
-              inspectRef = csnRefs(csn).inspectRef; // Refresh caches - we need to resolve stuff in the newly created subquery
             leftovers.reverse();
             toProcess.push(...leftovers); // any leftovers - schedule for further processing
           }
+          // Make sure we leave csnRefs usable
+          dropDefinitionCache(artifact);
+          initDefinition(artifact);
         }
       }, [ 'definitions', artifactName, 'query' ]);
     }
@@ -719,6 +724,10 @@ function handleExists( csn, options, error ) {
         where: [],
       },
     };
+
+    setProp(subselect.SELECT.from, '_art', csn.definitions[target]);
+    setProp(subselect.SELECT.from, '_links', [ { idx: 0, art: csn.definitions[target] } ]);
+
     // Because the generated things don't have _links, _art etc. set
     // We could also make getParent more robust to calculate the links JIT if they are missing
     generatedExists.set(subselect, true);
@@ -764,7 +773,7 @@ function handleExists( csn, options, error ) {
    */
   function remapExistingWhere( target, where ) {
     return where.map((part) => {
-      if (part.ref) {
+      if (part.ref && part.$scope !== '$magic') {
         part.ref = [ target, ...part.ref ];
         return part;
       }