@cap-js/db-service 1.10.3 → 1.11.0

package/CHANGELOG.md

@@ -4,6 +4,21 @@
 - The format is based on [Keep a Changelog](http://keepachangelog.com/).
 - This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [1.11.0](https://github.com/cap-js/cds-dbs/compare/db-service-v1.10.3...db-service-v1.11.0) (2024-07-08)
+
+
+### Added
+
+* **search:** enable deep search with path expressions ([#590](https://github.com/cap-js/cds-dbs/issues/590)) ([e9e9461](https://github.com/cap-js/cds-dbs/commit/e9e9461362b3a521c06075e32fb1405c8ee4dee6))
+
+### Changed
+
+* `search` interprets only first search term instead of raising an error ([#707](https://github.com/cap-js/cds-dbs/issues/707)) ([0b9108c](https://github.com/cap-js/cds-dbs/commit/0b9108c11a61b18704e36f93fbd654e0942bf40a))
+
+### Fixed
+
+* optimize foreign key access for expand with aggregations ([#734](https://github.com/cap-js/cds-dbs/issues/734)) ([77b7978](https://github.com/cap-js/cds-dbs/commit/77b79788931c9c45f156d54a4b2ec76ba9ba629a))
+
 ## [1.10.3](https://github.com/cap-js/cds-dbs/compare/db-service-v1.10.2...db-service-v1.10.3) (2024-07-05)
 
 

package/lib/SQLService.js

@@ -116,6 +116,8 @@ class SQLService extends DatabaseService {
    * @type {Handler}
    */
   async onSELECT({ query, data }) {
+    // REVISIT: for custom joins, infer is called twice, which is bad
+    //          --> make cds.infer properly work with custom joins and remove this
     if (!query.target) {
       try { this.infer(query) } catch { /**/ }
     }

package/lib/cql-functions.js

@@ -22,6 +22,8 @@ const StandardFunctions = {
    */
   search: function (ref, arg) {
     if (!('val' in arg)) throw new Error(`Only single value arguments are allowed for $search`)
+    // only apply first search term, rest is ignored
+    arg.val = arg.__proto__.val = arg.val.split(' ')[0].replace(/"/g, '')
     const refs = ref.list || [ref],
       { toString } = ref
     return '(' + refs.map(ref2 => this.contains(this.tolower(toString(ref2)), this.tolower(arg))).join(' or ') + ')'

package/lib/cqn4sql.js

@@ -1,9 +1,9 @@
 'use strict'
 
 const cds = require('@sap/cds')
-const { computeColumnsToBeSearched } = require('./search')
 
 const infer = require('./infer')
+const { computeColumnsToBeSearched } = require('./search')
 
 /**
  * For operators of <eqOps>, this is replaced by comparing all leaf elements with null, combined with and.
@@ -44,8 +44,26 @@ const { pseudos } = require('./infer/pseudos')
  * @returns {object} transformedQuery the transformed query
  */
 function cqn4sql(originalQuery, model) {
-  const inferred = infer(originalQuery, model)
-  if (originalQuery.SELECT?.from.args && !originalQuery.joinTree) return inferred
+  let inferred = typeof originalQuery === 'string' ? cds.parse.cql(originalQuery) : cds.ql.clone(originalQuery)
+  const hasCustomJoins =
+    originalQuery.SELECT?.from.args && (!originalQuery.joinTree || originalQuery.joinTree.isInitial)
+
+  if (!hasCustomJoins && inferred.SELECT?.search) {
+    // we need an instance of query because the elements of the query are needed for the calculation of the search columns
+    if (!inferred.SELECT.elements) Object.setPrototypeOf(inferred, Object.getPrototypeOf(SELECT()))
+    const searchTerm = getSearchTerm(inferred.SELECT.search, inferred)
+    if (searchTerm) {
+      // Search target can be a navigation, in that case use _target to get the correct entity
+      const { where, having } = transformSearch(searchTerm)
+      if (where) inferred.SELECT.where = where
+      else if (having) inferred.SELECT.having = having
+    }
+  }
+  inferred = infer(inferred, model)
+  // if the query has custom joins we don't want to transform it
+  // TODO: move all the way to the top of this function once cds.infer supports joins as well
+  //       we need to infer the query even if no transformation will happen because cds.infer can't calculate the target
+  if (hasCustomJoins) return originalQuery
 
   let transformedQuery = cds.ql.clone(inferred)
   const kind = inferred.kind || Object.keys(inferred)[0]
@@ -111,7 +129,7 @@ function cqn4sql(originalQuery, model) {
 
         // calculate the primary keys of the target entity, there is always exactly
         // one query source for UPDATE / DELETE
-        const queryTarget = Object.values(originalQuery.sources)[0].definition
+        const queryTarget = Object.values(inferred.sources)[0].definition
         const keys = Object.values(queryTarget.elements).filter(e => e.key === true)
         const primaryKey = { list: [] }
         keys.forEach(k => {
@@ -155,7 +173,7 @@ function cqn4sql(originalQuery, model) {
     if (columns) {
       transformedQuery.SELECT.columns = getTransformedColumns(columns)
     } else {
-      transformedQuery.SELECT.columns = getColumnsForWildcard(originalQuery.SELECT?.excluding)
+      transformedQuery.SELECT.columns = getColumnsForWildcard(inferred.SELECT?.excluding)
     }
 
     // Like the WHERE clause, aliases from the SELECT list are not accessible for `group by`/`having` (in most DB's)
@@ -178,13 +196,6 @@ function cqn4sql(originalQuery, model) {
         transformedQuery.SELECT.orderBy = transformedOrderBy
       }
     }
-
-    if (inferred.SELECT.search) {
-      // Search target can be a navigation, in that case use _target to get the correct entity
-      const { where, having } = transformSearch(inferred.SELECT.search, transformedFrom) || {}
-      if (where) transformedQuery.SELECT.where = where
-      else if (having) transformedQuery.SELECT.having = having
-    }
     return transformedQuery
   }
 
@@ -206,46 +217,29 @@ function cqn4sql(originalQuery, model) {
    * Transforms a search expression into a WHERE or HAVING clause for a SELECT operation, depending on the context of the query.
    * The function decides whether to use a WHERE or HAVING clause based on the presence of aggregated columns in the search criteria.
    *
-   * @param {object} search - The search expression to be applied to the searchable columns within the query source.
+   * @param {object} searchTerm - The search expression to be applied to the searchable columns within the query source.
    * @param {object} from - The FROM clause of the CQN statement.
    *
-   * @returns {(Object|Array|null)} - The function returns an object representing the WHERE or HAVING clause of the query:
-   *     - If the target of the query contains searchable elements, an array representing the WHERE or HAVING clause is returned.
-   *       This includes appending to an existing clause with an AND condition or creating a new clause solely with the 'contains' clause.
-   *     - If the SELECT query does not initially contain a WHERE or HAVING clause, the returned object solely consists of the 'contains' clause.
-   *     - If the target entity of the query does not contain searchable elements, the function returns null.
+   * @returns {Object} - The function returns an object representing the WHERE or HAVING clause of the query.
    *
    * Note: The WHERE clause is used for filtering individual rows before any aggregation occurs.
    * The HAVING clause is utilized for conditions on aggregated data, applied after grouping operations.
    */
-  function transformSearch(search, from) {
-    const entity = getDefinition(from.$refLinks[0].definition.target) || from.$refLinks[0].definition
-    // pass transformedQuery because we may need to search in the columns directly
-    // in case of aggregation
-    const searchIn = computeColumnsToBeSearched(transformedQuery, entity, from.as)
-    if (searchIn.length > 0) {
-      const xpr = search
-      const contains = {
-        func: 'search',
-        args: [
-          searchIn.length > 1 ? { list: searchIn } : { ...searchIn[0] },
-          xpr.length === 1 && 'val' in xpr[0] ? xpr[0] : { xpr },
-        ],
-      }
-
-      // if the query is grouped and the queries columns contain an aggregate function,
-      // we must put the search term into the `having` clause, as the search expression
-      // is defined on the aggregated result, not on the individual rows
-      let prop = 'where'
-
-      if (inferred.SELECT.groupBy && searchIn.some(c => c.func || c.xpr)) prop = 'having'
-      if (transformedQuery.SELECT[prop]) {
-        return { [prop]: [asXpr(transformedQuery.SELECT.where), 'and', contains] }
-      } else {
-        return { [prop]: [contains] }
-      }
+  function transformSearch(searchTerm) {
+    let prop = 'where'
+
+    // if the query is grouped and the queries columns contain an aggregate function,
+    // we must put the search term into the `having` clause, as the search expression
+    // is defined on the aggregated result, not on the individual rows
+    const usesAggregation =
+      inferred.SELECT.groupBy &&
+      (searchTerm.args[0].func || searchTerm.args[0].xpr || searchTerm.args[0].list?.some(c => c.func || c.xpr))
+
+    if (usesAggregation) prop = 'having'
+    if (inferred.SELECT[prop]) {
+      return { [prop]: [asXpr(inferred.SELECT.where), 'and', searchTerm] }
     } else {
-      return null
+      return { [prop]: [searchTerm] }
     }
   }
 
@@ -484,7 +478,7 @@ function cqn4sql(originalQuery, model) {
       if (replaceWith === -1) transformedColumns.push(transformedColumn)
       else transformedColumns.splice(replaceWith, 1, transformedColumn)
 
-      setElementOnColumns(transformedColumn, originalQuery.elements[col.as])
+      setElementOnColumns(transformedColumn, inferred.elements[col.as])
     }
 
     function getTransformedColumn(col) {
@@ -812,7 +806,8 @@ function cqn4sql(originalQuery, model) {
 
     // we need to respect the aliases of the outer query, so the columnAlias might not be suitable
     // as table alias for the correlated subquery
-    const uniqueSubqueryAlias = getNextAvailableTableAlias(columnAlias, originalQuery.outerQueries)
+    const uniqueSubqueryAlias = getNextAvailableTableAlias(columnAlias, inferred.outerQueries)
+
     // `SELECT from Authors {  books.genre as genreOfBooks { name } } becomes `SELECT from Books:genre as genreOfBooks`
     const from = { ref: subqueryFromRef, as: uniqueSubqueryAlias }
     const subqueryBase = Object.fromEntries(
@@ -830,7 +825,10 @@ function cqn4sql(originalQuery, model) {
     }
     const expanded = transformSubquery(subquery)
     const correlated = _correlate({ ...expanded, as: columnAlias }, outerAlias)
-    Object.defineProperty(correlated, 'elements', { value: subquery.elements, writable: true })
+    Object.defineProperty(correlated, 'elements', {
+      value: expanded.elements,
+      writable: true,
+    })
     return correlated
 
     function _correlate(subq, outer) {
@@ -881,7 +879,7 @@ function cqn4sql(originalQuery, model) {
 
       // to be attached to dummy query
       const elements = {}
-      const expandedColumns = column.expand.map(expand => {
+      const expandedColumns = column.expand.flatMap(expand => {
         const fullRef = [...baseRef, ...expand.ref]
 
         if (expand.expand) {
@@ -901,13 +899,11 @@ function cqn4sql(originalQuery, model) {
         if (expand.as) {
           columnCopy.as = expand.as
         }
-        if (columnCopy.isJoinRelevant) {
-          const tableAlias = getQuerySourceName(columnCopy)
-          const name = calculateElementName(columnCopy)
-          columnCopy.ref = [tableAlias, name]
-        }
-        elements[expand.as || expand.ref.map(idOnly).join('_')] = columnCopy.$refLinks.at(-1).definition
-        return columnCopy
+        const res = getFlatColumnsFor(columnCopy, { tableAlias: getQuerySourceName(columnCopy) })
+        res.forEach(c => {
+          elements[c.as || c.ref.at(-1)] = c.element
+        })
+        return res
       })
 
       const SELECT = {
@@ -938,15 +934,6 @@ function cqn4sql(originalQuery, model) {
       if (isCalculatedOnRead(col.$refLinks?.[col.$refLinks.length - 1].definition)) {
         const calcElement = resolveCalculatedElement(col, true)
         res.push(calcElement)
-      } else if (col.isJoinRelevant) {
-        const tableAlias = getQuerySourceName(col)
-        const name = calculateElementName(col)
-        const transformedColumn = {
-          ref: [tableAlias, name],
-        }
-        if (col.sort) transformedColumn.sort = col.sort
-        if (col.nulls) transformedColumn.nulls = col.nulls
-        res.push(transformedColumn)
       } else if (pseudos.elements[col.ref?.[0]]) {
         res.push({ ...col })
       } else if (col.ref) {
@@ -1004,8 +991,14 @@ function cqn4sql(originalQuery, model) {
          */
         if (inOrderBy && flatColumns.length > 1)
           throw new Error(`"${getFullName(leaf)}" can't be used in order by as it expands to multiple fields`)
-        if (col.nulls) flatColumns[0].nulls = col.nulls
-        if (col.sort) flatColumns[0].sort = col.sort
+        flatColumns.forEach(fc => {
+          if (col.nulls)
+            fc.nulls = col.nulls
+          if (col.sort)
+            fc.sort = col.sort
+          if (fc.as)
+            delete fc.as
+        })
         res.push(...flatColumns)
       } else {
         let transformedColumn
@@ -1051,7 +1044,7 @@ function cqn4sql(originalQuery, model) {
       const last = q.SELECT.from.ref.at(-1)
       const uniqueSubqueryAlias = inferred.joinTree.addNextAvailableTableAlias(
         getLastStringSegment(last.id || last),
-        originalQuery.outerQueries,
+        inferred.outerQueries,
       )
       Object.defineProperty(q.SELECT.from, 'uniqueSubqueryAlias', { value: uniqueSubqueryAlias })
     }
@@ -1659,7 +1652,7 @@ function cqn4sql(originalQuery, model) {
         transformedFrom.as = from.as
       } else {
         // select from anonymous query, use artificial alias
-        transformedFrom.as = Object.keys(originalQuery.sources)[0]
+        transformedFrom.as = Object.keys(inferred.sources)[0]
       }
       return { transformedFrom, transformedWhere: existingWhere }
     } else {
@@ -1702,7 +1695,7 @@ function cqn4sql(originalQuery, model) {
            * --> This is an artificial query, which will later be correlated
            * with the main query alias. see @function expandColumn()
            */
-          if (!(originalQuery.SELECT?.expand === true)) {
+          if (!(inferred.SELECT?.expand === true)) {
             as = getNextAvailableTableAlias(as)
           }
           nextStepLink.alias = as
@@ -2162,6 +2155,35 @@ function cqn4sql(originalQuery, model) {
     return getDefinition(assoc.target) || null
   }
 
+  /**
+   * For a given search expression return a function "search" which holds the search expression
+   * as well as the searchable columns as arguments.
+   *
+   * @param {object} search - The search expression which shall be applied to the searchable columns on the query source.
+   * @param {object} query - The FROM clause of the CQN statement.
+   *
+   * @returns {(Object|null)} returns either:
+   * - a function with two arguments: The first one being the list of searchable columns, the second argument holds the search expression.
+   * - or null, if no searchable columns are found in neither in `@cds.search` nor in the target entity itself.
+   */
+  function getSearchTerm(search, query) {
+    const entity = query.SELECT.from.SELECT ? query.SELECT.from : query.target
+    const searchIn = computeColumnsToBeSearched(inferred, entity)
+    if (searchIn.length > 0) {
+      const xpr = search
+      const searchFunc = {
+        func: 'search',
+        args: [
+          searchIn.length > 1 ? { list: searchIn } : { ...searchIn[0] },
+          xpr.length === 1 && 'val' in xpr[0] ? xpr[0] : { xpr },
+        ],
+      }
+      return searchFunc
+    } else {
+      return null
+    }
+  }
+
   /**
    * Calculates the name of the source which can be used to address the given node.
    *
@@ -2207,11 +2229,11 @@ function cqn4sql(originalQuery, model) {
          * - but differs from the explicit alias, assigned by cqn4sql (i.e. <subquery>.from.uniqueSubqueryAlias)
          */
         if (
-          originalQuery.SELECT?.from.uniqueSubqueryAlias &&
-          !originalQuery.SELECT?.from.as &&
+          inferred.SELECT?.from.uniqueSubqueryAlias &&
+          !inferred.SELECT?.from.as &&
           firstStep === getLastStringSegment(transformedQuery.SELECT.from.ref[0])
         ) {
-          return originalQuery.SELECT?.from.uniqueSubqueryAlias
+          return inferred.SELECT?.from.uniqueSubqueryAlias
         }
         return node.ref[0]
       }

package/lib/infer/index.js

@@ -24,7 +24,7 @@ for (const each in cdsTypes) cdsTypes[`cds.${each}`] = cdsTypes[each]
  */
 function infer(originalQuery, model) {
   if (!model) throw new Error('Please specify a model')
-  const inferred = typeof originalQuery === 'string' ? cds.parse.cql(originalQuery) : cds.ql.clone(originalQuery)
+  const inferred = originalQuery
 
   // REVISIT: The more edge use cases we support, thes less optimized are we for the 90+% use cases
   // e.g. there's a lot of overhead for infer( SELECT.from(Books) )

package/lib/search.js

@@ -39,7 +39,6 @@ const getColumns = (
 
   for (const each in elements) {
     const element = elements[each]
-    if (element.isAssociation) continue
     if (filterVirtual && element.virtual) continue
     if (removeIgnore && element['@cds.api.ignore']) continue
     if (filterDraft && each in DRAFT_COLUMNS_UNION) continue
@@ -58,7 +57,7 @@ const _isColumnCalculated = (query, columnName) => {
 
 const _getSearchableColumns = entity => {
   const columnsOptions = { removeIgnore: true, filterVirtual: true }
-  const columns = getColumns(entity, columnsOptions)
+  const columns = entity.SELECT?.columns || getColumns(entity, columnsOptions)
   const cdsSearchTerm = '@cds.search'
   const cdsSearchKeys = []
   const cdsSearchColumnMap = new Map()
@@ -68,30 +67,35 @@ const _getSearchableColumns = entity => {
   }
 
   let atLeastOneColumnIsSearchable = false
+  const deepSearchCandidates = []
 
   // build a map of columns annotated with the @cds.search annotation
   for (const key of cdsSearchKeys) {
     const columnName = key.split(cdsSearchTerm + '.').pop()
-
-    // REVISIT: for now, exclude search using path expression, as deep search is not currently
-    // supported
-    if (columnName.includes('.')) {
-      continue
-    }
-
     const annotationKey = `${cdsSearchTerm}.${columnName}`
     const annotationValue = entity[annotationKey]
     if (annotationValue) atLeastOneColumnIsSearchable = true
+
+    const column = entity.elements[columnName]
+    if (column?.isAssociation || columnName.includes('.')) {
+      deepSearchCandidates.push({ ref: columnName.split('.') })
+      continue;
+    }
     cdsSearchColumnMap.set(columnName, annotationValue)
   }
 
   const searchableColumns = columns.filter(column => {
     const annotatedColumnValue = cdsSearchColumnMap.get(column.name)
+    const elementName = column.as || column.ref?.at(-1) || column.name
+    const element = entity.elements[elementName]
 
     // the element is searchable if it is annotated with the @cds.search, e.g.:
     // `@cds.search { element1: true }` or `@cds.search { element1 }`
     if (annotatedColumnValue) return true
 
+    // calculated elements are only searchable if requested through `@cds.search` 
+    if(column.value) return false
+
     // if at least one element is explicitly annotated as searchable, e.g.:
     // `@cds.search { element1: true }` or `@cds.search { element1 }`
     // and it is not the current column name, then it must be excluded from the search
@@ -101,32 +105,48 @@ const _getSearchableColumns = entity => {
     // if it is not annotated and the column is typed as a string (excluding elements/elements expressions)
     return (
       annotatedColumnValue === undefined &&
-      column._type === DEFAULT_SEARCHABLE_TYPE &&
+      element?.type === DEFAULT_SEARCHABLE_TYPE &&
       !_isColumnCalculated(entity?.query, column.name)
     )
   })
 
-  // if the @cds.search annotation is provided -->
-  // Early return to ignore the interpretation of the @Search.defaultSearchElement
-  // annotation when an entity is annotated with the @cds.search annotation.
-  // The @cds.search annotation overrules the @Search.defaultSearchElement annotation.
-  if (cdsSearchKeys.length > 0) {
-    return searchableColumns.map(column => column.name)
+  if (deepSearchCandidates.length) {
+    deepSearchCandidates.forEach(c => {
+      const element = c.ref.reduce((resolveIn, curr, i) => {
+        const next = resolveIn.elements?.[curr] || resolveIn._target.elements[curr]
+        if (next.isAssociation && !c.ref[i + 1]) {
+          const searchInTarget = _getSearchableColumns(next._target)
+          searchInTarget.forEach(elementRefInTarget => {
+            searchableColumns.push({ ref: c.ref.concat(...elementRefInTarget.ref) })
+          })
+        }
+        return next
+      }, entity)
+      if (element?.type === DEFAULT_SEARCHABLE_TYPE) {
+        searchableColumns.push({ ref: c.ref })
+      }
+    })
   }
 
-  return searchableColumns.map(column => column.name)
+  return searchableColumns.map(column => {
+    if(column.ref)
+      return column
+    return { ref: [ column.name ] }
+  })
 }
 
 /**
  * @returns {Array<object>} - array of columns
  */
-const computeColumnsToBeSearched = (cqn, entity = { __searchableColumns: [] }, alias) => {
+const computeColumnsToBeSearched = (cqn, entity = { __searchableColumns: [] }) => {
   let toBeSearched = []
 
   // aggregations case
   // in the new parser groupBy is moved to sub select.
   if (cqn._aggregated || /* new parser */ cqn.SELECT.groupBy || cqn.SELECT?.from?.SELECT?.groupBy) {
     cqn.SELECT.columns?.forEach(column => {
+      const elementName = column.as || column.ref?.at(-1) || column.name
+      const element = cqn.elements[elementName]
       if (column.func || column.xpr) {
         // exclude $count by SELECT of number of Items in a Collection
         if (
@@ -138,7 +158,7 @@ const computeColumnsToBeSearched = (cqn, entity = { __searchableColumns: [] }, a
         }
 
         // only strings can be searched
-        if (column.element.type !== DEFAULT_SEARCHABLE_TYPE) {
+        if (element?.type !== DEFAULT_SEARCHABLE_TYPE) {
           if (column.xpr) return
           if (column.func && !(column.func in aggregateFunctions)) return
         }
@@ -156,18 +176,22 @@ const computeColumnsToBeSearched = (cqn, entity = { __searchableColumns: [] }, a
 
       // no need to set ref[0] to alias, because columns were already properly transformed
       if (column.ref) {
-        if (column.element.type !== DEFAULT_SEARCHABLE_TYPE) return
+        if (element?.type !== DEFAULT_SEARCHABLE_TYPE) return
         column = { ref: [...column.ref] }
         toBeSearched.push(column)
         return
       }
     })
   } else {
-    toBeSearched = entity.own('__searchableColumns') || entity.set('__searchableColumns', _getSearchableColumns(entity))
-    if (cqn.SELECT.groupBy) toBeSearched = toBeSearched.filter(tbs => cqn.SELECT.groupBy.some(gb => gb.ref[0] === tbs))
+    if(entity.kind === 'entity') {
+      // first check cache
+      toBeSearched = entity.own('__searchableColumns') || entity.set('__searchableColumns', _getSearchableColumns(entity))
+    } else {
+      // if we search on a subquery, we don't have a cache
+      toBeSearched = _getSearchableColumns(entity)
+    }
     toBeSearched = toBeSearched.map(c => {
-      const column = { ref: [c] }
-      if (alias) column.ref.unshift(alias)
+      const column = {ref: [...c.ref]}
       return column
     })
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cap-js/db-service",
-  "version": "1.10.3",
+  "version": "1.11.0",
   "description": "CDS base database service",
   "homepage": "https://github.com/cap-js/cds-dbs/tree/main/db-service#cds-base-database-service",
   "repository": {