@cap-js/db-service 1.5.0 → 1.6.0

package/CHANGELOG.md

@@ -4,6 +4,47 @@
 - The format is based on [Keep a Changelog](http://keepachangelog.com/).
 - This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [1.6.0](https://github.com/cap-js/cds-dbs/compare/db-service-v1.5.1...db-service-v1.6.0) (2024-02-02)
+
+
+### Added
+
+* Add fallback for @cap-js/hana for unknown entities ([#403](https://github.com/cap-js/cds-dbs/issues/403)) ([e7dd6de](https://github.com/cap-js/cds-dbs/commit/e7dd6de4ef65881ef66f7ba9c164ff2b4e9b1111))
+* SELECT returns binaries as Buffers ([#416](https://github.com/cap-js/cds-dbs/issues/416)) ([d4240d5](https://github.com/cap-js/cds-dbs/commit/d4240d5efb7789851593c83a430e601d6ff87118))
+* SELECT returns LargeBinaries as streams unless feature flag "stream_compat" is set ([#251](https://github.com/cap-js/cds-dbs/issues/251)) ([8165a4a](https://github.com/cap-js/cds-dbs/commit/8165a4a3f6bb21c970668c8873f9d9c662b43780))
+* strict mode to validate input for `INSERT`, `UPDATE` and `UPSERT` ([#384](https://github.com/cap-js/cds-dbs/issues/384)) ([4644483](https://github.com/cap-js/cds-dbs/commit/464448384145d934933c473ae2f20d49cc75554d))
+* Support Readable Streams inside INSERT.entries ([#343](https://github.com/cap-js/cds-dbs/issues/343)) ([f6faf89](https://github.com/cap-js/cds-dbs/commit/f6faf8955b7888479c66f1727ade65b382611c2f))
+
+
+### Fixed
+
+* **`cqn4sql`:** only transform list if necessary ([#438](https://github.com/cap-js/cds-dbs/issues/438)) ([8a7ec65](https://github.com/cap-js/cds-dbs/commit/8a7ec65fe46c2dae668bb536671943a76d5e8206))
+* always generate unique subquery aliases ([#435](https://github.com/cap-js/cds-dbs/issues/435)) ([c875b7d](https://github.com/cap-js/cds-dbs/commit/c875b7d07a83693febb2543d202fd53b43172f7b))
+* consider `list` in `from.where` ([#429](https://github.com/cap-js/cds-dbs/issues/429)) ([3288e94](https://github.com/cap-js/cds-dbs/commit/3288e943f53a2ba08d97018e016c06932b5c8f88))
+* **cqn2sql:** $user.locale refs ([#431](https://github.com/cap-js/cds-dbs/issues/431)) ([ec55276](https://github.com/cap-js/cds-dbs/commit/ec55276409ccd56d8b831bbff3d3915e078d3f72))
+* **cqn4sql:** expand structured keys in on-conditions ([#421](https://github.com/cap-js/cds-dbs/issues/421)) ([b1e0677](https://github.com/cap-js/cds-dbs/commit/b1e06777ccfce80f50443e61a10ae5d86c6bc232))
+* Do not generate UUIDs for association keys ([#398](https://github.com/cap-js/cds-dbs/issues/398)) ([9970e14](https://github.com/cap-js/cds-dbs/commit/9970e14352679711a9c60807608becff05151fc4))
+* enumeration issue with session context in @cap-js/hana ([#399](https://github.com/cap-js/cds-dbs/issues/399)) ([8106a20](https://github.com/cap-js/cds-dbs/commit/8106a207543be700d37b1f1b510d00d5dd1370e4))
+* make @cap-js/sqlite work with better-sqlite3@9.3.0 ([#422](https://github.com/cap-js/cds-dbs/issues/422)) ([44c0a59](https://github.com/cap-js/cds-dbs/commit/44c0a59277b14be0b81b7f80555e18377ddbfe3c))
+* pass context of navigation for list within infix filter ([#433](https://github.com/cap-js/cds-dbs/issues/433)) ([0ca077f](https://github.com/cap-js/cds-dbs/commit/0ca077f071a1569fa3b46f6ccfb003feaebd1ea0))
+* Restore former deep upsert behavior / error ([#406](https://github.com/cap-js/cds-dbs/issues/406)) ([284b1e3](https://github.com/cap-js/cds-dbs/commit/284b1e3a605957d91dd867794ab1a7dcdf345c40))
+* Skip virtual fields on UPSERTs ([#405](https://github.com/cap-js/cds-dbs/issues/405)) ([1a05dcb](https://github.com/cap-js/cds-dbs/commit/1a05dcb1d032a85e826c76f5a8a710161fa2b679))
+* sqlite date string compatibility parsing only for valid dates ([#410](https://github.com/cap-js/cds-dbs/issues/410)) ([2a8bb2d](https://github.com/cap-js/cds-dbs/commit/2a8bb2d60940760c6280d8cc06100cb9087194b5)), closes [#409](https://github.com/cap-js/cds-dbs/issues/409)
+* UPSERT for @cap-js/hana for entities with multiple keys ([#418](https://github.com/cap-js/cds-dbs/issues/418)) ([9bbac6e](https://github.com/cap-js/cds-dbs/commit/9bbac6ebbbddfa2f620833ce195eedeb0a79f43e))
+
+## [1.5.1](https://github.com/cap-js/cds-dbs/compare/db-service-v1.5.0...db-service-v1.5.1) (2023-12-20)
+
+
+### Fixed
+
+* **cqn2sql:** supporting calculated elements ([#387](https://github.com/cap-js/cds-dbs/issues/387)) ([2153fb9](https://github.com/cap-js/cds-dbs/commit/2153fb9a3910cd4afa3a91918e6cf682646492b7))
+* do not rely on db constraints for deep delete ([#390](https://github.com/cap-js/cds-dbs/issues/390)) ([9623af6](https://github.com/cap-js/cds-dbs/commit/9623af64db97cfe15ef07b659635850fc908f77c))
+
+
+### Performance Improvements
+
+* HANA list placeholder ([#380](https://github.com/cap-js/cds-dbs/issues/380)) ([3eadfea](https://github.com/cap-js/cds-dbs/commit/3eadfea7b94f485030cc8bd0bd298ce088586422))
+
 ## [1.5.0](https://github.com/cap-js/cds-dbs/compare/db-service-v1.4.0...db-service-v1.5.0) (2023-12-06)
 
 

package/lib/SQLService.js

@@ -1,9 +1,15 @@
 const cds = require('@sap/cds/lib'),
   DEBUG = cds.debug('sql|db')
+const { Readable } = require('stream')
 const { resolveView } = require('@sap/cds/libx/_runtime/common/utils/resolveView')
 const DatabaseService = require('./common/DatabaseService')
 const cqn4sql = require('./cqn4sql')
 
+const BINARY_TYPES = {
+  'cds.Binary': 1,
+  'cds.hana.BINARY': 1
+}
+
 /** @typedef {import('@sap/cds/apis/services').Request} Request */
 
 /**
@@ -14,12 +20,29 @@ const cqn4sql = require('./cqn4sql')
  */
 
 class SQLService extends DatabaseService {
-
   init() {
-    this.on(['SELECT'], this.transformStreamFromCQN)
-    this.on(['UPDATE'], this.transformStreamIntoCQN)
-    this.on(['INSERT', 'UPSERT', 'UPDATE'], require('./fill-in-keys')) // REVISIT: should be replaced by correct input processing eventually
+    this.on(['INSERT', 'UPSERT', 'UPDATE'], require('./fill-in-keys')) // REVISIT should be replaced by correct input processing eventually
     this.on(['INSERT', 'UPSERT', 'UPDATE'], require('./deep-queries').onDeep)
+    if (cds.env.features.db_strict) {
+      this.before(['INSERT', 'UPSERT', 'UPDATE'], ({ query }) => {
+        const elements = query.target?.elements; if (!elements) return
+        const kind = query.kind || Object.keys(query)[0]
+        const operation = query[kind]
+        if (!operation.columns && !operation.entries && !operation.data) return
+        const columns =
+          operation.columns ||
+          Object.keys(
+            operation.data || operation.entries?.reduce((acc, obj) => {
+              return Object.assign(acc, obj)
+            }, {}),
+          )
+          const invalidColumns = columns.filter(c => !(c in elements))
+
+        if (invalidColumns.length > 0) {
+          cds.error(`STRICT MODE: Trying to ${kind} non existent columns (${invalidColumns})`)
+        }
+      })
+    }
     this.on(['SELECT'], this.onSELECT)
     this.on(['INSERT'], this.onINSERT)
     this.on(['UPSERT'], this.onUPSERT)
@@ -27,46 +50,63 @@ class SQLService extends DatabaseService {
     this.on(['DELETE'], this.onDELETE)
     this.on(['CREATE ENTITY', 'DROP ENTITY'], this.onSIMPLE)
     this.on(['BEGIN', 'COMMIT', 'ROLLBACK'], this.onEVENT)
-    this.on(['STREAM'], this.onSTREAM)
     this.on(['*'], this.onPlainSQL)
     return super.init()
   }
 
-  /** @type {Handler} */
-  async transformStreamFromCQN({ query }, next) {
-    if (!query._streaming) return next()
-    const cqn = STREAM.from(query.SELECT.from).column(query.SELECT.columns[0].ref[0])
-    if (query.SELECT.where) cqn.STREAM.where = query.SELECT.where
-    const stream = await this.run(cqn)
-    return stream && { value: stream }
-  }
+  _changeToStreams(columns, rows, one, compat) {
+    if (!rows || !columns) return
+    if (!Array.isArray(rows)) rows = [rows]
+    if (!rows.length || !Object.keys(rows[0]).length) return
 
-  /** @type {Handler} */
-  async transformStreamIntoCQN({ query, data, target }, next) {
-    let col, type, etag
-    const elements = query._target?.elements || target?.elements
-    if (!elements) next()
-    for (const key in elements) {
-      const element = elements[key]
-      if (element['@Core.MediaType'] && data[key]?.pipe) col = key
-      if (element['@Core.IsMediaType'] && data[key]) type = key
-      if (element['@odata.etag'] && data[key]) etag = key
+    // REVISIT: remove after removing stream_compat feature flag
+    if (compat) {
+      rows[0][Object.keys(rows[0])[0]] = this._stream(Object.values(rows[0])[0])
+      return
     }
 
-    if (!col) return next()
-
-    const cqn = STREAM.into(query.UPDATE.entity).column(col).data(data[col])
-    if (query.UPDATE.where) cqn.STREAM.where = query.UPDATE.where
-    const result = await this.run(cqn)
-    if (type || etag) {
-      const d = { ...data }
-      delete d[col]
-      const cqn = UPDATE.entity(query.UPDATE.entity).with(d)
-      if (query.UPDATE.where) cqn.UPDATE.where = query.UPDATE.where
-      await this.run(cqn)
+    for (let col of columns) {
+      const name = col.as || col.ref?.[col.ref.length - 1] || (typeof col === 'string' && col)
+      if (col.element?.isAssociation) {
+        if (one) this._changeToStreams(col.SELECT.columns, rows[0][name], false, compat)
+        else
+          rows.forEach(row => {
+            this._changeToStreams(col.SELECT.columns, row[name], false, compat)
+          })
+      } else if (col.element?.type === 'cds.LargeBinary') {
+        if (one) rows[0][name] = this._stream(rows[0][name])
+        else
+          rows.forEach(row => {
+            row[name] = this._stream(row[name])
+          })
+      } else if (col.element?.type in BINARY_TYPES) {
+        if (one) rows[0][name] = this._buffer(rows[0][name])
+        else
+          rows.forEach(row => {
+            row[name] = this._buffer(row[name])
+          })
+      }
     }
+  }
 
-    return result
+  _stream(val) {
+    if (val === null) return null
+    if (val instanceof Readable) return val
+    // Buffer.from only applies encoding when the input is a string
+    let raw = typeof val === 'string' ? Buffer.from(val.toString(), 'base64') : val
+    return new Readable({
+      read(size) {
+        if (raw.length === 0) return this.push(null)
+        const chunk = raw.slice(0, size)
+        raw = raw.slice(size)
+        this.push(chunk)
+      },
+    })
+  }
+
+  _buffer(val) {
+    if (val === null) return null
+    return Buffer.from(val, 'base64')
   }
 
   /**
@@ -74,15 +114,27 @@ class SQLService extends DatabaseService {
    * @type {Handler}
    */
   async onSELECT({ query, data }) {
+    query.SELECT.expand = 'root'
     const { sql, values, cqn } = this.cqn2sql(query, data)
     let ps = await this.prepare(sql)
     let rows = await ps.all(values)
     if (rows.length)
       if (cqn.SELECT.expand) rows = rows.map(r => (typeof r._json_ === 'string' ? JSON.parse(r._json_) : r._json_ || r))
+
+    if (cds.env.features.stream_compat) {
+      if (query._streaming) {
+        this._changeToStreams(cqn.SELECT.columns, rows, true, true)
+        return rows.length ? { value: Object.values(rows[0])[0] } : undefined
+      }
+    } else {
+      this._changeToStreams(cqn.SELECT.columns, rows, query.SELECT.one, false)
+    }
+
     if (cqn.SELECT.count) {
       // REVISIT: the runtime always expects that the count is preserved with .map, required for renaming in mocks
       return SQLService._arrayWithCount(rows, await this.count(query, rows))
     }
+
     return cqn.SELECT.one || query.SELECT.from?.ref?.[0].cardinality?.max === 1 ? rows[0] : rows
   }
 
@@ -126,22 +178,6 @@ class SQLService extends DatabaseService {
     return this.onSIMPLE(req)
   }
 
-  /**
-   * Handler for Stream
-   * @type {Handler}
-   */
-  async onSTREAM(req) {
-    const { one, sql, values } = this.cqn2sql(req.query)
-    // writing stream
-    if (req.query.STREAM.into) {
-      const ps = await this.prepare(sql)
-      return (await ps.run(values)).changes
-    }
-    // reading stream
-    const ps = await this.prepare(sql)
-    return ps.stream(values, one)
-  }
-
   /**
    * Handler for CREATE, DROP, UPDATE, DELETE, with simple CQN
    * @type {Handler}
@@ -153,7 +189,8 @@ class SQLService extends DatabaseService {
   }
 
   get onDELETE() {
-    return super.onDELETE = cds.env.features.assert_integrity === 'db' ? this.onSIMPLE : deep_delete
+    // REVISIT: It's not yet 100 % clear under which circumstances we can rely on db constraints
+    return (super.onDELETE = /* cds.env.features.assert_integrity === 'db' ? this.onSIMPLE : */ deep_delete)
     async function deep_delete(/** @type {Request} */ req) {
       let { compositions } = req.target
       if (compositions) {
@@ -162,24 +199,29 @@ class SQLService extends DatabaseService {
         if (typeof from === 'string') from = { ref: [from] }
         if (where) {
           let last = from.ref.at(-1)
-          if (last.where) [ last, where ] = [ last.id, [ { xpr: last.where }, 'and', { xpr: where } ] ]
-          from = {ref:[ ...from.ref.slice(0,-1), { id: last, where }]}
+          if (last.where) [last, where] = [last.id, [{ xpr: last.where }, 'and', { xpr: where }]]
+          from = { ref: [...from.ref.slice(0, -1), { id: last, where }] }
         }
         // Process child compositions depth-first
-        let { depth=0, visited=[] } = req
-        visited.push (req.target.name)
-        await Promise.all (Object.values(compositions).map(c => {
-          if (c._target['@cds.persistence.skip'] === true) return
-          if (c._target === req.target) { // the Genre.children case
-            if (++depth > (c['@depth'] || 3)) return
-          } else if (visited.includes(c._target.name)) throw new Error(
-            `Transitive circular composition detected: \n\n`+
-            `  ${visited.join(' > ')} > ${c._target.name} \n\n`+
-            `These are not supported by deep delete.`)
-          // Prepare and run deep query, à la CQL`DELETE from Foo[pred]:comp1.comp2...`
-          const query = DELETE.from({ref:[ ...from.ref, c.name ]})
-          return this.onDELETE({ query, depth, visited: [...visited], target: c._target })
-        }))
+        let { depth = 0, visited = [] } = req
+        visited.push(req.target.name)
+        await Promise.all(
+          Object.values(compositions).map(c => {
+            if (c._target['@cds.persistence.skip'] === true) return
+            if (c._target === req.target) {
+              // the Genre.children case
+              if (++depth > (c['@depth'] || 3)) return
+            } else if (visited.includes(c._target.name))
+              throw new Error(
+                `Transitive circular composition detected: \n\n` +
+                `  ${visited.join(' > ')} > ${c._target.name} \n\n` +
+                `These are not supported by deep delete.`,
+              )
+            // Prepare and run deep query, à la CQL`DELETE from Foo[pred]:comp1.comp2...`
+            const query = DELETE.from({ ref: [...from.ref, c.name] })
+            return this.onDELETE({ query, depth, visited: [...visited], target: c._target })
+          }),
+        )
       }
       return this.onSIMPLE(req)
     }
@@ -232,8 +274,8 @@ class SQLService extends DatabaseService {
     // REVISIT: made uppercase count because of HANA reserved word quoting
     const cq = SELECT.one([{ func: 'count', as: 'COUNT' }]).from(
       cds.ql.clone(query, {
-      localized: false,
-      expand: false,
+        localized: false,
+        expand: false,
         limit: undefined,
         orderBy: undefined,
       }),
@@ -258,10 +300,12 @@ class SQLService extends DatabaseService {
   // preserves $count for .map calls on array
   static _arrayWithCount = function (a, count) {
     const _map = a.map
-    const map = function (..._) { return SQLService._arrayWithCount(_map.call(a, ..._), count) }
+    const map = function (..._) {
+      return SQLService._arrayWithCount(_map.call(a, ..._), count)
+    }
     return Object.defineProperties(a, {
       $count: { value: count, enumerable: false, configurable: true, writable: true },
-      map: { value: map, enumerable: false, configurable: true, writable: true }
+      map: { value: map, enumerable: false, configurable: true, writable: true },
     })
   }
 
@@ -279,10 +323,8 @@ class SQLService extends DatabaseService {
    */
   cqn2sql(query, values) {
     let q = this.cqn4sql(query)
-    if (q.SELECT && 'elements' in q) q.SELECT.expand ??= 'root'
-
     let kind = q.kind || Object.keys(q)[0]
-    if (kind in { INSERT: 1, DELETE: 1, UPSERT: 1, UPDATE: 1 } || q.STREAM?.into) {
+    if (kind in { INSERT: 1, DELETE: 1, UPSERT: 1, UPDATE: 1 }) {
       q = resolveView(q, this.model, this) // REVISIT: before resolveView was called on flat cqn obtained from cqn4sql -> is it correct to call on original q instead?
       let target = q[kind]._transitions?.[0].target
       if (target) q.target = target // REVISIT: Why isn't that done in resolveView?
@@ -296,7 +338,14 @@ class SQLService extends DatabaseService {
    * @returns {import('./infer/cqn').Query}
    */
   cqn4sql(q) {
-    if (!q.SELECT?.from?.join && !q.SELECT?.from?.SELECT && !this.model?.definitions[_target_name4(q)]) return _unquirked(q)
+    if (
+      !cds.env.features.db_strict &&
+      !q.SELECT?.from?.join &&
+      !q.SELECT?.from?.SELECT &&
+      !this.model?.definitions[_target_name4(q)]
+    ) {
+      return _unquirked(q)
+    }
     return cqn4sql(q, this.model)
   }
 
@@ -329,7 +378,6 @@ class SQLService extends DatabaseService {
  * @interface
  */
 class PreparedStatement {
-
   /**
    * Executes a prepared DML query, i.e., INSERT, UPDATE, DELETE, CREATE, DROP
    * @abstract
@@ -379,9 +427,7 @@ const _target_name4 = q => {
     q.UPDATE?.entity ||
     q.DELETE?.from ||
     q.CREATE?.entity ||
-    q.DROP?.entity ||
-    q.STREAM?.from ||
-    q.STREAM?.into
+    q.DROP?.entity
   if (target?.SET?.op === 'union') throw new cds.error('UNION-based queries are not supported')
   if (!target?.ref) return target
   const [first] = target.ref
@@ -400,8 +446,11 @@ const _unquirked = q => {
   return q
 }
 
-
-const sqls = new class extends SQLService { get factory() { return null } }
+const sqls = new (class extends SQLService {
+  get factory() {
+    return null
+  }
+})()
 cds.extend(cds.ql.Query).with(
   class {
     forSQL() {

package/lib/common/DatabaseService.js

@@ -128,22 +128,14 @@ class DatabaseService extends cds.Service {
   }
 
   // REVISIT: should happen automatically after a configurable time
-  /**
-   * @param {string} tenant
-   */
-  async disconnect(tenant) {
-    const _disconnect = async tenant => {
-      const pool = this.pools[tenant]
-      if (!pool) {
-        return
-      }
+  async disconnect (tenant) {
+    const tenants = tenant ? [tenant] : Object.keys(this.pools)
+    await Promise.all (tenants.map (async t => {
+      const pool = this.pools[t]; if (!pool) return
       await pool.drain()
       await pool.clear()
-      delete this.pools[tenant]
-    }
-    if (tenant == null)
-      return Promise.all(Object.keys(this.pools).map(_disconnect))
-    return _disconnect(tenant)
+      delete this.pools[t]
+    }))
   }
 
   /**

package/lib/common/session-context.js

@@ -28,5 +28,19 @@ class TemporalSessionContext extends SessionContext {
   }
 }
 
+// Set all getters as enumerable
+const iterate = { enumerable: true }
+const getters = (obj) => {
+  const prot = obj.prototype
+  const patch = {}
+  for (const [key, value] of Object.entries(Object.getOwnPropertyDescriptors(prot))) {
+    if (!value.get) continue
+    patch[key] = iterate
+  }
+  Object.defineProperties(prot, patch)
+}
+getters(SessionContext)
+getters(TemporalSessionContext)
+
 // REVISIT: only set temporal context if required!
 module.exports = TemporalSessionContext