@cap-js/db-service 1.5.0 → 1.6.0

package/lib/cqn2sql.js

@@ -2,6 +2,12 @@ const cds = require('@sap/cds/lib')
 const cds_infer = require('./infer')
 const cqn4sql = require('./cqn4sql')
 
+const BINARY_TYPES = {
+  'cds.Binary': 1,
+  'cds.LargeBinary': 1,
+  'cds.hana.BINARY': 1,
+}
+
 const { Readable } = require('stream')
 
 const DEBUG = (() => {
@@ -27,7 +33,7 @@ class CQN2SQLRenderer {
     this.class._init() // is a noop for subsequent calls
   }
 
-  static _add_mixins (aspect, mixins) {
+  static _add_mixins(aspect, mixins) {
     const fqn = this.name + aspect
     const types = cds.builtin.types
     for (let each in mixins) {
@@ -52,7 +58,7 @@ class CQN2SQLRenderer {
       this.ReservedWords[each[0] + each.slice(1).toLowerCase()] = 1 // Order
       this.ReservedWords[each.toLowerCase()] = 1 // order
     }
-    this._init = () => {} // makes this a noop for subsequent calls
+    this._init = () => { } // makes this a noop for subsequent calls
   }
 
   /**
@@ -208,7 +214,7 @@ class CQN2SQLRenderer {
     if (limit) sql += ` LIMIT ${this.limit(limit)}`
     // Expand cannot work without an inferred query
     if (expand) {
-      if ('elements' in q) sql = this.SELECT_expand (q,sql)
+      if ('elements' in q) sql = this.SELECT_expand(q, sql)
       else cds.error`Query was not inferred and includes expand. For which the metadata is missing.`
     }
     return (this.sql = sql)
@@ -237,7 +243,7 @@ class CQN2SQLRenderer {
 
     let cols = SELECT.columns.map(x => {
       const name = this.column_name(x)
-      let col = `'${name}',${this.output_converter4(x.element, this.quote(name))}`
+      let col = `'$."${name}"',${this.output_converter4(x.element, this.quote(name))}`
       if (x.SELECT?.count) {
         // Return both the sub select and the count for @odata.count
         const qc = cds.ql.clone(x, { columns: [{ func: 'count' }], one: 1, limit: 0, orderBy: 0 })
@@ -246,21 +252,14 @@ class CQN2SQLRenderer {
       return col
     }).flat()
 
-    // Prevent SQLite from hitting function argument limit of 100
-    let obj = ''
+    const isRoot = SELECT.expand === 'root'
 
-    if(cols.length < 50) obj =  `json_object(${cols.slice(0, 50)})`
-    else {
-      const chunks = []
-      for (let i = 0; i < cols.length; i += 50) {
-        chunks.push(`json_object(${cols.slice(i, i + 50)})`)
-      }
-      // REVISIT: json_merge is a user defined function, bad performance!
-      obj = `json_merge(${chunks})`
+    // Prevent SQLite from hitting function argument limit of 100
+    let obj = "'{}'"
+    for (let i = 0; i < cols.length; i += 48) {
+      obj = `jsonb_insert(${obj},${cols.slice(i, i + 48)})`
     }
-
-
-    return `SELECT ${SELECT.one || SELECT.expand === 'root' ? obj : `json_group_array(${obj.includes('json_merge') ? `json_insert(${obj})` : obj})`} as _json_ FROM (${sql})`
+    return `SELECT ${isRoot || SELECT.one ? obj.replace('jsonb', 'json') : `jsonb_group_array(${obj})`} as _json_ FROM (${sql})`
   }
 
   /**
@@ -342,9 +341,9 @@ class CQN2SQLRenderer {
     return orderBy.map(
       localized
         ? c =>
-            this.expr(c) +
-            (c.element?.[this.class._localized] ? ' COLLATE NOCASE' : '') +
-            (c.sort === 'desc' || c.sort === -1 ? ' DESC' : ' ASC')
+          this.expr(c) +
+          (c.element?.[this.class._localized] ? ' COLLATE NOCASE' : '') +
+          (c.sort === 'desc' || c.sort === -1 ? ' DESC' : ' ASC')
         : c => this.expr(c) + (c.sort === 'desc' || c.sort === -1 ? ' DESC' : ' ASC'),
     )
   }
@@ -372,12 +371,12 @@ class CQN2SQLRenderer {
     return INSERT.entries
       ? this.INSERT_entries(q)
       : INSERT.rows
-      ? this.INSERT_rows(q)
-      : INSERT.values
-      ? this.INSERT_values(q)
-      : INSERT.as
-      ? this.INSERT_select(q)
-      : cds.error`Missing .entries, .rows, or .values in ${q}`
+        ? this.INSERT_rows(q)
+        : INSERT.values
+          ? this.INSERT_values(q)
+          : INSERT.as
+            ? this.INSERT_select(q)
+            : cds.error`Missing .entries, .rows, or .values in ${q}`
   }
 
   /**
@@ -394,12 +393,18 @@ class CQN2SQLRenderer {
       return // REVISIT: mtx sends an insert statement without entries and no reference entity
     }
     const columns = elements
-      ? ObjectKeys(elements).filter(c => c in elements && !elements[c].virtual && !elements[c].isAssociation)
+      ? ObjectKeys(elements).filter(c => c in elements && !elements[c].virtual && !elements[c].value && !elements[c].isAssociation)
       : ObjectKeys(INSERT.entries[0])
 
     /** @type {string[]} */
     this.columns = columns.filter(elements ? c => !elements[c]?.['@cds.extension'] : () => true).map(c => this.quote(c))
 
+    if (!elements) {
+      this.entries = INSERT.entries.map(e => columns.map(c => e[c]))
+      const param = this.param.bind(this, { ref: ['?'] })
+      return (this.sql = `INSERT INTO ${this.quote(entity)}${alias ? ' as ' + this.quote(alias) : ''} (${this.columns}) VALUES (${columns.map(param)})`)
+    }
+
     const extractions = this.managed(
       columns.map(c => ({ name: c })),
       elements,
@@ -426,10 +431,119 @@ class CQN2SQLRenderer {
 
     // Include this.values for placeholders
     /** @type {unknown[][]} */
-    this.entries = [[...this.values, JSON.stringify(INSERT.entries)]]
-    return (this.sql = `INSERT INTO ${this.quote(entity)}${alias ? ' as ' + this.quote(alias) : ''} (${
-      this.columns
-    }) SELECT ${extraction} FROM json_each(?)`)
+    this.entries = []
+    if (INSERT.entries[0] instanceof Readable) {
+      INSERT.entries[0].type = 'json'
+      this.entries = [[...this.values, INSERT.entries[0]]]
+    } else {
+      const stream = Readable.from(this.INSERT_entries_stream(INSERT.entries), { objectMode: false })
+      stream.type = 'json'
+      this.entries = [[...this.values, stream]]
+    }
+
+    return (this.sql = `INSERT INTO ${this.quote(entity)}${alias ? ' as ' + this.quote(alias) : ''} (${this.columns
+      }) SELECT ${extraction} FROM json_each(?)`)
+  }
+
+  async *INSERT_entries_stream(entries, binaryEncoding = 'base64') {
+    const elements = this.cqn.target?.elements || {}
+    const transformBase64 = binaryEncoding === 'base64'
+      ? a => a
+      : a => a != null ? Buffer.from(a, 'base64').toString(binaryEncoding) : a
+    const bufferLimit = 65536 // 1 << 16
+    let buffer = '['
+
+    let sep = ''
+    for (const row of entries) {
+      buffer += `${sep}{`
+      if (!sep) sep = ','
+
+      let sepsub = ''
+      for (const key in row) {
+        const keyJSON = `${sepsub}${JSON.stringify(key)}:`
+        if (!sepsub) sepsub = ','
+
+        let val = row[key]
+        if (val instanceof Readable) {
+          buffer += `${keyJSON}"`
+
+          // TODO: double check that it works
+          val.setEncoding(binaryEncoding)
+          for await (const chunk of val) {
+            buffer += chunk
+            if (buffer.length > bufferLimit) {
+              yield buffer
+              buffer = ''
+            }
+          }
+
+          buffer += '"'
+        } else {
+          if (elements[key]?.type in BINARY_TYPES) {
+            val = transformBase64(val)
+          }
+          buffer += `${keyJSON}${val === undefined ? 'null' : JSON.stringify(val)}`
+        }
+      }
+      buffer += '}'
+      if (buffer.length > bufferLimit) {
+        yield buffer
+        buffer = ''
+      }
+    }
+
+    buffer += ']'
+    yield buffer
+  }
+
+  async *INSERT_rows_stream(entries, binaryEncoding = 'base64') {
+    const elements = this.cqn.target?.elements || {}
+    const transformBase64 = binaryEncoding === 'base64'
+      ? a => a
+      : a => a != null ? Buffer.from(a, 'base64').toString(binaryEncoding) : a
+    const bufferLimit = 65536 // 1 << 16
+    let buffer = '['
+
+    let sep = ''
+    for (const row of entries) {
+      buffer += `${sep}[`
+      if (!sep) sep = ','
+
+      let sepsub = ''
+      for (let key = 0; key < row.length; key++) {
+        let val = row[key]
+        if (val instanceof Readable) {
+          buffer += `${sepsub}"`
+
+          // TODO: double check that it works
+          val.setEncoding(binaryEncoding)
+          for await (const chunk of val) {
+            buffer += chunk
+            if (buffer.length > bufferLimit) {
+              yield buffer
+              buffer = ''
+            }
+          }
+
+          buffer += '"'
+        } else {
+          if (elements[this.columns[key]]?.type in BINARY_TYPES) {
+            val = transformBase64(val)
+          }
+          buffer += `${sepsub}${val === undefined ? 'null' : JSON.stringify(val)}`
+        }
+
+        if (!sepsub) sepsub = ','
+      }
+      buffer += ']'
+      if (buffer.length > bufferLimit) {
+        yield buffer
+        buffer = ''
+      }
+    }
+
+    buffer += ']'
+    yield buffer
   }
 
   /**
@@ -443,21 +557,35 @@ class CQN2SQLRenderer {
     const alias = INSERT.into.as
     const elements = q.elements || q.target?.elements
     const columns = INSERT.columns
-    || cds.error`Cannot insert rows without columns or elements`
+      || cds.error`Cannot insert rows without columns or elements`
 
     const inputConverter = this.class._convertInput
-    const extraction = columns.map((c,i) => {
+    const extraction = columns.map((c, i) => {
       const extract = `value->>'$[${i}]'`
       const element = elements?.[c]
       const converter = element?.[inputConverter]
-      return converter?.(extract,element) || extract
+      return converter?.(extract, element) || extract
     })
 
     this.columns = columns.map(c => this.quote(c))
-    this.entries = [[JSON.stringify(INSERT.rows)]]
-    return (this.sql = `INSERT INTO ${this.quote(entity)}${alias ? ' as ' + this.quote(alias) : ''} (${
-      this.columns
-    }) SELECT ${extraction} FROM json_each(?)`)
+
+    if (!elements) {
+      this.entries = INSERT.rows
+      const param = this.param.bind(this, { ref: ['?'] })
+      return (this.sql = `INSERT INTO ${this.quote(entity)}${alias ? ' as ' + this.quote(alias) : ''} (${this.columns}) VALUES (${columns.map(param)})`)
+    }
+
+    if (INSERT.rows[0] instanceof Readable) {
+      INSERT.rows[0].type = 'json'
+      this.entries = [[...this.values, INSERT.rows[0]]]
+    } else {
+      const stream = Readable.from(this.INSERT_rows_stream(INSERT.rows), { objectMode: false })
+      stream.type = 'json'
+      this.entries = [[...this.values, stream]]
+    }
+
+    return (this.sql = `INSERT INTO ${this.quote(entity)}${alias ? ' as ' + this.quote(alias) : ''} (${this.columns
+      }) SELECT ${extraction} FROM json_each(?)`)
   }
 
   /**
@@ -517,16 +645,23 @@ class CQN2SQLRenderer {
    * @returns {string} SQL
    */
   UPSERT(q) {
-    let { UPSERT } = q,
-      sql = this.INSERT({ __proto__: q, INSERT: UPSERT })
+    const { UPSERT } = q
+    const elements = q.target?.elements || {}
+    let sql = this.INSERT({ __proto__: q, INSERT: UPSERT })
     let keys = q.target?.keys
-    if (!keys) return (this.sql = sql) // REVISIT: We should converge q.target and q._target
+    if (!keys) return this.sql = sql
     keys = Object.keys(keys).filter(k => !keys[k].isAssociation)
 
     let updateColumns = q.UPSERT.entries ? Object.keys(q.UPSERT.entries[0]) : this.columns
-    updateColumns = updateColumns
-      .filter(c => !keys.includes(c))
-      .map(c => `${this.quote(c)} = excluded.${this.quote(c)}`)
+    updateColumns = updateColumns.filter(c => {
+      if (keys.includes(c)) return false //> keys go into ON CONFLICT clause
+      let e = elements[c]
+      if (!e) return true //> pass through to native SQL columns not in CDS model
+      if (e.virtual) return true //> skip virtual elements
+      if (e.value) return true //> skip calculated elements
+      // if (e.isAssociation) return true //> this breaks a a test in @sap/cds -> need to follow up how to correctly handle deep upserts
+      else return true
+    }).map(c => `${this.quote(c)} = excluded.${this.quote(c)}`)
 
     // temporal data
     keys.push(...Object.values(q.target.elements).filter(e => e['@cds.valid.from']).map(e => e.name))
@@ -552,9 +687,9 @@ class CQN2SQLRenderer {
     if (entity.as) sql += ` AS ${entity.as}`
 
     let columns = []
-    if (data) _add (data, val => this.val({val}))
-    if (_with) _add (_with, x => this.expr(x))
-    function _add (data, sql4) {
+    if (data) _add(data, val => this.val({ val }))
+    if (_with) _add(_with, x => this.expr(x))
+    function _add(data, sql4) {
       for (let c in data) {
         if (!elements || (c in elements && !elements[c].virtual)) {
           columns.push({ name: c, sql: sql4(data[c]) })
@@ -565,7 +700,7 @@ class CQN2SQLRenderer {
     columns = columns.map(c => {
       if (q.elements?.[c.name]?.['@cds.extension']) return {
         name: 'extensions__',
-        sql: `json_set(extensions__,${this.string('$."' + c.name + '"')},${c.sql})`,
+        sql: `jsonb_set(extensions__,${this.string('$."' + c.name + '"')},${c.sql})`,
       }
       return c
     })
@@ -590,73 +725,6 @@ class CQN2SQLRenderer {
     return (this.sql = sql)
   }
 
-  // STREAM Statement -------------------------------------------------
-
-  /**
-   * Renders a STREAM query into generic SQL
-   * @param {import('./infer/cqn').STREAM} q
-   * @returns {string} SQL
-   */
-  STREAM(q) {
-    const { STREAM } = q
-    return STREAM.from
-      ? this.STREAM_from(q)
-      : STREAM.into
-      ? this.STREAM_into(q)
-      : cds.error`Missing .form or .into in ${q}`
-  }
-
-  /**
-   * Renders a STREAM.into query into generic SQL
-   * @param {import('./infer/cqn').STREAM} q
-   * @returns {string} SQL
-   */
-  STREAM_into(q) {
-    const { into, column, where, data } = q.STREAM
-
-    let sql
-    if (!_empty(column)) {
-      data.type = 'binary'
-      const update = UPDATE(into)
-        .with({ [column]: data })
-        .where(where)
-      Object.defineProperty(update, 'target', { value: q.target })
-      sql = this.UPDATE(update)
-    } else {
-      data.type = 'json'
-      // REVISIT: decide whether dataset streams should behave like INSERT or UPSERT
-      sql = this.UPSERT(UPSERT([{}]).into(into).forSQL())
-      this.values = [data]
-    }
-
-    return (this.sql = sql)
-  }
-
-  /**
-   * Renders a STREAM.from query into generic SQL
-   * @param {import('./infer/cqn').STREAM} q
-   * @returns {string} SQL
-   */
-  STREAM_from(q) {
-    const { column, from, where, columns } = q.STREAM
-
-    const select = cds.ql
-      .SELECT(column ? [column] : columns)
-      .where(where)
-      .limit(column ? 1 : undefined)
-
-    // SELECT.from() does not accept joins
-    select.SELECT.from = from
-
-    if (column) {
-      this.one = true
-    } else {
-      select.SELECT.expand = 'root'
-      this.one = !!from.SELECT?.one
-    }
-    return this.SELECT(select.forSQL())
-  }
-
   // Expression Clauses ---------------------------------------------
 
   /**
@@ -668,7 +736,7 @@ class CQN2SQLRenderer {
   expr(x) {
     const wrap = x.cast ? sql => `cast(${sql} as ${this.type4(x.cast)})` : sql => sql
     if (typeof x === 'string') throw cds.error`Unsupported expr: ${x}`
-    if ('param' in x) return wrap(this.param(x))
+    if (x.param) return wrap(this.param(x))
     if ('ref' in x) return wrap(this.ref(x))
     if ('val' in x) return wrap(this.val(x))
     if ('xpr' in x) return wrap(this.xpr(x))
@@ -704,15 +772,15 @@ class CQN2SQLRenderer {
   operator(x, i, xpr) {
 
     // Translate = to IS NULL for rhs operand being NULL literal
-    if (x === '=')  return xpr[i+1]?.val === null ? 'is' : '='
+    if (x === '=') return xpr[i + 1]?.val === null ? 'is' : '='
 
     // Translate == to IS NOT NULL for rhs operand being NULL literal, otherwise ...
     // Translate == to IS NOT DISTINCT FROM, unless both operands cannot be NULL
-    if (x === '==') return xpr[i+1]?.val === null ? 'is' : _not_null(i-1) && _not_null(i+1) ? '=' : this.is_not_distinct_from_
+    if (x === '==') return xpr[i + 1]?.val === null ? 'is' : _not_null(i - 1) && _not_null(i + 1) ? '=' : this.is_not_distinct_from_
 
     // Translate != to IS NULL for rhs operand being NULL literal, otherwise...
     // Translate != to IS DISTINCT FROM, unless both operands cannot be NULL
-    if (x === '!=') return xpr[i+1]?.val === null ? 'is not' : _not_null(i-1) && _not_null(i+1) ? '<>' : this.is_distinct_from_
+    if (x === '!=') return xpr[i + 1]?.val === null ? 'is not' : _not_null(i - 1) && _not_null(i + 1) ? '<>' : this.is_distinct_from_
 
     else return x
 
@@ -749,9 +817,8 @@ class CQN2SQLRenderer {
    */
   ref({ ref }) {
     switch (ref[0]) {
-      case '$now': return this.func({ func: 'session_context', args: [{ val: '$now' }]})
-      case '$user':
-      case '$user.id': return this.func({ func: 'session_context', args: [{ val: '$user.id' }]})
+      case '$now':  return this.func({ func: 'session_context', args: [{ val: '$now', param: false }] }) // REVISIT: why do we need param: false here?
+      case '$user': return this.func({ func: 'session_context', args: [{ val: '$user.'+ref[1]||'id', param: false }] }) // REVISIT: same here?
       default: return ref.map(r => this.quote(r)).join('.')
     }
   }
@@ -761,7 +828,7 @@ class CQN2SQLRenderer {
    * @param {import('./infer/cqn').val} param0
    * @returns {string} SQL
    */
-  val({ val }) {
+  val({ val, param }) {
     switch (typeof val) {
       case 'function': throw new Error('Function values not supported.')
       case 'undefined': return 'NULL'
@@ -769,14 +836,14 @@ class CQN2SQLRenderer {
       case 'number': return `${val}` // REVISIT for HANA
       case 'object':
         if (val === null) return 'NULL'
-        if (val instanceof Date) return `'${val.toISOString()}'`
-        if (val instanceof Readable) ; // go on with default below
-        else if (Buffer.isBuffer(val)) val = val.toString('base64')
+        if (val instanceof Date) val = val.toJSON() // returns null if invalid
+        else if (val instanceof Readable); // go on with default below
+        else if (Buffer.isBuffer(val)); // go on with default below
         else if (is_regexp(val)) val = val.source
         else val = JSON.stringify(val)
       case 'string': // eslint-disable-line no-fallthrough
     }
-    if (!this.values) return this.string(val)
+    if (!this.values || param === false) return this.string(val)
     else this.values.push(val)
     return '?'
   }
@@ -860,12 +927,12 @@ class CQN2SQLRenderer {
     const requiredColumns = !elements
       ? []
       : Object.keys(elements)
-          .filter(
-            e =>
-              (elements[e]?.[annotation] || (!isUpdate && elements[e]?.default && !elements[e].virtual && !elements[e].isAssociation)) &&
-              !columns.find(c => c.name === e),
-          )
-          .map(name => ({ name, sql: 'NULL' }))
+        .filter(
+          e =>
+            (elements[e]?.[annotation] || (!isUpdate && elements[e]?.default && !elements[e].virtual && !elements[e].isAssociation)) &&
+            !columns.find(c => c.name === e),
+        )
+        .map(name => ({ name, sql: 'NULL' }))
 
     return [...columns, ...requiredColumns].map(({ name, sql }) => {
       let element = elements?.[name] || {}
@@ -875,14 +942,13 @@ class CQN2SQLRenderer {
       if (converter && sql[0] !== '$') sql = converter(sql, element)
 
       let val = _managed[element[annotation]?.['=']]
-      if (val) sql = `coalesce(${sql}, ${this.func({ func: 'session_context', args: [{ val }] })})`
+      if (val) sql = `coalesce(${sql}, ${this.func({ func: 'session_context', args: [{ val, param: false }] })})`
       else if (!isUpdate && element.default) {
         const d = element.default
         if (d.val !== undefined || d.ref?.[0] === '$now') {
           // REVISIT: d.ref is not used afterwards
-          sql = `(CASE WHEN json_type(value,'$."${name}"') IS NULL THEN ${
-            this.defaultValue(d.val) // REVISIT: this.defaultValue is a strange function
-          } ELSE ${sql} END)`
+          sql = `(CASE WHEN json_type(value,'$."${name}"') IS NULL THEN ${this.defaultValue(d.val) // REVISIT: this.defaultValue is a strange function
+            } ELSE ${sql} END)`
         }
       }