@sap/cds 5.6.2 → 5.7.2

package/libx/_runtime/messaging/outbox/utils.js

@@ -0,0 +1,192 @@
+const cds = require('../../cds')
+const waitingTime = require('../common-utils/waitingTime')
+const OutboxRunner = require('./OutboxRunner')
+const { isStandardError } = require('../../common/error/standardError')
+const LOG = cds.log('persistent-outbox')
+const _safeJSONParse = string => {
+  try {
+    return string && JSON.parse(string)
+  } catch (e) {}
+}
+
+const outboxRunner = new OutboxRunner()
+const cdsUser = 'cds.internal.user'
+const messageProcessorRegistered = Symbol('message processor registered')
+
+const _getMessagesEntity = () => {
+  const messagesDbName = 'cds.outbox.Messages'
+  const messagesEntity = cds.model.definitions[messagesDbName]
+  if (!messagesEntity)
+    throw new Error(`The entity '${messagesDbName}' is missing but needed for the persistent outbox.`)
+  return messagesEntity
+}
+
+// REVISIT: Is there a better way?
+const _isProviderTenant = tenant =>
+  cds.requires.uaa && cds.requires.uaa.credentials && cds.requires.uaa.credentials.identityzoneid === tenant
+
+const hasPersistentOutbox = (srv, tenant) => {
+  if (!cds.requires.outbox || cds.requires.outbox.kind !== 'persistent-outbox') return false
+  if (srv.options && srv.options.outbox && srv.options.outbox.kind && srv.options.outbox.kind !== 'persistent-outbox')
+    return false
+  if (cds._mtxEnabled && tenant && _isProviderTenant(tenant)) return false // no persistence for provider account
+  return true
+}
+
+const isUnrecoverable = (service, error) => {
+  let unrecoverable = service.isUnrecoverableError && service.isUnrecoverableError(error)
+  if (unrecoverable === undefined) unrecoverable = error.unrecoverable
+  return unrecoverable || isStandardError(error)
+}
+
+const _processSingleMessage = async (service, message, succeededMessages) => {
+  const msg = _safeJSONParse(message.msg)
+  const userId = msg[cdsUser]
+  delete msg[cdsUser]
+  if (!msg) return message.ID
+  // Promise resolve is necessary because we want to set `cds.context` only
+  // inside this call
+  return Promise.resolve().then(async () => {
+    if (userId) cds.context.user = new cds.User.Privileged(userId)
+    try {
+      service._emitImmediate && (await service._emitImmediate(msg))
+      succeededMessages.push(message.ID)
+    } catch (e) {
+      const failedMessage = {
+        event: msg.event,
+        ID: message.ID,
+        attempts: message.attempts,
+        error: e,
+        unrecoverable: isUnrecoverable(service, e)
+      }
+      throw Object.assign(new Error('processing failed'), { failedMessage })
+    }
+  })
+}
+
+// Note: This function can also run for each tenant on startup
+const processMessages = async (service, tenant, _opts = {}) => {
+  const opts = Object.assign({ attempt: 0 }, _opts)
+  const name = service.name
+  const messagesEntity = _getMessagesEntity()
+
+  outboxRunner.run({ name, tenant }, () => {
+    let letAppCrash = false
+    const config = tenant ? { tenant, user: new cds.User.Privileged() } : { user: new cds.User.Privileged() }
+    const spawn = cds.spawn(async () => {
+      let messages
+      try {
+        const messagesQuery = SELECT.from(messagesEntity)
+          .where({ target: name })
+          .orderBy('timestamp')
+          .limit(opts.chunkSize)
+          .forUpdate()
+        if (opts.maxAttempts) messagesQuery.where({ attempts: { '<': opts.maxAttempts } })
+        messages = await messagesQuery
+      } catch (e) {
+        // could potentially be a timeout
+        const _waitingTime = waitingTime(opts.attempt)
+        LOG._error &&
+          LOG.error(
+            'Outbox SELECT FOR UPDATE failed',
+            opts.attempt > 0
+              ? ''
+              : {
+                  cause: e
+                },
+            `Retrying in ${Math.round(_waitingTime / 1000)} s`
+          )
+        outboxRunner.schedule(
+          {
+            name,
+            tenant,
+            waitingTime: _waitingTime
+          },
+          () => processMessages(service, tenant, { ...opts, attempt: opts.attempt + 1 })
+        )
+        return
+      }
+      const messagesToBeDeleted = []
+      let error
+      for (const m of messages) {
+        try {
+          await _processSingleMessage(service, m, messagesToBeDeleted)
+        } catch (e) {
+          error = e
+          break
+        }
+      }
+      if (error && error.failedMessage.unrecoverable) {
+        // opts.crashOnError is not official!!!
+        if (opts.crashOnError !== false) letAppCrash = true
+        messagesToBeDeleted.push(error.failedMessage.ID)
+      }
+      if (messagesToBeDeleted.length) await DELETE.from(messagesEntity).where('ID in', messagesToBeDeleted)
+      if (error) {
+        if (!error.failedMessage.unrecoverable) {
+          const _waitingTime = waitingTime(error.failedMessage.attempts)
+          const info = { service: name, event: error.failedMessage.event }
+          if (error.failedMessage.attempts > 0) info.cause = error.failedMessage.error
+          LOG._error && LOG.error('Emit failed', info, `Retrying in ${Math.round(_waitingTime / 1000)} s`)
+          await UPDATE(messagesEntity)
+            .where({ ID: error.failedMessage.ID })
+            .set({ attempts: { '+=': 1 } })
+          outboxRunner.schedule(
+            {
+              name,
+              tenant,
+              waitingTime: _waitingTime
+            },
+            () => processMessages(service, tenant, opts)
+          )
+        } else {
+          LOG._error &&
+            LOG.error(
+              'Emit failed',
+              { service: name, event: error.failedMessage.event, cause: error.failedMessage.error },
+              'Unrecoverable, outbox entry deleted'
+            )
+        }
+      } else {
+        outboxRunner.success({ name, tenant })
+        if (messages.length === opts.chunkSize) {
+          processMessages(service, tenant, opts) // We only processed max. opts.chunkSize, so there might be more
+        } else {
+          LOG._trace && LOG.trace(`All persistent-outbox messages processed for service '${service.name}'`)
+        }
+      }
+    }, config)
+    spawn.on('done', () => {
+      if (letAppCrash) process.exit(1)
+      outboxRunner.end({ name, tenant }, () => processMessages(service, tenant, opts))
+    })
+  })
+}
+
+const registerMessageProcessor = (name, context) => {
+  const registry = context[messageProcessorRegistered] || (context[messageProcessorRegistered] = new Set())
+  if (!registry.has(name)) {
+    registry.add(name)
+    return true
+  }
+  return false
+}
+
+const _createMessage = (name, msg, context) => {
+  const _msg = { ...msg, [cdsUser]: context.user.id }
+  const outboxMsg = {
+    ID: cds.utils.uuid(),
+    target: name,
+    timestamp: new Date().toISOString(), // needs to be different for each emit
+    msg: JSON.stringify(_msg)
+  }
+  return outboxMsg
+}
+
+const writeInOutbox = async (name, msg, context) => {
+  const outboxMsg = _createMessage(name, msg, context)
+  const messagesEntity = _getMessagesEntity()
+  return INSERT.into(messagesEntity).entries(outboxMsg)
+}
+
+module.exports = { processMessages, registerMessageProcessor, writeInOutbox, hasPersistentOutbox, isUnrecoverable }

package/libx/_runtime/messaging/service.js

@@ -1,6 +1,7 @@
 const cds = require('../cds')
 const LOG = cds.log('messaging')
 const queued = require('./common-utils/queued')
+const OutboxService = require('./Outbox')
 
 const _topic = declared => declared['@topic'] || declared.name
 
@@ -18,7 +19,7 @@ const _warnAndStripTopicPrefix = event => {
 }
 
 // There's currently no mechanism to detect mocked services, this is the best we can do.
-class MessagingService extends cds.Service {
+class MessagingService extends OutboxService {
   init() {
     // enables queued async operations (without awaiting)
     this.queued = queued()
@@ -56,22 +57,6 @@ class MessagingService extends cds.Service {
       })
     }
 
-    // if outbox is switched on, decorate the emit method to actually do
-    // the emit only when the request succeeded
-    if (this.options.outbox) {
-      const { emit } = this
-      this.emit = function (...args) {
-        const context = this.context || cds.context
-        if (context && typeof context.on === 'function')
-          return context.on('succeeded', () =>
-            emit.call(this, ...args).catch(e => {
-              LOG._error && LOG.error(e)
-            })
-          )
-        return emit.call(this, ...args)
-      }
-    }
-
     const { on } = this
     this.on = function (...args) {
       if (Array.isArray(args[0])) {
@@ -80,6 +65,7 @@ class MessagingService extends cds.Service {
       }
       return on.call(this, ...args)
     }
+    return super.init()
   }
 
   emit(event, data, headers) {
@@ -114,22 +100,22 @@ class MessagingService extends cds.Service {
     }
   }
 
-  message4(event, data, headers) {
-    const msg = typeof event === 'object' ? { ...event } : { event, data, headers }
-    msg.event = _warnAndStripTopicPrefix(msg.event)
-    if (!msg.headers) msg.headers = {}
-    if (!msg.inbound) {
-      msg.headers = { ...msg.headers } // don't change the original object
-      this.prepareHeaders(msg.headers, msg.event)
-      msg.event = this.prepareTopic(msg.event, false)
+  message4(msg) {
+    const _msg = { ...msg }
+    _msg.event = _warnAndStripTopicPrefix(_msg.event)
+    if (!_msg.headers) _msg.headers = {}
+    if (!_msg.inbound) {
+      _msg.headers = { ..._msg.headers } // don't change the original object
+      this.prepareHeaders(_msg.headers, _msg.event)
+      _msg.event = this.prepareTopic(_msg.event, false)
     } else if (this.subscribedTopics) {
       const subscribedEvent =
-        this.subscribedTopics.get(msg.event) ||
-        (this.wildcarded && this.subscribedTopics.get(this.wildcarded(msg.event)))
-      if (!subscribedEvent) throw new Error(`No handler for incoming message with topic '${msg.event}' found.`)
-      msg.event = subscribedEvent
+        this.subscribedTopics.get(_msg.event) ||
+        (this.wildcarded && this.subscribedTopics.get(this.wildcarded(_msg.event)))
+      if (!subscribedEvent) throw new Error(`No handler for incoming message with topic '${_msg.event}' found.`)
+      _msg.event = subscribedEvent
     }
-    return msg
+    return _msg
   }
 }
 

package/libx/_runtime/remote/Service.js

@@ -185,6 +185,21 @@ class RemoteService extends cds.Service {
     })
   }
 
+  // FIXME: This is a dirty hack for this situation:
+  // - This PR has cds.Service.model setter to always consistently apply cds.compile.for.odata, also for RemoteServices, which wasn't the case before
+  // - because of that tests/_runtime/remote/__tests__/integration/odata.test.js fails, which relies on the former behavoir of RemoteServices
+  // NOTE: that test would never have worked for RemoteServices bootstrapped from single cds.model, which is always cds.compiled.for.odata
+  // REVISIT: should become obsolete with Universal CSN
+  set model(m) {
+    const fn = cds.compile.for.odata
+    try {
+      cds.compile.for.odata = m => m
+      super.model = m
+    } finally {
+      cds.compile.for.odata = fn
+    }
+  }
+
   // Overload .handle in order to resolve projections up to a definition that is known by the remote service instance.
   // Result is post processed according to the inverse projection in order to reflect the correct result of the original query.
   async handle(req) {

package/libx/_runtime/remote/utils/client.js

@@ -1,9 +1,11 @@
 const cds = require('../../cds')
 const LOG = cds.log('remote')
 
+const SANITIZE_VALUES = process.env.NODE_ENV === 'production' && cds.env.log.sanitize_values !== false
+
 const cdsLocale = require('../../../../lib/req/locale')
 
-const { convertV2ResponseData } = require('./dataConversion')
+const { convertV2ResponseData, deepSanitize } = require('./data')
 
 let _cloudSdkCore
 
@@ -16,6 +18,12 @@ const PPPD = {
 
 const KINDS_SUPPORTING_BATCH = { odata: 1, 'odata-v2': 1, 'odata-v4': 1 }
 
+const _sanitizeHeaders = headers => {
+  if (headers && headers.authorization) headers.authorization = headers.authorization.split(' ')[0] + ' ...'
+
+  return headers
+}
+
 const _executeHttpRequest = async ({ requestConfig, destination, destinationOptions, jwt }) => {
   const { getDestination, executeHttpRequest } = cloudSdkCore()
 
@@ -41,6 +49,11 @@ const _executeHttpRequest = async ({ requestConfig, destination, destinationOpti
     requestOptions = { fetchCsrfToken: true }
   }
 
+  LOG._debug &&
+    LOG.debug(`${requestConfig.method} ${destination.url}${requestConfig.url}`, {
+      headers: _sanitizeHeaders({ ...requestConfig.headers }),
+      data: requestConfig.data && SANITIZE_VALUES ? deepSanitize(requestConfig.data) : requestConfig.data
+    })
   return executeHttpRequest(destination, requestConfig, requestOptions)
 }
 
@@ -194,8 +207,7 @@ const _getSanitizedError = (e, reqOptions, suppressRemoteResponseBody) => {
   if (correlationId) e.correlationId = correlationId
 
   // sanitize authorization
-  if (e.request.headers && e.request.headers.authorization)
-    e.request.headers.authorization = e.request.headers.authorization.split(' ')[0] + ' ...'
+  _sanitizeHeaders(e.request.headers)
 
   // delete functions and complex objects in config
   for (const k in e) if (typeof e[k] === 'function') delete e[k]

package/libx/_runtime/remote/utils/{dataConversion.js → data.js}

@@ -49,7 +49,6 @@ const _getConvertRecordFn = (target, ieee754Compatible) => record => {
   return record
 }
 
-// eslint-disable-next-line complexity
 const _convertValue = (value, type, ieee754Compatible) => {
   if (value == null) {
     return value
@@ -121,6 +120,17 @@ const convertV2ResponseData = (data, target, ieee754Compatible) => {
   return _convertData(data, target, ieee754Compatible)
 }
 
+const deepSanitize = arg => {
+  if (Array.isArray(arg)) return arg.map(deepSanitize)
+  if (typeof arg === 'object')
+    return Object.keys(arg).reduce((acc, cur) => {
+      acc[cur] = deepSanitize(arg[cur])
+      return acc
+    }, {})
+  return '***'
+}
+
 module.exports = {
-  convertV2ResponseData
+  convertV2ResponseData,
+  deepSanitize
 }

package/libx/_runtime/sqlite/Service.js

@@ -45,12 +45,6 @@ module.exports = class SQLiteDatabase extends DatabaseService {
     this.dbcs = new Map()
   }
 
-  set model(csn) {
-    const m = csn && 'definitions' in csn ? cds.linked(cds.compile.for.odata(csn)) : csn
-    cds.alpha_localized(m)
-    super.model = m
-  }
-
   init() {
     this._registerBeforeHandlers()
     this._registerOnHandlers()
@@ -114,8 +108,12 @@ module.exports = class SQLiteDatabase extends DatabaseService {
    * connection
    */
   async acquire(arg) {
-    // REVISIT: remove fallback arg.user.tenant with cds^6
-    const tenant = (typeof arg === 'string' ? arg : arg.tenant || (arg.user && arg.user.tenant)) || 'anonymous'
+    // in non-multi-tenant scenarios, the default db should be returned regardless of arg
+    let tenant = 'anonymous'
+    if (this.options.multiTenant && arg) {
+      if (typeof arg === 'string') tenant = arg
+      else tenant = arg.tenant || (arg.user && arg.user.tenant) || tenant // > REVISIT: remove fallback arg.user.tenant with cds^6
+    }
 
     let dbc = this.dbcs.get(tenant)
     if (!dbc) {
@@ -129,9 +127,8 @@ module.exports = class SQLiteDatabase extends DatabaseService {
       dbc = await _new(dbUrl)
 
       dbc._queued = []
-      dbc._tenant = tenant
 
-      if (cds.env.features._foreign_key_constraints) {
+      if (cds.env.features._db_foreign_key_constraints) {
         await new Promise((resolve, reject) => {
           dbc.exec('PRAGMA foreign_keys = ON', err => {
             if (err) reject(err)

package/libx/_runtime/sqlite/customBuilder/CustomExpressionBuilder.js

@@ -18,6 +18,25 @@ class CustomExpressionBuilder extends ExpressionBuilder {
     Object.defineProperty(this, 'FunctionBuilder', { value: FunctionBuilder })
     return FunctionBuilder
   }
+
+  _addListToOutputObj(list) {
+    this._outputObj.sql.push('(')
+
+    // sqlite requires "VALUES" keyword for list of list of values
+    if (list[0] && list[0].list && list[0].list[0] && 'val' in list[0].list[0]) {
+      this._outputObj.sql.push('VALUES')
+    }
+
+    for (let i = 0, len = list.length; i < len; i++) {
+      this._expressionElementToSQL(list[i])
+
+      if (len > 1 && i + 1 < len) {
+        this._outputObj.sql.push(',')
+      }
+    }
+
+    this._outputObj.sql.push(')')
+  }
 }
 
 module.exports = CustomExpressionBuilder

package/libx/_runtime/sqlite/execute.js

@@ -2,7 +2,7 @@ const { SQLITE_TYPE_CONVERSION_MAP } = require('./conversion')
 const CustomBuilder = require('./customBuilder')
 const { sqlFactory } = require('../db/sql-builder/')
 const { getPostProcessMapper, postProcess } = require('../db/data-conversion/post-processing')
-const { createJoinCQNFromExpanded, hasExpand, rawToExpanded } = require('../db/expand')
+const { createJoinCQNFromExpanded, hasExpand, rawToExpanded, expandV2 } = require('../db/expand')
 const { Readable } = require('stream')
 
 const cds = require('../cds')
@@ -10,6 +10,8 @@ const LOG = cds.log('sqlite|db|sql')
 // && {_debug:true, debug(sql){ cds._debug && console.log(sql+';\n') }} //> please keep that for debugging stakeholder tests
 const { inspect } = require('util')
 
+const SANITIZE_VALUES = process.env.NODE_ENV === 'production' && cds.env.log.sanitize_values !== false
+
 /*
  * capture stack trace on the way to exec to know origin
  * -> very expensive
@@ -32,20 +34,21 @@ const _colored = {
   COMMIT: '\x1b[1m\x1b[32mCOMMIT\x1b[0m',
   ROLLBACK: '\x1b[1m\x1b[91mROLLBACK\x1b[0m'
 }
-const _augmented = (err, sql, o) => {
+const _augmented = (err, sql, values, o) => {
   err.query = sql
+  if (values) err.values = SANITIZE_VALUES ? ['***'] : values
   err.message += ' in: \n' + sql
   if (o) err.stack = err.message + o.stack.slice(5)
   return err
 }
 
 function _executeSimpleSQL(dbc, sql, values) {
-  LOG._debug && LOG.debug(_colored[sql] || sql, values || '')
+  LOG._debug && LOG.debug(_colored[sql] || sql, Array.isArray(values) ? (SANITIZE_VALUES ? ['***'] : values) : '')
 
   return new Promise((resolve, reject) => {
     const o = _captureStack()
     dbc.run(sql, values, function (err) {
-      if (err) return reject(_augmented(err, sql, o))
+      if (err) return reject(_augmented(err, sql, values, o))
 
       resolve(this.changes)
     })
@@ -53,12 +56,12 @@ function _executeSimpleSQL(dbc, sql, values) {
 }
 
 function executeSelectSQL(dbc, sql, values, isOne, postMapper) {
-  LOG._debug && LOG.debug(sql, values)
+  LOG._debug && LOG.debug(sql, SANITIZE_VALUES ? ['***'] : values)
 
   return new Promise((resolve, reject) => {
     const o = _captureStack()
     dbc[isOne ? 'get' : 'all'](sql, values, (err, result) => {
-      if (err) return reject(_augmented(err, sql, o))
+      if (err) return reject(_augmented(err, sql, values, o))
 
       // REVISIT
       // .get returns undefined if nothing in db
@@ -96,6 +99,10 @@ function _processExpand(model, dbc, cqn, user, locale, txTimestamp) {
 
 function executeSelectCQN(model, dbc, query, user, locale, txTimestamp) {
   if (hasExpand(query)) {
+    // expand: '**' or '*3' is handled by new impl
+    if (query.SELECT.columns.some(c => c.expand && typeof c.expand === 'string' && /^\*{1}[\d|*]+/.test(c.expand))) {
+      return expandV2(model, dbc, query, user, locale, txTimestamp, executeSelectCQN)
+    }
     return _processExpand(model, dbc, query, user, locale, txTimestamp)
   }
   const { sql, values = [] } = sqlFactory(
@@ -138,10 +145,10 @@ const _executeBulkInsertSQL = (dbc, sql, values) =>
       return reject(new Error(`Cannot execute SQL statement. Invalid values provided: ${inspect(values)}`))
     }
 
-    LOG._debug && LOG.debug(sql, values)
+    LOG._debug && LOG.debug(sql, SANITIZE_VALUES ? ['***'] : values)
     const o = _captureStack()
     const stmt = dbc.prepare(sql, err => {
-      if (err) return reject(_augmented(err, sql, o))
+      if (err) return reject(_augmented(err, sql, values, o))
 
       if (!Array.isArray(values[0])) values = [values]
 
@@ -155,11 +162,10 @@ const _executeBulkInsertSQL = (dbc, sql, values) =>
         i++
         stmt.run(each, function (err) {
           if (err) {
-            err.values = each
             if (!isFinalized) {
               isFinalized = true
               stmt.finalize()
-              return reject(_augmented(err, sql, o))
+              return reject(_augmented(err, sql, each, o))
             }
           }
 
@@ -209,12 +215,12 @@ function executeInsertSQL(dbc, sql, values, query) {
     }
   }
 
-  LOG._debug && LOG.debug(sql, values)
+  LOG._debug && LOG.debug(sql, SANITIZE_VALUES ? ['***'] : values)
 
   return new Promise((resolve, reject) => {
     const o = _captureStack()
     dbc.run(sql, values, function (err) {
-      if (err) return reject(_augmented(err, sql, o))
+      if (err) return reject(_augmented(err, sql, values, o))
 
       // InsertResult needs an object per row with its values
       if (query && values.length > 0) {

package/libx/_runtime/types/api.js

@@ -101,7 +101,8 @@
 
 /**
  * @typedef {object} cqn2cqn4sqlOptions
- * @property {boolean} suppressSearch=false Indicates whether the search handler is called.
+ * @property {boolean} [suppressSearch=false] Indicates whether the search handler is called.
+ * @property {import('../db/Service')} [service]
  */
 
 module.exports = {}

package/libx/gql/resolvers/parse/ast2cqn/columns.js

@@ -19,7 +19,7 @@ const astToColumns = selections => {
 
     const filter = getArgumentByName(selection.arguments, ARGUMENT.FILTER)
     if (filter) {
-      column.where = astToWhere(filter).xpr
+      column.where = astToWhere(filter)
     }
 
     const orderBy = getArgumentByName(selection.arguments, ARGUMENT.ORDER_BY)