@sap/cds 7.9.4 → 8.0.4

package/libx/odata/middleware/batch.js

@@ -1,13 +1,13 @@
 const cds = require('../../../')
-const { AsyncResource } = require('async_hooks')
 
-// eslint-disable-next-line cds/no-missing-dependencies
+const { AsyncResource } = require('async_hooks')
 const express = require('express')
 const { STATUS_CODES } = require('http')
 const qs = require('querystring')
 const { URL } = require('url')
 
 const multipartToJson = require('../parse/multipartToJson')
+const { getBoundary } = require('../utils')
 
 const HTTP_METHODS = { GET: 1, POST: 1, PUT: 1, PATCH: 1, DELETE: 1 }
 const CT = { JSON: 'application/json', MULTIPART: 'multipart/mixed' }
@@ -47,7 +47,7 @@ const _validateBatch = body => {
     if (typeof request !== 'object')
       throw _deserializationError(`Element of 'requests' array at index ${i} must be type of 'object'.`)
 
-    const { id, method, url, body, atomicityGroup, dependsOn } = request
+    const { id, method, url, body, atomicityGroup } = request
 
     _validateProperty('id', id, 'string')
 
@@ -83,20 +83,31 @@ const _validateBatch = body => {
       }
     }
 
-    if (dependsOn) {
-      _validateProperty('dependsOn', dependsOn, 'Array')
-      dependsOn.forEach(dependsOnId => {
+    if (url.startsWith('$')) {
+      request.dependsOn ??= []
+      const dependencyId = url.split('/')[0].replace(/^\$/, '')
+      if (!request.dependsOn.includes(dependencyId)) {
+        request.dependsOn.push(dependencyId)
+      }
+    }
+
+    if (request.dependsOn) {
+      _validateProperty('dependsOn', request.dependsOn, 'Array')
+      request.dependsOn.forEach(dependsOnId => {
         _validateProperty('dependent request ID', dependsOnId, 'string')
 
         const dependency = ids[dependsOnId]
-        if (!dependency)
-          throw _deserializationError(`Request ID '${dependsOnId}' used in dependsOn has not been defined before.`)
-
-        const dependencyAtomicityGroup = dependency.atomicityGroup
-        if (dependencyAtomicityGroup && !dependsOn.includes(dependencyAtomicityGroup))
+        if (!dependency) {
           throw _deserializationError(
-            `The group '${dependencyAtomicityGroup}' of the referenced request '${dependsOnId}' must be listed in dependsOn of request '${id}'.`
+            `"${dependsOnId}" does not match the id or atomicity group of any preceding request`
           )
+        }
+
+        // automatically add the atomicityGroup of the dependency as a dependency (actually a client error)
+        const dag = dependency.atomicityGroup
+        if (dag && dag !== atomicityGroup && !request.dependsOn.includes(dag)) {
+          request.dependsOn.push(dag)
+        }
       })
     }
 
@@ -108,6 +119,7 @@ const _validateBatch = body => {
   return ids
 }
 
+// REVISIT: Why not simply use {__proto__:req, ...}?
 const _createExpressReqResLookalike = (request, _req, _res) => {
   const { id, method, url } = request
   const ret = { id }
@@ -123,11 +135,16 @@ const _createExpressReqResLookalike = (request, _req, _res) => {
   const u = new URL(url, 'http://cap')
   req.query = qs.parse(u.search.slice(1))
   req.headers = request.headers || {}
+  if (request.content_id) req.headers['content-id'] = request.content_id
   req.body = request.body
+  if (_req._login) req._login = _req._login
 
   const res = (ret.res = new express.response.constructor(req))
   res.__proto__ = express.response
 
+  // REVISIT: mark as subrequest
+  req._subrequest = true
+
   // express internals
   res.app = _res.app
 
@@ -135,10 +152,11 @@ const _createExpressReqResLookalike = (request, _req, _res) => {
   req.res = res
 
   // resolve promise for subrequest via res.end()
-  ret.promise = new Promise((resolve, _reject) => {
+  ret.promise = new Promise((resolve, reject) => {
     res.end = (chunk, encoding) => {
       res._chunk = chunk
       res._encoding = encoding
+      if (res.statusCode >= 400) return reject(ret)
       resolve(ret)
     }
   })
@@ -146,35 +164,156 @@ const _createExpressReqResLookalike = (request, _req, _res) => {
   return ret
 }
 
-const _transaction = async (srv, router) => {
+const _writeResponseMultipart = (responses, res, rejected, group, boundary) => {
+  if (group) {
+    res.write(`--${boundary}${CRLF}`)
+    res.write(`content-type: multipart/mixed;boundary=${group}${CRLF}${CRLF}`)
+  }
+  const header = group || boundary
+  if (rejected) {
+    const resp = responses.find(r => r.status === 'fail')
+    if (resp.separator && res._writeSeparator) res.write(resp.separator)
+    resp.txt.forEach(txt => {
+      res.write(`--${header}${CRLF}`)
+      res.write(`${txt}`)
+    })
+  } else {
+    for (const resp of responses) {
+      if (resp.separator) res.write(resp.separator)
+      resp.txt.forEach(txt => {
+        res.write(`--${header}${CRLF}`)
+        res.write(`${txt}`)
+      })
+    }
+  }
+  if (group) res.write(`${CRLF}--${group}--${CRLF}`)
+  // indicates that we need to write a potential separator before the next error response
+  res._writeSeparator = true
+}
+
+const _writeResponseJson = (responses, res) => {
+  for (const resp of responses) {
+    if (resp.separator) res.write(resp.separator)
+    resp.txt.forEach(txt => res.write(txt))
+  }
+}
+
+let error_mws
+const _getNextForLookalike = lookalike => {
+  error_mws ??= cds.middlewares.after.filter(mw => mw.length === 4) // error middleware has 4 params
+  return err => {
+    let _err = err
+    let _next_called
+    const _next = e => {
+      _err = e
+      _next_called = true
+    }
+    for (const mw of error_mws) {
+      _next_called = false
+      mw(_err, lookalike.req, lookalike.res, _next)
+      if (!_next_called) break //> next chain was interrupted -> done
+    }
+    if (_next_called) {
+      // here, final error middleware called next (which actually shouldn't happen!)
+      if (_err.statusCode) lookalike.res.status(_err.statusCode)
+      if (typeof _err === 'object') lookalike.res.json({ error: _err })
+      else lookalike.res.send(_err)
+    }
+  }
+}
+
+// REVISIT: This looks frightening -> need to review
+const _transaction = async srv => {
   return new Promise(res => {
     const ret = {}
-    srv.tx(
+    const _tx = (ret._tx = srv.tx(
       async () =>
         (ret.promise = new Promise((resolve, reject) => {
           const proms = []
-          ret.add = AsyncResource.bind(function (request, req, res) {
-            const lookalike = _createExpressReqResLookalike(request, req, res)
-            router.handle(lookalike.req, lookalike.res)
-            request.promise = lookalike.promise
-            proms.push(request.promise)
-            return request.promise
+          // It's important to run `makePromise` in the current execution context (cb of srv.tx),
+          // otherwise, it will use a different transaction.
+          // REVISIT: This looks frightening -> need to review
+          ret.add = AsyncResource.bind(function (makePromise) {
+            const p = makePromise()
+            proms.push(p)
+            return p
           })
-          ret.done = function () {
-            return Promise.allSettled(proms).then(resolve, reject)
+          ret.done = async function () {
+            const result = await Promise.allSettled(proms)
+            if (result.some(r => r.status === 'rejected')) {
+              reject()
+              // REVISIT: workaround to wait for commit/rollback
+              await _tx
+              return 'rejected'
+            }
+            resolve(result)
+            // REVISIT: workaround to wait for commit/rollback
+            await _tx
           }
           res(ret)
         }))
-    )
+    ))
   })
 }
 
+const _tx_done = async (tx, responses, isJson) => {
+  let rejected
+  try {
+    rejected = await tx.done()
+  } catch (e) {
+    // here, the commit was rejected even though all requests were successful (e.g., by custom handler or db consistency check)
+    rejected = 'rejected'
+    // construct commit error
+    let statusCode = e.statusCode || e.status || (e.code && Number(e.code))
+    if (isNaN(statusCode)) statusCode = 500
+    const code = String(e.code || statusCode)
+    const message = e.message || 'Internal Server Error'
+    const error = { error: { ...e, code, message } }
+    // replace all responses with commit error
+    for (const res of responses) {
+      res.status = 'fail'
+      // REVISIT: should error go through any error middleware/ customization logic?
+      if (isJson) {
+        let txt = ''
+        for (let i = 0; i < res.txt.length; i++) txt += Buffer.isBuffer(res.txt[i]) ? res.txt[i].toString() : res.txt[i]
+        txt = JSON.parse(txt)
+        txt.status = statusCode
+        txt.body = error
+        // REVISIT: content-length needed? not there in multipart case...
+        delete txt.headers['content-length']
+        res.txt = [JSON.stringify(txt)]
+      } else {
+        let txt = res.txt[0]
+        txt = txt.replace(/HTTP\/1\.1 \d\d\d \w+/, `HTTP/1.1 ${statusCode} ${message}`)
+        txt = txt.split(/\r\n/)
+        txt.splice(-1, 1, JSON.stringify(error))
+        txt = txt.join('\r\n')
+        res.txt = [txt]
+      }
+    }
+  }
+  return rejected
+}
+
 const _processBatch = async (srv, router, req, res, next, body, ct, boundary) => {
   body ??= req.body
   ct ??= 'JSON'
-  const isJson = ct === 'JSON'
+  // respond with requested content type (i.e., accept) with fallback to the content type used in the request
+  let isJson = ct === 'JSON'
+  if (req.headers.accept) {
+    if (req.headers.accept.indexOf('multipart/mixed') > -1) isJson = false
+    else if (req.headers.accept.indexOf('application/json') > -1) isJson = true
+  }
   const _formatResponse = isJson ? _formatResponseJson : _formatResponseMultipart
 
+  // continue-on-error defaults to true in json batch
+  let continue_on_error = req.headers.prefer?.match(/odata\.continue-on-error(=(\w+))?/)
+  if (!continue_on_error) {
+    continue_on_error = isJson ? true : false
+  } else {
+    continue_on_error = continue_on_error[2] === 'false' ? false : true
+  }
+
   try {
     const ids = _validateBatch(body) // REVISIT: we will not be able to validate the whole once we stream
 
@@ -183,41 +322,130 @@ const _processBatch = async (srv, router, req, res, next, body, ct, boundary) =>
     let previousAtomicityGroup
     let separator
     let tx
-
-    res.setHeader('content-type', CT[ct] + (!isJson ? ';boundary=' + boundary : ''))
-    res.setHeader('OData-Version', '4.0') //> REVISIT: Fiori/ UI5 wants this
-    res.status(200)
-    res.write(isJson ? '{"responses":[' : '')
+    let responses
+
+    // IMPORTANT: Avoid sending headers and responses too eagerly, as we might still have to send a 401
+    let sendPreludeOnce = () => {
+      res.setHeader('Content-Type', isJson ? CT.JSON : CT.MULTIPART + ';boundary=' + boundary)
+      res.status(200)
+      res.write(isJson ? '{"responses":[' : '')
+      sendPreludeOnce = () => {} //> only once
+    }
 
     const { requests } = body
     for await (const request of requests) {
+      // for json payloads, normalize headers to lowercase
+      if (ct === 'JSON') {
+        request.headers = request.headers
+          ? Object.keys(request.headers).reduce((acc, cur) => {
+              acc[cur.toLowerCase()] = request.headers[cur]
+              return acc
+            }, {})
+          : {}
+      }
+
       const { atomicityGroup } = request
 
       if (!atomicityGroup || atomicityGroup !== previousAtomicityGroup) {
-        if (tx) await tx.done()
-        tx = await _transaction(srv, router)
-        if (atomicityGroup) ids[atomicityGroup].promise = tx.promise
-      }
+        if (tx) {
+          // Each change in `atomicityGroup` results in a new transaction. We execute them in sequence to avoid too many database connections.
+          // In the future, we might make this configurable (e.g. allow X parallel connections per HTTP request).
+          const rejected = await _tx_done(tx, responses, isJson)
+          if (tx.failed?.res.statusCode === 401 && req._login) return req._login()
+          else sendPreludeOnce()
+          isJson
+            ? _writeResponseJson(responses, res)
+            : _writeResponseMultipart(responses, res, rejected, previousAtomicityGroup, boundary)
+          if (rejected && !continue_on_error) {
+            tx = null
+            break
+          }
+        }
 
-      const dependencies = request.dependsOn?.map(id => ids[id].promise)
-      if (dependencies) {
-        // TODO: fail the dependent request if dependency fails
-        await Promise.allSettled(dependencies)
+        responses = []
+        tx = await _transaction(srv)
+        if (atomicityGroup) ids[atomicityGroup].promise = tx._tx
       }
 
-      tx.add(request, req, res).then(request => {
-        if (separator) res.write(separator)
-        else separator = isJson ? Buffer.from(',') : Buffer.from(CRLF)
-        _formatResponse(request, res, boundary)
-      })
+      tx.add(() => {
+        return (request.promise = (async () => {
+          const dependencies = request.dependsOn?.filter(id => id !== request.atomicityGroup).map(id => ids[id].promise)
+          if (dependencies) {
+            // first, wait for dependencies
+            const results = await Promise.allSettled(dependencies)
+            const dependendOnFailed = results.some(({ status }) => status === 'rejected')
+            if (dependendOnFailed) {
+              tx.id = request.id
+              tx.res = {
+                getHeaders: () => {},
+                statusCode: 424,
+                _chunk: JSON.stringify({
+                  code: '424',
+                  message: 'Failed Dependency'
+                })
+              }
+              throw tx
+            }
+
+            const dependsOnId = request.url.split('/')[0].replace(/^\$/, '')
+            if (dependsOnId in ids) {
+              const dependentResult = results.find(r => r.value.id === dependsOnId)
+              const dependentOnUrl = dependentResult.value.req.originalUrl
+              const dependentOnResult = JSON.parse(dependentResult.value.res._chunk)
+              const recentUrl = request.url
+              const cqn = cds.odata.parse(dependentOnUrl, { service: srv, baseUrl: req.baseUrl, strict: true })
+              const { target } = cqn
+              const keyString =
+                '(' +
+                target.keys
+                  .filter(k => !k.isAssociation)
+                  .map(k => {
+                    let v = dependentOnResult[k.name]
+                    if (typeof v === 'string' && k._type !== 'cds.UUID') v = `'${v}'`
+                    return k.name + '=' + v
+                  })
+                  .join(',') +
+                ')'
+              request.url = recentUrl.replace(`$${dependsOnId}`, dependentOnUrl + keyString)
+            }
+          }
 
-      if (!atomicityGroup) tx.done()
+          // REVIST: That sends each request through the whole middleware chain again and again, including authentication and authorization.
+          // -> We should optimize this!
+          const lookalike = _createExpressReqResLookalike(request, req, res)
+          const lookalike_next = _getNextForLookalike(lookalike)
+          router.handle(lookalike.req, lookalike.res, lookalike_next)
+          return lookalike.promise
+        })())
+      })
+        .then(req => {
+          const resp = { status: 'ok' }
+          if (separator) resp.separator = separator
+          else separator = isJson ? Buffer.from(',') : Buffer.from(CRLF)
+          resp.txt = _formatResponse(req, atomicityGroup)
+          responses.push(resp)
+        })
+        .catch(failedReq => {
+          const resp = { status: 'fail' }
+          if (separator) resp.separator = separator
+          else separator = isJson ? Buffer.from(',') : Buffer.from(CRLF)
+          resp.txt = _formatResponse(failedReq, atomicityGroup)
+          tx.failed = failedReq
+          responses.push(resp)
+        })
 
       previousAtomicityGroup = atomicityGroup
     }
 
-    if (tx) await tx.done()
-
+    if (tx) {
+      // The last open transaction must be finished
+      const rejected = await _tx_done(tx, responses, isJson)
+      if (tx.failed?.res.statusCode === 401 && req._login) return req._login()
+      else sendPreludeOnce()
+      isJson
+        ? _writeResponseJson(responses, res)
+        : _writeResponseMultipart(responses, res, rejected, previousAtomicityGroup, boundary)
+    } else sendPreludeOnce()
     res.write(isJson ? ']}' : `${CRLF}--${boundary}--${CRLF}`)
     res.end()
 
@@ -231,8 +459,8 @@ const _processBatch = async (srv, router, req, res, next, body, ct, boundary) =>
  * multipart/mixed
  */
 
-const _multipartBatch = (srv, router) => async (req, res, next) => {
-  const boundary = req.headers['content-type']?.match(/boundary=([\w_-]+)/i)?.[1]
+const _multipartBatch = async (srv, router, req, res, next) => {
+  const boundary = getBoundary(req)
   if (!boundary) return next(cds.error('No boundary found in Content-Type header', { code: 400 }))
 
   try {
@@ -243,20 +471,22 @@ const _multipartBatch = (srv, router) => async (req, res, next) => {
   }
 }
 
-const _formatResponseMultipart = (request, res, boundary) => {
-  const { /* id, */ res: response } = request
+const _formatResponseMultipart = request => {
+  const { res: response } = request
+  const content_id = request.req?.headers['content-id']
 
-  let txt = `--${boundary}${CRLF}content-type: application/http${CRLF}content-transfer-encoding: binary${CRLF}${CRLF}`
+  let txt = `content-type: application/http${CRLF}content-transfer-encoding: binary${CRLF}`
+  if (content_id) txt += `content-id: ${content_id}${CRLF}`
+  txt += CRLF
   txt += `HTTP/1.1 ${response.statusCode} ${STATUS_CODES[response.statusCode]}${CRLF}`
 
   // REVISIT: tests require specific sequence
   const headers = {
-    'odata-version': '4.0',
-    'content-type': 'DUMMY',
-    ...response.getHeaders()
+    ...response.getHeaders(),
+    'content-type': 'application/json;odata.metadata=minimal' //> REVISIT: expected by tests
   }
-  headers['content-type'] = 'application/json;odata.metadata=minimal' //> REVISIT: expected by tests
   delete headers['content-length'] //> REVISIT: expected by tests
+
   for (const key in headers) {
     txt += key + ': ' + headers[key] + CRLF
   }
@@ -271,7 +501,7 @@ const _formatResponseMultipart = (request, res, boundary) => {
       let meta = [],
         data = []
       for (const [k, v] of Object.entries(_json)) {
-        if (k.startsWith('@')) meta.push(`"${k}":"${v.replaceAll('"', '\\"')}"`)
+        if (k.startsWith('@')) meta.push(`"${k}":${typeof v === 'string' ? `"${v.replaceAll('"', '\\"')}"` : v}`)
         else data.push(JSON.stringify({ [k]: v }).slice(1, -1))
       }
       const _json_as_txt = '{' + meta.join(',') + (meta.length && data.length ? ',' : '') + data.join(',') + '}'
@@ -283,7 +513,7 @@ const _formatResponseMultipart = (request, res, boundary) => {
     }
   }
 
-  res.write(txt)
+  return [txt]
 }
 
 /*
@@ -296,46 +526,52 @@ const _formatStatics = {
   close: Buffer.from('}')
 }
 
-const _formatResponseJson = (request, res) => {
+const _formatResponseJson = (request, atomicityGroup) => {
   const { id, res: response } = request
-  const raw = Buffer.from(
-    JSON.stringify({
-      id,
-      status: response.statusCode,
-      headers: {
-        'odata-version': '4.0',
-        'content-type': 'application/json',
-        ...response.getHeaders()
-      }
-    })
-  )
+
+  const chunk = {
+    id,
+    status: response.statusCode,
+    headers: {
+      ...response.getHeaders(),
+      'content-type': 'application/json' //> REVISIT: why?
+    }
+  }
+  if (atomicityGroup) chunk.atomicityGroup = atomicityGroup
+  const raw = Buffer.from(JSON.stringify(chunk))
+
+  // body?
+  if (!response._chunk) return [raw]
+
   // change last "}" into ","
   raw[raw.byteLength - 1] = _formatStatics.comma
-  res.write(raw)
-  res.write(_formatStatics.body)
-  res.write(response._chunk)
-  res.write(_formatStatics.close)
+  return [raw, _formatStatics.body, response._chunk, _formatStatics.close]
 }
 
-const _jsonBatch = (srv, router) => (req, res, next) => {
-  _processBatch(srv, router, req, res, next)
-}
+/*
+ * exports
+ */
+
+module.exports = adapter => {
+  const { options: config, router, service } = adapter
+  const { max_content_length } = config //> max_content_length is unofficial config
 
-module.exports = (srv, router) => {
-  const handleJsonBatch = _jsonBatch(srv, router)
-  const handleMultipartBatch = _multipartBatch(srv, router)
+  const options = { type: '*/*' }
+  if (max_content_length) options.limit = max_content_length
+  const textBodyParser = express.text(options)
 
-  return function batch(req, res, next) {
+  return function odata_batch(req, res, next) {
     if (req.headers['content-type'].includes('application/json')) {
-      return handleJsonBatch(req, res, next)
+      return _processBatch(service, router, req, res, next)
     }
 
     if (req.headers['content-type'].includes('multipart/mixed')) {
-      return express.text({ type: '*/*' })(req, res, () => {
-        handleMultipartBatch(req, res, next)
+      return textBodyParser(req, res, function odata_batch_next(err) {
+        if (err) return next(err)
+        return _multipartBatch(service, router, req, res, next)
       })
     }
 
-    throw cds.error('Batch requests must have content type multipart/mixed or application/json', { code: 400 })
+    throw cds.error('Batch requests must have content type multipart/mixed or application/json', { statusCode: 400 })
   }
 }

package/libx/odata/middleware/body-parser.js

@@ -0,0 +1,33 @@
+const express = require('express')
+
+// basically express.json() with string representation of body stored in req._raw for recovery
+// REVISIT: why do we need our own body parser? Only because of req._raw?
+module.exports = function bodyParser4(adapter, options = {}) {
+  if (!options.type) options.type = 'json'
+  const { max_content_length } = adapter.options //> max_content_length is unofficial config
+  if (!options.limit && max_content_length) options.limit = max_content_length
+  const textParser = express.text(options)
+  return function http_body_parser(req, res, next) {
+    if (typeof req.body === 'object') {
+      //> body already deserialized (e.g., batch subrequest or custom body parser)
+      if (!req._raw) req._raw = JSON.stringify(req.body) //> ensure req._raw is set
+      return next()
+    }
+    textParser(req, res, function http_body_parser_next(err) {
+      if (err) return next(Object.assign(err, { statusCode: 400 }))
+      if (typeof req.body !== 'string') return next()
+
+      req._raw = req.body || '{}'
+      try {
+        req.body = JSON.parse(req._raw)
+        if (typeof req.body !== 'object') {
+          res.status(400).json({ error: { message: 'Expected JSON object in body', statusCode: 400, code: '400' } })
+          return
+        }
+      } catch (e) {
+        return next(Object.assign(e, { statusCode: 400 }))
+      }
+      next()
+    })
+  }
+}