@sap/cds-dk 9.9.2 → 10.0.3

package/node_modules/@sap/cds-mtxs/srv/extensibility-service.js

@@ -0,0 +1,550 @@
+const express = require('express')
+const cds = require('@sap/cds/lib')
+const { SELECT, DELETE } = cds.ql
+const { fs, path, tar, rimraf, read, readdir, mkdirp, write } = cds.utils
+const { token: authToken } = require('../lib/utils')
+const linter = require('./extensibility/linter')
+const { getMigratedProjects } = require('../lib/migration/migration')
+
+
+const BUILT_IN_NAMESPACES = ['cds.core', 'cds.outbox', 'cds.xt'] // REVISIT: move to a more central place to be consumed also by linter
+const TOMBSTONE_ID = '__tombstone'
+const TEMP_DIR = fs.realpathSync(require('os').tmpdir())
+const LOG = cds.log('mtx')
+const DEBUG = cds.debug('mtx')
+const production = process.env.NODE_ENV === 'production'
+
+const ACTIVE = 2, DRAFT = 1, INACTIVE = 0
+
+// determine appid for separation of extensions
+//const packageJson = require(main.root + '/package.json');
+const APPID = require('../lib/utils').getAppId()
+
+const _async = () => {
+  return cds.context.http?.req?.headers?.prefer === 'respond-async'
+}
+
+const _includeExt = () => {
+  return (cds.env.requires['cds.xt.ExtensibilityService']?.['check-existing-extensions'] !== false)
+}
+class DbAdapter {
+
+  static activated(status) {
+    return ['inactive', 'draft', 'database'][status]
+  }
+
+  static status(activated) {
+    const level = {
+      inactive: INACTIVE,
+      draft: DRAFT,
+      database: ACTIVE
+    }
+    return level[activated]
+  }
+
+  // compatible put
+  static async put(ext) {
+
+    // status 'active' is only set by the activation later
+    if (ext.status === ACTIVE) {
+      throw Error('Internal error: direct activation')
+    }
+
+    // TODO must be urgently changed in favor of additional database field for csvs
+    if (ext.csvs && !ext.archive) {
+      const temp = await fs.promises.mkdtemp(`${TEMP_DIR}${path.sep}extension-`)
+      try {
+        const root = await mkdirp(path.join(temp, 'data'))
+        for (const csv in ext.csvs) {
+          await write(path.join(root, csv), ext.csvs[csv])
+        }
+        ext.archive = await tar.cz(temp)
+      } finally {
+        await rimraf(temp)
+      }
+    }
+
+    if (ext.csvs && Object.keys(ext.csvs).length > 0 && cds.env.requires['cds.xt.ExtensibilityService']?.activate?.['skip-csv']) throw Error('CSV deployment is disabled')
+
+    // insert extension
+    const tenant = ext.tenant
+    const ID = cds.utils.uuid()
+    const csn = JSON.stringify(ext.csn)
+    const i18n = ext.i18n ? JSON.stringify(ext.i18n) : null
+    const tag = ext.ID
+    await cds.tx({ tenant, user: new cds.User.Privileged() }, async tx => {
+      // remove current inactive extension with tag
+      // use native table names to avoid issues with missing appid
+      if (tag) await tx.run(DELETE.from('cds_xt_Extensions').where({ tag, activated: { '!=': this.activated(ACTIVE) } }))
+      try {
+        await tx.run(INSERT.into('cds_xt_Extensions').entries({ ID, csn, i18n, sources: ext.archive ?? null, activated: this.activated(ext.status), tag, appid: ext.appid ?? APPID, timestamp: new Date().toISOString() }))
+      } catch (error) {
+        DEBUG?.('Trying legacy request without appid', error)
+        await tx.run(INSERT.into('cds_xt_Extensions').entries({ ID, csn, i18n, sources: ext.archive ?? null, activated: this.activated(ext.status), tag, timestamp: new Date().toISOString() }))
+      }
+    })
+    return ID
+  }
+
+  static async delete(tag) {
+    const ID = cds.utils.uuid()
+    const result = !tag ? await DELETE.from('cds.xt.Extensions') : await DELETE.from('cds.xt.Extensions').where({ tag })
+    // leave tombstone for deployment - ID cannot be used in case of all extensions (no ID passed)
+    await DELETE.from('cds.xt.Extensions').where({ tag: TOMBSTONE_ID }) // cleanup
+    await INSERT.into('cds.xt.Extensions').entries({ ID, csn: '{}', i18n: null, sources: null, activated: null, tag: TOMBSTONE_ID })
+    return result
+  }
+}
+
+module.exports = class ExtensibilityService extends cds.ApplicationService {
+
+  async init() {
+
+    cds.on('serving:cds.xt.JobsService', js => {
+      js.on('taskUpdate', async (msg) => {
+        if (msg.data?.task?.service === 'cds.xt.ExtensibilityService' && msg.data?.task?.status === 'FINISHED') { // Only filter for service. There is only one operation that triggers jobs
+          await this.emit(
+            'activated',
+            { tenant: msg.tenant, job_ID: msg.data.task.job_ID, args: msg.data.task.args, status: msg.data.task.status, error: msg.data.task.error },
+            { 'x-tenant-id': msg.tenant }
+          )
+        }
+      })
+    })
+
+    // import compatibility handlers
+    require('./extensibility/compat/update')(this)
+
+    this.before('push', async req => {
+      let { extension } = req.data
+      if (!req.user.is('internal-user') && req.data.tenant && req.data.tenant !== req.tenant)
+        req.reject(403, `No permission to push extensions to tenants other than ${production ? 'the user\'s tenant' : req.tenant}`)
+      const tenant = (req.user.is('internal-user') && req.data.tenant) || req.tenant // TODO: revisit magic
+      if (tenant) cds.context = { tenant }
+
+      if (!extension) req.reject(400, 'Missing extension')
+
+      const { root, archive } = await _unpackExtension(extension, tenant) // REVISIT part of handler contract?
+      req.data.sources = root
+      req.data.archive = archive
+    })
+
+    // REVISIT - change to after handler or rely on handler cooperation with the right use of next()
+    this.on('push', async req => {
+      const { sources } = req.data
+      req.data.csn = await _readExtension(req, sources)
+      req.data.csvs = await _readInitialData(sources)
+      req.data.i18n = await _readI18n(sources)
+      req.data.ID = req.data.tag ?? null
+      req.data.status = ACTIVE // immediately activate
+
+      try {
+        const { 'cds.xt.ExtensibilityService': es } = cds.services
+        await es.put(`/Extensions`, req.data)
+      } finally {
+        rimraf(req.data.sources)
+      }
+    })
+
+    async function _unpackExtension(extension) {
+      // REVISIT: string ?
+      const sources = typeof extension === 'string' ? Buffer.from(extension, 'base64') : extension
+      const root = await fs.promises.mkdtemp(`${TEMP_DIR}${path.sep}extension-`)
+      await tar.xvz(sources).to(root)
+      return { root, archive: sources }
+    }
+
+    // retrieves extension from archive
+    async function _readExtension(req, root) {
+      try {
+        const extCsn = JSON.parse(await read(path.join(root, 'extension.csn')))
+        return [JSON.stringify(extCsn)]
+      } catch (e) {
+        if (e.code !== 'ENOENT') throw e
+        req.reject(400, 'Missing or bad extension')
+      }
+    }
+
+    // retrieves i18n from archive
+    async function _readI18n(root) {
+      try { return await read(path.join(root, 'i18n', 'i18n.json')) }
+      catch (e) { if (e.code !== 'ENOENT') throw e }
+    }
+
+    // retrieves initial data from archive
+    async function _readInitialData(root) {
+      const csvs = []
+      try {
+        const dirents = await readdir(path.join(root, 'data'))
+        for (const dirent of dirents) {
+          const basename = path.basename(dirent)
+          csvs.push({ name: basename, content: await read(path.join(root, 'data', dirent)) })
+        }
+      }
+      catch (e) { if (e.code !== 'ENOENT') throw e }
+
+      return csvs
+    }
+
+    this.on(['UPDATE', 'UPSERT'], 'Extensions', async req => {
+      const tenant = _tenant(req)
+      if (tenant) cds.context = { tenant }
+
+      req.data.status = req.data.status ?? ACTIVE // set the default manually as it had to be removed from the model
+      req.data.internalExtensionId = await DbAdapter.put({ ...req.data, status: req.data.status === ACTIVE ? DRAFT : req.data.status })
+    })
+
+    this.on('DELETE', 'Extensions', async req => {
+      const tenant = _tenant(req)
+      if (tenant) cds.context = { tenant }
+
+      // put empty extension instead of deletion
+      req.data.internalExtensionId = await DbAdapter.put({ ID: req.data.ID, tenant: cds.context.tenant, csn: {}, status: DRAFT })
+    })
+
+    // final activation
+    this.after(['UPDATE', 'UPSERT'], 'Extensions', async (results, req) => { // TODO populate sources
+      const async = _async()
+      const { ID, status } = req.data
+
+      // direct activation
+      if (status === ACTIVE) await this.activate(ID, ACTIVE, { _internalExtensionId: req.data.internalExtensionId }) // TODO: what status? How to enforce activation?
+
+      if (async) { // 202 is set by the job service
+        return
+      }
+      const res = await _fetchExtensions(req.data.ID)
+      req.results = { ID: res?.tag, csn: res?.csn, i18n: res?.i18n !== '{}' ? res?.i18n : undefined, timestamp: res?.timestamp, appid: res?.appid, status: DbAdapter.status(res?.activated) } // REVISIT: return csvs?
+    })
+
+    this.after('DELETE', 'Extensions', async (results, req) => {
+      const async = _async(req)
+      const { ID } = req.data
+
+      // delegate to activate action
+      if (!req.http.req.query.status || req.http.req.query?.status === ACTIVE)
+        await this.activate(ID, ACTIVE, { _internalExtensionId: req.data.internalExtensionId })
+      if (async) {
+        return
+      }
+      return results
+    })
+
+    // difference to direct activation: no insert of extensions into db
+    this.on('activate', async req => {
+      const { ID: tag, status = ACTIVE, options } = req.data
+      const tenant = _tenant(req)
+      if (tenant) cds.context = { tenant }
+
+      // TODO: check if level promotion is allowed
+      // TODO get full csn from model provider for other handlers? Or leave it to them to run the model provider?
+
+      const async = _async()
+
+      const internalEntryId = req.data.options?._internalExtensionId
+      try {
+        const js = await cds.connect.to('cds.xt.JobsService')
+
+        async function cleanup(tag, internalEntryId) {
+          if (internalEntryId) { // cleanup only for direct activation
+            await cds.tx({ tenant, user: new cds.User.Privileged() }, async () => DELETE.from('cds.xt.Extensions').where({ tag, ID: internalEntryId }))
+          }
+        }
+
+        const postProcess = async (error) => {
+          if (error) {
+            await cleanup(tag, internalEntryId)
+          }
+        }
+
+        // eslint-disable-next-line no-async-promise-executor
+        return await new Promise(async (resolve, reject) => {
+          const cb = !async ? async error => {
+            if (error) {
+              try {
+                const errorObject = _decodeError(error)
+                await cleanup(tag, internalEntryId)
+                return reject(errorObject)
+              } catch {
+                return reject(error)
+              }
+            }
+            return resolve()
+          } : postProcess
+
+          // separate user needed as job service is protected
+          // protection is necessary because parts of the service are exposed to the outside
+          const tx = js.tx({ tenant: cds.context.tenant, user: new cds.User.Privileged() })
+          const job = await tx.enqueue('cds.xt.ExtensibilityService', 'activateExtension', [new Set([cds.context.tenant])], { tag, status, options }, cb)
+          if (async) {
+            resolve(job)
+          }
+        })
+      } catch (err) {
+        if (err.code === 'ERR_CDS_COMPILATION_FAILURE') req.reject({ status: 422, message: _getCompilerError(err.messages), messages: err.messages })
+        else req.reject(err)
+      }
+    })
+
+    this.on('validate', async req => {
+      let tenant = _tenant(req)
+      if (tenant) cds.context = { tenant }
+      else tenant = cds.context.tenant
+
+      let messages
+      try {
+        LOG.info('validating extension', req.data.ID, '...')
+
+        const { 'cds.xt.ModelProviderService': mps } = cds.services
+        const extFullCsn = await mps.getCsn(tenant, ['*'], undefined, undefined, false, false, { inactive: [req.data.ID]})
+        const extCsn = _includeExt()
+          ? await mps.getExtensions(tenant, { inactive: [req.data.ID]})
+          : await mps.getExtensions(tenant, { inactive: [req.data.ID], compat: true }) // only check current extension
+
+        // verify edmx
+        cds.compile.to.edmx(extFullCsn, { service: 'all', version: cds.env.odata?.version ?? 'v4' })
+
+        messages = linter.lint(extCsn, extFullCsn, cds.env, /* clone */ true)
+      } catch (error) {
+        let validationError = error
+        if (error.code === 'ERR_CDS_COMPILATION_FAILURE') messages = error.messages
+        else throw validationError
+      }
+      return {
+        errors: messages.filter(m => m.severity === 'Error').map(m => ({ message: m.message, severity: m.severity ?? 'Error' })),
+        messages: messages.filter(m => m.severity !== 'Error')
+      }
+    })
+
+    this.on('discard', async req => {
+      const tenant = _tenant(req)
+      if (tenant) cds.context = { tenant }
+      await DELETE.from('cds.xt.Extensions').where({ tag: req.data.ID, activated: { '!=': 'database' } })
+    })
+
+    this.on('set', req => {
+      let { extension, tag, resources, activate = 'database' } = req.data
+      const tenant = (req.user.is('internal-user') && req.data.tenant) || req.tenant || '' // REVISIT: magic
+
+      const { 'cds.xt.ExtensibilityService': es } = cds.services
+
+      return es.put(`/Extensions`,
+        {
+          ID: tag,
+          csn: extension,
+          i18n: resources,
+          csvs: resources,
+          tenant: tenant,
+          status: (activate === 'database') ? ACTIVE : DRAFT
+        })
+
+    })
+
+    this.on('pull', async req => {
+      LOG.info('pulling latest model for tenant', req.tenant)
+      const { tag } = req.data
+      const { 'cds.xt.ModelProviderService': mps } = cds.services
+      const csn = await mps.getCsn({
+        tenant: req.tenant,
+        toggles: cds.context.features, // with all enabled feature extensions
+        base: !_includeExt(), // without any custom extensions
+        flavor: 'xtended',
+        for: 'pull',
+        options: {
+          skipExt: [tag]
+        }
+      })
+      // filter system entries
+      // filter @impl from csn - danger, must be cloned as it comes from the cache
+      const extProjectBaseCsn = cds.clone(csn)
+      for (const key of Object.keys(extProjectBaseCsn.definitions)) {
+        for (const namespace of BUILT_IN_NAMESPACES) {
+          if (key.startsWith(`${namespace}.`)) {
+            delete extProjectBaseCsn.definitions[key]
+            break
+          }
+          const def = extProjectBaseCsn.definitions[key]
+          if (def) delete def['@impl']
+        }
+      }
+      req.http.res?.set('content-type', 'application/octet-stream; charset=binary')
+      const temp = await fs.promises.mkdtemp(`${TEMP_DIR}${path.sep}extension-`)
+      try {
+        await fs.promises.writeFile(path.join(temp, 'index.csn'), cds.compile.to.json(extProjectBaseCsn))
+        const config = linter.configCopyFrom(cds.env)
+        await fs.promises.writeFile(path.join(temp, '.cdsrc.json'), JSON.stringify(config, null, 2))
+        return await tar.cz(temp)
+      } finally {
+        rimraf(temp)
+      }
+    })
+
+    async function _fetchExtensions(ID) {
+      try {
+        return !ID ? await SELECT.from('cds.xt.Extensions') : await SELECT.one.from('cds.xt.Extensions').where({ tag: ID })
+      } catch (error) {
+        DEBUG?.('Trying legacy request without appid', error)
+        const fields = ['ID', 'csn', 'tag', 'activated', 'timestamp', 'i18n']
+        return !ID ? await SELECT(fields).from('cds.xt.Extensions') : await SELECT(fields).one.from('cds.xt.Extensions').where({ tag: ID })
+      }
+    }
+
+    // TODO read csvs -> will fail because csvs are not yet stored
+    this.on('READ', 'Extensions', async req => {
+      const tenant = _tenant(req)
+      if (tenant) cds.context = { tenant }
+      const ext = await _fetchExtensions(req.data?.ID)
+      if (Array.isArray(ext)) return ext.filter(e => e.tag !== TOMBSTONE_ID).map(e => ({ ID: e.tag, csn: e.csn, i18n: e.i18n !== '{}' ? e.i18n : undefined, timestamp: e.timestamp, status: DbAdapter.status(e.activated), appid: e.appid })) // REVISIT: csvs?
+      if (ext) return { ID: ext.tag, csn: ext.csn, i18n: ext.i18n !== '{}' ? ext.i18n : undefined, timestamp: ext.timestamp, status: DbAdapter.status(ext.activated),  appid: ext.appid } // REVISIT: csvs?
+      return ext
+
+
+    })
+
+    this.on('getMigratedProjects', req => {
+      const { tagRule, defaultTag } = req.data
+      const tenant = req.tenant // REVISIT: check if access for arbitrary tenants needed
+      if (!tenant)  req.reject(401, 'User not assigned to any tenant')
+      return getMigratedProjects(req, tagRule || undefined, defaultTag || undefined, tenant)
+    })
+
+    cds.on('served', () => {
+      cds.app?.post('/-/cds/login/token', express.urlencoded({ extended: false }), async (req, res) => {
+        try {
+          if (req.method === 'HEAD') return res.status(204).send()
+
+          const cred = cds.env.requires.auth.credentials || {}
+          if (!cred.clientid) throw cds.error('missing authentication service binding', { status: 500 })
+
+          const query = req.method === 'POST' ? (req.body || {}) : (req.query || {})
+
+          const form = query.refresh_token ? { grant_type: 'refresh_token', refresh_token: query.refresh_token } :
+                       query.passcode      ? { grant_type: 'password', passcode: query.passcode } : {}
+          form.scope = `${cred.xsappname}.cds.ExtensionDeveloper`
+          const raw = await authToken(cred, { form, subdomain: query.subdomain })
+          const data = JSON.parse(raw)
+          if (data.error) return res.status(401).json(data)
+          res.json(data)
+        } catch (err) {
+          const status = err.status || err.statusCode || 500
+          res.status(status).json({
+            error: err.message,
+            ...(err.error_description && { error_description: err.error_description })
+          })
+        }
+      })
+
+      cds.app?.get('/-/cds/login/authorization-metadata', authMeta)
+    })
+
+    const _tenant = function (req) {
+      const tenant = req.headers['x-tenant-id']?? req.data.tenant // the latter can be provided by 'set' when delegating to 'UPDATE'
+      if (!req.user.is('internal-user') && tenant && tenant !== req.tenant)
+        req.reject(403, `No permission to activate extensions by tenants other than ${req.tenant}`)
+      return tenant
+    }
+
+    return super.init()
+  }
+
+  // REMEMBER: changing extension table schema will potentially require migration
+  // REMEMBER: async execution -> ensure correct transaction handling
+  // REMEMBER: async execution -> avoid using cds.context.tenant
+  /**
+   * Final activation called by the job
+   */
+  async activateExtension(tenant, tag, status, options = {}) {
+    let extEntry
+    try {
+      // TODO: handle '*'?
+      await cds.tx(async () => {
+        // use native table names to avoid issues with missing appid
+        extEntry = await SELECT(['ID','csn', 'tag','activated']).one.from('cds.xt.Extensions').where({ tag, activated: { '!=': 'database' } }).forUpdate() // select only entries with activated != 'database'
+        if (!extEntry) {
+          LOG.info('Extension', tag, 'not found or already activated') // REVISIT: in case of parallel activation, any entry for 'tag' is used
+          return
+        }
+
+        // validate extension before activation
+        // TODO: uses cds.context.tenant -> potentially dangerous in async jobs
+        // -> REVISIT: Why not always check inactive extensions before activation?
+        const validationResult = await this.validate(tag)
+        if (validationResult.errors.length > 0) {
+          const findings = [...validationResult.errors, ...validationResult.messages]
+          const text = findings.length === 1 ? 'finding' : 'findings'
+          let message = `Validation for tag '${tag}' failed with ${findings.length} ${text}:\n\n`
+          message += findings.map(f => '  - ' + f.message).join('\n') + '\n'
+          throw new cds.error({ message, status: 422 })
+        }
+
+        const deleted = extEntry.csn === '{}' // REVISIT mark differently?
+        await DELETE.from('cds.xt.Extensions').where({ tag, activated: 'database' }) // remove current extension with tag
+        if (deleted) await DbAdapter.delete(tag) // delete entries and add tombstone
+        else await UPDATE('cds.xt.Extensions', { tag, ID: extEntry.ID }).with({ activated: DbAdapter.activated(status) }) // update status
+
+        // really activate extension
+        if (status === ACTIVE && !(cds.env.requires['cds.xt.ExtensibilityService']?.activate?.['skip-db'] === true)) {
+          const { 'cds.xt.DeploymentService': ds } = cds.services
+          if (!ds) {
+            LOG.info('database activation skipped for extension', tag, '- no deployment service')
+            return
+          }
+          LOG.info('activating extension', tag, '...')
+          await ds.extend(tenant, options)
+        }
+      })
+    } catch (error) {
+      let activationError = error
+      // needs to be serialized because it is stored in the db by the job service - TODO check for HDI error somehow?
+      if (!error.code && !error.status) activationError = { status: 422, message: error.message } // for HDI errors
+      throw new Error(_encodeError(activationError), { cause: error })
+    }
+  }
+}
+
+function _passcodeUrl(c, sub) {
+  const u = new URL(c.url)
+  if (sub) u.hostname = u.hostname.replace(/^[^.]+/, sub)
+  u.pathname = '/passcode'
+  return u.toString()
+}
+
+function authMeta(req, res) {
+  const cred = cds.env.requires?.auth?.credentials || {}
+  res.send({ passcode_url: _passcodeUrl(cred, req.query?.subdomain) })
+}
+
+const _getCompilerError = messages => {
+  const defaultMsg = 'Error while compiling extension'
+  if (!messages) return defaultMsg
+  for (const msg of messages) {
+    if (msg.severity === 'Error') return msg.message
+  }
+  return defaultMsg
+}
+
+const _encodeError = (activationError) => {
+  if (activationError.messages) {
+    // cut out first 10 elements of the messages array to avoid db problems
+    activationError.messages = activationError.messages.slice(0, 10)
+  }
+  return JSON.stringify(activationError, (element, value) => {
+    if (element === 'message') return JSON.stringify(value)
+    return value
+  })
+}
+
+const _decodeError = (activationError) => {
+  return JSON.parse(activationError, (element, value) => {
+    if (element === 'message') {
+      try {
+        return JSON.parse(value)
+      } catch (e) {
+        // in case of problems with parsing because of HANA bug
+        LOG.error('Failed to parse error. Please use at least HANA database version (2024-26)', e)
+        return value
+      }
+    }
+    return value
+  })
+}

package/node_modules/@sap/cds-mtxs/srv/jobs-service.cds

@@ -0,0 +1,37 @@
+using from '../db/t0';
+
+@path     : '/-/cds/jobs'
+@protocol : 'rest'
+@impl     : '@sap/cds-mtxs/srv/jobs-service.js'
+service cds.xt.JobsService {
+
+  type Set: {}
+
+  @open @requires: ['internal-user', 'cds.Subscriber', 'mtcallback']
+  action enqueue(
+    service: String,
+    op: String,
+    @cds.validate: false clusters: array of Set,
+    @cds.validate: false args: array of {},
+    @cds.validate: false callback: {}
+  ) returns Jobs;
+
+  @open function pollJob(ID: String) returns {};
+  @open function pollTask(ID: String) returns {};
+
+  @cds.persistence.exists
+  @readonly entity Jobs as projection on cds.xt.Jobs;
+
+  @cds.persistence.exists
+  @readonly entity Tasks as projection on cds.xt.Tasks;
+
+  // event jobUpdate { job: {} };
+  // event taskUpdate { task: {} };
+
+  @private @requires: ['internal-user', 'cds.Developer'] 
+  function config() returns {};
+}
+
+annotate cds.xt.Jobs with @cds.persistence.exists;
+annotate cds.xt.Tasks with @cds.persistence.exists;
+annotate cds.xt.Tenants with @cds.persistence.exists;