@sap/cds-dk 9.9.3 → 10.0.4

package/node_modules/@sap/cds-mtxs/srv/jobs-service.js

@@ -0,0 +1,346 @@
+const { inspect } = require('util')
+const cds = require('@sap/cds'), { uuid } = cds.utils
+const main = require('./config')
+const LOG = cds.log('mtx'), DEBUG = cds.debug('mtx')
+
+const Jobs = 'cds.xt.Jobs', Tasks = 'cds.xt.Tasks'
+
+const { t0_ } = require('../lib/utils')
+
+const {
+  queueSize = 100, clusterSize = 3, workerSize = 4, poolSize = 1
+} = {
+  ...(cds.env.requires.multitenancy?.jobs ?? {}),
+  ...(cds.env.requires['cds.xt.SaasProvisioningService']?.jobs ?? {}),
+  ...(cds.env.requires['cds.xt.SmsProvisioningService']?.jobs ?? {})
+}
+
+const RUNNING = 'RUNNING', FINISHED = 'FINISHED', FAILED = 'FAILED', QUEUED = 'QUEUED', TIMEOUT = 'TIMEOUT'
+
+// A queue, implemented as a circular buffer for O(1) insert + delete
+class Queue {
+  constructor(capacity) {
+    this.buffer = new Array(capacity)
+    this.pointer = 0
+    this.size = 0
+    this.capacity = capacity
+    DEBUG?.('initialized tenant operation job queue with capacity', capacity)
+  }
+  enqueue(value) {
+    if (this.size === this.capacity) cds.error('Tenant operation job queue is full. Please try again later.', { status: 429 })
+    this.buffer[(this.pointer + this.size) % this.capacity] = value
+    this.size++
+  }
+  peek() {
+    return this.buffer[this.pointer]
+  }
+  dequeue() {
+    const value = this.buffer[this.pointer]
+    this.pointer = (this.pointer + 1) % this.capacity
+    this.size--
+    return value
+  }
+}
+const jobQueue = new Queue(queueSize)
+let runningJobs = new Map()
+
+module.exports = class JobsService extends cds.ApplicationService {
+
+  async init() {
+    if (cds.requires.multitenancy?.diagnostics) _diagnostics4(this)
+    cds.on('listening', () => _scheduled(this))
+
+    this.on('READ', 'Jobs', async () => {
+      const jobs = await t0_(
+        SELECT.from(Jobs, j => { j.ID, j.service, j.op, j.status, j.error, j.tasks(t => { t.ID, t.status, t.tenant, t.database, t.error } )}).orderBy('createdAt desc')
+      )
+      return jobs.map(job => ({ ...job, tasks: job.tasks.sort((a, b) => a.tenant.localeCompare(b.tenant))}))
+    })
+    this.on('config', () => {
+      const conf = main.requires.extensibility, es = main.requires['cds.xt.ExtensibilityService']
+      const allowlist = 'extension-allowlist', blocklist = 'namespace-blocklist'
+      return {
+        pool: main.requires.db.pool,
+        poolMtx: cds.requires.db.pool,
+        jobs: cds.requires.multitenancy.jobs,
+        extensibility: {
+          tenantCheckInterval: conf?.tenantCheckInterval,
+          [allowlist]: conf?.[allowlist] ?? es?.[allowlist],
+          [blocklist]: conf?.[blocklist] ?? es?.[blocklist]
+        }
+      }
+    })
+
+    return super.init()
+  }
+
+  async enqueue(service, op, clusters, args, onJobDone) {
+    const _inspect = obj => obj && Object.values(obj).filter(Boolean).length > 0 ? inspect(obj, { depth: 5, colors: cds.utils.colors.enabled }) : []
+    const inspected = Object.entries(args).reduce((acc, [k, v]) => {
+      const inspectedValue = _inspect(v)
+      if (Buffer.isBuffer(v)) acc.push(`${k}: <Buffer>`)
+      else if (v?.length || v && typeof v === 'object' && Object.keys(v).length > 0) acc.push(`${k}: ${inspectedValue}`)
+      return acc
+    }, [])
+    const _args = inspected.length ? ['with', ...inspected] : []
+    const _format = clusters => {
+      if (Array.isArray(clusters)) return clusters.flatMap(c => [...c])
+      return Object.fromEntries(Object.entries(clusters).map(([k, v]) => [k, [...v]]))
+    }
+
+    LOG.info('enqueuing', { service, op }, 'for', _format(clusters), ..._args)
+
+    const job_ID = uuid()
+    const job = { ID: job_ID, service, op, args, status: QUEUED }
+    const clustered = Object.entries(clusters).map(([database, cluster]) => {
+      if (database !== '0') return Array.from(cluster).map(tenant => ({ job_ID, ID: uuid(), tenant, service, op, args, database, status: QUEUED }))
+      else return Array.from(cluster).map(tenant => ({ job_ID, ID: uuid(), tenant, service, op, args, status: QUEUED }))
+    })
+    const tasks = clustered.flat()
+    await t0_(async () => {
+      await INSERT.into(Jobs, job)
+      if (tasks.length) {
+        await INSERT.into(Tasks, tasks)
+        job.tasks = tasks
+        await this.emit('jobUpdate', { job })
+      } else {
+        await UPDATE(Jobs, { ID: job_ID }).with({ status: FINISHED })
+        await this.emit('jobUpdate', { job: { ...job, status: FINISHED } })
+      }
+    })
+
+    if (tasks.length) {
+      if (cds.services[service]) {
+        const fn = task => cds.services[service].tx({ tenant: task.tenant }, tx => tx[op](task.tenant, ...Object.values(args)))
+        jobQueue.enqueue({ job_ID, clusters: clustered, fn, onJobDone })
+      } else {
+        jobQueue.enqueue({ job_ID, clusters: clustered, fn: () => {
+          LOG.warn('service', service, 'is not available, skipping job')
+        }, onJobDone })
+      }
+      this.pickJob()
+    }
+
+    const url = process.env.VCAP_APPLICATION ? 'https://' + JSON.parse(process.env.VCAP_APPLICATION).uris?.[0] : cds.server.url
+    cds.context.http?.res.set('Location', `${url}/-/cds/jobs/pollJob(ID='${job_ID}')`)
+    cds.context.http?.res.set('x-job-id', job_ID)
+    const { headers } = cds.context.http?.req ?? {}
+    if (headers?.prefer?.includes('respond-async') || headers?.status_callback || headers?.mtx_status_callback) {
+      cds.context.http.res.status(202)
+    }
+    return {
+      ...job,
+      tenants: Object.fromEntries(tasks.map(task =>
+        [task.tenant, { ...task, job_ID: undefined, tenant: undefined, op: undefined }]
+      )),
+      tasks: Object.fromEntries(tasks.sort((a, b) => a.tenant.localeCompare(b.tenant)).map(task =>
+        [task.tenant, { ...task, job_ID: job.ID, tenant: task.tenant, service: job.service, op: job.op }]
+      ))
+    }
+  }
+
+  async pollJob(ID) {
+    const job = await t0_(
+      SELECT.one.from(Jobs, j => { j.ID, j.service, j.op, j.error, j.status, j.tasks(t => { t.ID, t.status, t.tenant, t.error } )}).where({ ID })
+    )
+    if (!job) cds.error(`No job found for ID ${ID}`, { status: 404 })
+    job.tasks.sort((a, b) => a.tenant.localeCompare(b.tenant)) // REVISIT: Ideally j.tasks supports orderBy
+    job.tenants = Object.fromEntries(job.tasks.map(task => [task.tenant ?? task.TENANT, {
+      status: task.status ?? task.STATUS,
+      error: task.error ?? task.ERROR ?? undefined
+    }]))
+
+    return job
+  }
+
+  async pollTask(ID) {
+    const task = await t0_(SELECT.one.from(Tasks).where({ ID }))
+    if (!task) cds.error(`No task found for ID ${ID}`, { status: 404 })
+    return {
+      status: task.status ?? task.STATUS,
+      op: task.op ?? task.OP,
+      service: task.service ?? task.SERVICE,
+      error: task.error ?? task.ERROR ?? undefined
+    }
+  }
+
+  async limiter(limit, payloads, fn, asTask = false) {
+    const pending = [], all = []
+    for (const payload of payloads) {
+      const { ID, tenant, job_ID, service } = payload
+      if (asTask) {
+        await t0_(async () => {
+          await UPDATE(Tasks, { ID, tenant }).with({ status: RUNNING })
+          await this.emit('taskUpdate', { task: { ID, job_ID, tenant, service, status: RUNNING }})
+        })
+      }
+      const execute = asTask ? this._nextTask(payload, fn(payload)) : fn(payload)
+      all.push(execute)
+      const executeAndRemove = execute.finally(() => pending.splice(pending.indexOf(executeAndRemove), 1))
+      pending.push(executeAndRemove)
+      if (pending.length >= limit) {
+        await Promise.race(pending)
+      }
+    }
+    return Promise.allSettled(all)
+  }
+
+  async pickJob() {
+    if (jobQueue.size === 0) return
+
+    const next = new Set(jobQueue.peek().clusters.flat().map(t => t.tenant).flat())
+    const running = [...runningJobs.values()].flatMap(j => j.clusters.flat().map(t => t.tenant))
+    if (running.some(t => next.has(t))) return
+
+    const job = jobQueue.dequeue()
+    const { job_ID, clusters, fn, onJobDone } = job
+    try {
+      runningJobs.set(job_ID, job)
+      await t0_(async () => {
+        await UPDATE(Jobs, { ID: job_ID }).with({ status: RUNNING })
+        await this.emit('jobUpdate', { job: { ID: job_ID, status: RUNNING }})
+      })
+      await this._nextJob(clusters, fn, onJobDone)
+    } catch (e) {
+      await t0_(async () => {
+        await UPDATE(Jobs, { ID: job_ID }).with({ status: FAILED, error: _errorMessage(e) })
+        await this.emit('jobUpdate', { job: { ID: job_ID, status: FAILED, error: _errorMessage(e) }})
+      })
+    } finally {
+      runningJobs.delete(job_ID)
+    }
+    setImmediate(() => this.pickJob())
+  }
+
+  async _nextJob(clusters, fn, onJobDone) {
+    if (clusters.length > 1) {
+      await this.limiter(clusterSize, clusters, cluster => this.limiter(workerSize ?? poolSize, Array.from(cluster), fn, true))
+    } else {
+      await this.limiter(workerSize ?? poolSize, Array.from(clusters[0]), fn, true)
+    }
+
+    const { job_ID } = clusters[0][0] // all tasks have the same job ID -> just take the first
+    const failed = await t0_(SELECT.one.from(Tasks).where ({ job_ID, and: { status: FAILED }}))
+    const running = await t0_(SELECT.one.from(Tasks).where ({ job_ID, and: { status: RUNNING }}))
+
+    const patch = failed ? { status: FAILED } : { status: FINISHED }
+    await t0_(async () => {
+      await UPDATE(Jobs, { ID: job_ID }).with(patch)
+    })
+    if (failed) {
+      await onJobDone?.(failed.error ?? failed.ERROR ?? 'Unknown error')
+    } else if (!running) {
+      await onJobDone?.()
+    }
+    await this.emit('jobUpdate', { job: { ID: job_ID, ...patch } })
+  }
+
+  async _nextTask(task, _fn) {
+    const { ID, tenant, job_ID, op, service, args } = task
+    try {
+      await _fn
+      await t0_(async () => {
+        await UPDATE(Tasks, { ID, tenant }).with({ status: FINISHED })
+        await this.emit('taskUpdate', { task: { ID, job_ID, tenant, op, service, args, status: FINISHED}})
+      })
+    } catch (e) {
+      LOG.error(e)
+      const error = _errorMessage(e) ?? 'Unknown error'
+      await t0_(async () => {
+        await UPDATE(Tasks, { ID, tenant }).with({ status: FAILED, error })
+        await this.emit('taskUpdate', { task: { ID, job_ID, tenant, op, service, args, status: FAILED, error }})
+      })
+    }
+  }
+}
+
+function _errorMessage(e) {
+  let message = e.message ?? 'Unknown error'
+  if (e.error) message += ' ' + e.error
+  if (e.description) message += ': ' + e.description
+  return message.length > 5000 ? message.substring(0, 4997) + '...' : message // TODO use different datatype for errors -> next release
+}
+
+/*********************************************
+ * Clean-up
+ *********************************************/
+
+function _scheduled(srv) {
+  const minutes = 1000 * 60, hours = 60*minutes
+
+  const { jobs:conf, /* compat */ jobCleanup, jobCleanupInterval, jobCleanupAge } = cds.requires.multitenancy ?? {}
+  const {
+    heartbeatInterval, heartbeatAge,
+    cleanupInterval = jobCleanupInterval, cleanupAge = jobCleanupAge,
+    noCleanup = jobCleanup === false
+  } = conf ?? {}
+
+  const heartbeat = setInterval(async () => {
+    try {
+      const cutoff = new Date(new Date - (heartbeatAge ?? 90*minutes))
+      await t0_(async () => {
+        const timedOut = { status: { in: [RUNNING, QUEUED] }, and: { modifiedAt: { '<': cutoff.toISOString() }}}
+        const affectedJobs = await SELECT.from(Jobs, j => { j.ID, j.status }).where(timedOut)
+        const affectedTasks = await SELECT.from(Tasks, t => { t.ID, t.status, t.job_ID }).where(timedOut)
+        await UPDATE(Jobs, timedOut).with({ status: TIMEOUT })
+        await UPDATE(Tasks, timedOut).with({ status: TIMEOUT })
+        await Promise.all(affectedJobs.map(job => srv.emit('jobUpdate', { job: { ...job, status: TIMEOUT }})))
+        await Promise.all(affectedTasks.map(task => srv.emit('taskUpdate', { task: { ...task, status: TIMEOUT }})))
+      })
+    } catch (error) {
+      LOG.error('Error in heartbeat interval:', error)
+    }
+  }, heartbeatInterval ?? 5*minutes).unref()
+
+  let cleanup
+  if (!noCleanup) {
+    cleanup = setInterval(async () => {
+      try {
+        const cutoff = new Date(new Date - (cleanupAge ?? 24*hours))
+        await t0_(DELETE.from(Jobs, { status: FAILED, or: { status: FINISHED, and: { modifiedAt: { '<': cutoff.toISOString() }}}}))
+      } catch (error) {
+        LOG.error('Error in cleanup interval:', error)
+      }
+    }, cleanupInterval ?? 24*hours).unref()
+  }
+
+  cds.on('shutdown', () => {
+    clearInterval(heartbeat)
+    if (cleanup) clearInterval(cleanup)
+  })
+}
+
+/*********************************************
+ * Opt-in Websockets for diagnostics
+ *********************************************/
+const _diagnostics4 = srv => {
+  const WebSocket = require('ws')
+  let wss
+  cds.on('listening', async app => {
+    ;['jobUpdate', 'taskUpdate'].forEach(event => srv.on(event, forwardToWebSocketClients))
+    ;['hana:create', 'hana:destroy', 'hana:validate',
+      'pool:acquire', 'pool:release', 'pool:release:after',
+      'pool:destroy', 'pool:destroy:after', 'pool:drain', 'pool:drain:after',
+      'pool:createResource', 'pool:createResource:after', 'pool:dispense', 'pool:dispense:after',
+      'pool:scheduleEviction', 'pool:scheduleEviction:after'
+    ].forEach(event => cds.on(event, forwardToWebSocketClients))
+
+    wss = new WebSocket.Server({ server: app.server })
+    wss.on('connection', ws => {
+      DEBUG?.('Websocket client for jobs service connected')
+      ws.on('close', () => {
+        DEBUG?.('Websocket client for jobs service disconnected')
+      })
+    })
+  })
+
+  function forwardToWebSocketClients(data = {}) {
+    const message = JSON.stringify(data)
+    DEBUG?.('> to socket:', message)
+    wss.clients.forEach(client => {
+      if (client.readyState === WebSocket.OPEN) {
+        client.send(message)
+      }
+    })
+  }
+}

package/node_modules/@sap/cds-mtxs/srv/model-provider.cds

@@ -0,0 +1,85 @@
+context cds.xt { // IMPORTANT: we don't use namespaces to avoid issues with cds.db default namespace
+
+  @open type CSN {};
+  type TAR : LargeBinary;
+  type XML : LargeString;
+  type Locale : String;
+  type TenantID : String;
+  type CSNConsumer : String enum { nodejs; java; };
+  @open type I18N {}
+
+
+  /**
+   * Used by CAP runtimes to retrieve tenant-specific variants of
+   * deployed models used to serve requests.
+   */
+  @protocol: 'rest'
+  @requires: [ 'internal-user' ] //> we check that ourselves programmatically in production only
+  service ModelProviderService @(path:'/-/cds/model-provider', impl:'@sap/cds-mtxs/srv/model-provider.js') {
+
+    action getCsn(
+      tenant    : TenantID,
+      @cds.validate: false
+      toggles   : array of String,
+      for       : CSNConsumer @cds.java.name: 'runtime',
+      flavor    : String enum { parsed ; xtended ; inferred },
+      base      : Boolean,
+      activated : Boolean, // enable when only activated extensions should be considered
+      @open
+      options   : {}
+    ) returns CSN;
+
+    action getExtCsn(
+      tenant  : TenantID,
+      @cds.validate: false
+      toggles : array of String,
+      for     : CSNConsumer @cds.java.name: 'runtime',
+    ) returns CSN;
+
+    action getExtResources(
+      tenant    : TenantID
+    ) returns TAR;
+
+    action getExtCsvs(
+      tenant    : TenantID
+    ) returns CSN;
+
+    action getEdmx(
+      tenant  : TenantID,
+      @cds.validate: false
+      toggles : array of String,
+      service : String,
+      model   : CSN, // internal
+      locale  : Locale,
+      flavor  : String enum { v2; v4; w4; x4 },
+      for     : CSNConsumer @cds.java.name: 'runtime',
+    ) returns XML;
+
+    action getI18n(
+      tenant  : TenantID,
+      @cds.validate: false
+      toggles : array of String,
+      locale  : Locale
+    ) returns I18N;
+
+    action isExtended(
+      tenant  : TenantID,
+    ) returns Boolean;
+
+    action getExtensions(
+      tenant  : TenantID,
+      @open
+      options : {}
+    ) returns CSN;
+
+    action getResources () returns TAR;
+
+    // -------------------------------
+    // Later, possibly:
+    // function GET csn(...)
+    // function GET edmx(...)
+    // function GET extended(...)
+    // function GET resources(...)
+  }
+
+}