@sap/cds-dk 10.1.2 → 10.2.0

package/node_modules/@sap/cds-mtxs/srv/plugins/hana/serverless.js

@@ -59,12 +59,14 @@ class ServerlessClient extends AbstractContainerManagerClient {
     return new ServerlessClient(credentials)
   }
 
-  _tokenFor = async () => {
-    const cached = this.tokenCache.get('provider')
+  _tokenFor = async tenant => {
+    const t0 = cds.env.requires.multitenancy?.t0 ?? 't0'
+    const app_tid = tenant === t0 ? this.identity.app_tid : tenant
+    const cached = this.tokenCache.get(app_tid)
     if (cached && cached.expiry >= Date.now() + 30_000) return `Bearer ${cached.access_token}`
-    const raw = await token(this.identity, { form: { grant_type: 'client_credentials' }, path: '/oauth2/token' })
+    const raw = await token(this.identity, { form: { grant_type: 'client_credentials', ...(app_tid && { app_tid }) }, path: '/oauth2/token' })
     const { access_token, expires_in } = JSON.parse(raw)
-    this.tokenCache.set('provider', { access_token, expiry: Date.now() + expires_in * 1000 })
+    this.tokenCache.set(app_tid, { access_token, expiry: Date.now() + expires_in * 1000 })
     return `Bearer ${access_token}`
   }
 
@@ -74,8 +76,8 @@ class ServerlessClient extends AbstractContainerManagerClient {
     conf.headers['Accept'] ??= 'application/json'
     conf.headers['Content-Type'] ??= 'application/json'
     conf.headers['X-CorrelationID'] ??= cds.context?.id
-    const { retries, maxRetryAfter } = cds.requires?.multitenancy?.serviceManager ?? cds.requires?.multitenancy?.containerManager ?? {}
-    return fetchResiliently(this.gateway + path, conf, { retriesLeft: retries ?? 10, maxRetryAfter: maxRetryAfter ?? 5000 })
+    const { retries, maxRetryAfter } = this._config()
+    return fetchResiliently(this.gateway + path, conf, { retriesLeft: retries, maxRetryAfter })
   }
 
   _pollLocation = async (tenant, location, extract) => {
@@ -200,22 +202,17 @@ class ServerlessClient extends AbstractContainerManagerClient {
   }
 
   getAll = async (tenants = '*', options = {}) => {
-    const seed = tenants === '*' ? null : tenants
-    const listTenant = seed?.[0]
-    let containers = []
-    if (listTenant) {
-      const { data } = await this._api(listTenant, API, { method: 'GET' })
-      containers = (data.data || []).filter(c => (c.labels || []).some(l => l.startsWith('tenant_id=')))
-    }
+    if (tenants === '*') return []
     const results = []
-    for (const c of containers) {
-      const tenant_id = (c.labels.find(l => l.startsWith('tenant_id=')) || '').split('=')[1]
-      if (!tenant_id) continue
-      const entry = { labels: { tenant_id: [tenant_id] }, container: c }
-      if (options.skipCredentials) { results.push(entry); continue }
+    for (const tenant_id of tenants) {
       try {
-        const { credentials } = await this.get(tenant_id, options)
-        results.push({ ...entry, credentials })
+        if (options.skipCredentials) {
+          const container = await this._findContainer(tenant_id)
+          if (container) results.push({ labels: { tenant_id: [tenant_id] }, container })
+        } else {
+          const { container, credentials } = await this.get(tenant_id, options)
+          results.push({ labels: { tenant_id: [tenant_id] }, container, credentials })
+        }
       } catch (e) {
         if (e.status === 404) DEBUG?.('skipping tenant', tenant_id, 'without usable credentials')
         else throw e

package/node_modules/@sap/cds-mtxs/srv/plugins/hana/srv-mgr-v2.js

@@ -0,0 +1,511 @@
+const cds = require('@sap/cds')
+const LOG = cds.log('mtx'), DEBUG = cds.debug('mtx|sm')
+const { uuid } = cds.utils
+const AbstractContainerManagerClient = require('./ctnr-mgr-base')
+const { cacheBindings = true } = cds.env.requires.multitenancy ?? {}
+const { TMS_ONLY_PARAMS } = AbstractContainerManagerClient
+
+/* API */
+class ServiceManagerClient extends AbstractContainerManagerClient {
+  constructor({ sm_url, url, clientid, clientsecret, certurl, certificate, key }) {
+    super({ url, clientid, clientsecret })
+    this.sm_url = sm_url
+    this.url = url
+    this.clientid = clientid
+    this.clientsecret = clientsecret
+    this.certurl = certurl
+    this.certificate = certificate
+    this.key = key
+    this.sm_api_version = 'v2'
+
+    // In-memory storage -> later also distribute w/ Redis
+    this.instanceLocations = new Map()
+    this.bindingLocations = new Map()
+
+    this._bindingsCache = {}
+    this.delete = this.remove
+  }
+
+  static defaultInstance() {
+    ServiceManagerClient._defaultInstance ??= ServiceManagerClient.newInstance(cds.env.requires.db.credentials)
+    return ServiceManagerClient._defaultInstance
+  }
+
+  // TODO introduce instance cache
+  static newInstance(credentials) {
+    if (!credentials?.sm_url) cds.error('Service Manager URL is not configured in credentials')
+    return new ServiceManagerClient(credentials)
+  }
+
+  _pollPath4 = (resource, resourceId, operationId) => {
+    if (!resource || !resourceId || !operationId) return null
+    return `${resource}/${resourceId}/operations/${operationId}`
+  }
+
+  _nextPageToken4 = (data, headers) => {
+    const nextLink = headers?.link?.match(/<[^>]*[?&]page_token=([^&>]+)[^>]*>;\s*rel="next"/)
+    return data?.next_page_token
+      ?? data?.nextPageToken
+      ?? data?.page_token
+      ?? data?.pageToken
+      ?? headers?.['x-next-page-token']
+      ?? headers?.['x-nextPageToken']
+      ?? (nextLink && decodeURIComponent(nextLink[1]))
+  }
+
+  _pollError = (response) => response.data?.error?.broker_error ?? response.data?.error ?? response.data
+
+  _isConcurrencyError = (e) => {
+    const msg = `${e?.message ?? ''}`
+    const code = `${e?.code ?? ''}`
+    return e?.status === 422 && (
+      code.includes('ConcurrencyError')
+      || msg.includes('ConcurrencyError')
+      || msg.includes('Another operation is currently in progress')
+      || msg.includes('service instance is not ready for binding')
+    )
+  }
+
+  _resolveCreatedInstanceId = async (tenant, operation) => {
+    if (operation?.resource_id) return operation.resource_id
+    for (let attempt = 0; attempt < 6; attempt++) {
+      const instance = await this._instance4(tenant)
+      if (instance?.id) return instance.id
+      await new Promise(resolve => setTimeout(resolve, 400 * 2 ** attempt))
+    }
+    return null
+  }
+
+  _createBindingWithRetry = async (tenant, service_instance_id, service_plan_id, binding_parameters, customLabels = {}) => {
+    const maxAttempts = 8
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      try {
+        const response = await this.fetchApi('service_bindings', {
+          method: 'POST',
+          data: {
+            name: tenant + `-${uuid()}`, service_instance_id, parameters: binding_parameters,
+            labels: { tenant_id: [tenant], service_plan_id: [service_plan_id], managing_client_lib: ['instance-manager-client-lib'], ...customLabels }
+          }
+        })
+
+        const operation = response.data
+        const location = this._pollPath4('service_bindings', operation?.resource_id, operation?.id)
+        if (location) {
+          this.bindingLocations.set(tenant, location)
+          await this._poll(location)
+          this.bindingLocations.delete(tenant)
+        }
+        return
+      } catch (e) {
+        if (!this._isConcurrencyError(e) || attempt === maxAttempts - 1) throw e
+        await new Promise(resolve => setTimeout(resolve, Math.min(8000, 1000 * 2 ** attempt)))
+      }
+    }
+  }
+
+  _getCreatedBinding = async (tenant) => {
+    const retryUntil = Date.now() + 30_000
+    for (let attempt = 0; ; attempt++) {
+      try {
+        return await this.get(tenant, { disableCache: true, retryUntil })
+      } catch (e) {
+        if (e.status !== 404 || Date.now() >= retryUntil) throw e
+        await new Promise(resolve => setTimeout(resolve, Math.min(5000, 500 * 2 ** attempt)))
+      }
+    }
+  }
+
+  create = async (tenant, parameters) => {
+    LOG.info('creating HDI container for', { tenant }, ...(parameters ? ['with', { ...parameters }] : []))
+    const name = await this._instanceName4(tenant), service_plan_id = await this._planId()
+    const { binding_parameters, provisioning_parameters, customLabels = {} } = parameters ?? {}
+    let service_instance_id
+
+    if (this.instanceLocations.has(tenant)) {
+      const storedLocation = this.instanceLocations.get(tenant)
+      LOG.info('polling ongoing instance creation for', { tenant })
+      const { data: polledOperation } = await this._poll(storedLocation)
+      service_instance_id = polledOperation.resource_id
+      this.instanceLocations.delete(tenant)
+    } else {
+      try {
+        const response = await this.fetchApi('service_instances', {
+          method: 'POST',
+          data: {
+            name, service_plan_id, parameters: provisioning_parameters,
+            labels: { tenant_id: [tenant], ...customLabels },
+          }
+        })
+
+        const operation = response.data
+        const location = this._pollPath4('service_instances', operation?.resource_id, operation?.id)
+        if (location) {
+          this.instanceLocations.set(tenant, location)
+          service_instance_id = (await this._poll(location)).data?.resource_id
+          this.instanceLocations.delete(tenant)
+        } else {
+          service_instance_id = await this._resolveCreatedInstanceId(tenant, operation)
+          if (!service_instance_id) cds.error('Service Manager did not return operation metadata or resource id for service instance creation')
+        }
+      } catch (e) {
+        this.instanceLocations.delete(tenant)
+        const status = e.status ?? 500
+        if (status === 409 || e.error === 'Conflict') {
+          const instance = await this._instance4(tenant)
+          const isSucceeded = instance?.last_operation?.state === 'succeeded'
+          const isUsable = instance?.usable ?? true
+
+          const { type, state, id: operation_id, error } = instance?.last_operation ?? {}
+          const inProgress = state === 'in progress' || state === 'in_progress' || state === 'in-progress'
+
+          if (type === 'delete') {
+            if (inProgress && instance?.id && operation_id) {
+              const location = this._pollPath4('service_instances', instance.id, operation_id)
+              LOG.info('polling ongoing instance deletion for tenant', tenant, 'at location', location)
+              await this._poll(location)
+            } else if (state === 'failed') {
+              LOG.info('retrying failed instance deletion for tenant', tenant)
+              await this.remove(tenant)
+            } else {
+              const indexDeadline = Date.now() + 15_000
+              for (let attempt = 0; ; attempt++) {
+                if (!await this._instance4(tenant)) break
+                if (Date.now() >= indexDeadline) { LOG.warn('SM index still shows deleted instance for tenant', tenant, '- proceeding anyway'); break }
+                await new Promise(resolve => setTimeout(resolve, Math.min(2000, 500 * 2 ** attempt)))
+              }
+            }
+            return this.create(tenant, parameters)
+          }
+
+          if (instance && !(isSucceeded && isUsable)) {
+            const lastError = error?.error ?? error?.broker_error?.error_message
+            const lastDesc = error?.description ?? error?.broker_error?.description
+
+            LOG.info('detected unusable instance for tenant', tenant, 'in state', state, 'for operation type', type)
+            if (state === 'failed' && type === 'create') {
+              LOG.info('removing and recreating faulty instance for tenant', tenant,
+                DEBUG ? `with error: ${e.error}: ${e.description}. Last operation: ${lastError ?? ''} ${lastDesc ?? ''}` : ''
+              )
+              await this.remove(tenant)
+              return this.create(tenant, parameters)
+            } else if (type === 'create' && inProgress && instance?.id && operation_id) {
+              const location = this._pollPath4('service_instances', instance.id, operation_id)
+              LOG.info('polling ongoing instance creation for tenant', tenant, 'at location', location)
+              this.instanceLocations.set(tenant, location)
+              await this._poll(location)
+              this.instanceLocations.delete(tenant)
+            } else {
+              e.message ??= ''
+              e.message += `${e.error}: ${e.description}. Last operation: ${lastError ?? ''} ${lastDesc ?? ''}`
+              throw e
+            }
+          }
+
+          service_instance_id = instance?.id
+        } else {
+          cds.error(ServiceManagerClient._errorMessage(e, 'creating', tenant), { status })
+        }
+      }
+    }
+
+    if (this.bindingLocations.has(tenant)) {
+      const storedLocation = this.bindingLocations.get(tenant)
+      LOG.info('ongoing binding creation for tenant', tenant, 'polling existing request')
+      try {
+        await this._poll(storedLocation)
+      } finally {
+        this.bindingLocations.delete(tenant)
+      }
+    } else {
+      await this._createBindingWithRetry(tenant, service_instance_id, service_plan_id, binding_parameters, customLabels)
+    }
+
+    const binding = { ...await this._getCreatedBinding(tenant), tags: ['hana'] }
+    return cacheBindings ? this._bindingsCache[tenant] = binding : binding
+  }
+
+  acquire = async (tenant, parameters) => {
+    try { return await this.get(tenant, { disableCache: true }) } catch (e) {
+      if (e.status === 404) return this.create(tenant, parameters)
+      throw e
+    }
+  }
+
+  get = async (tenant, options = {}) => {
+    const { invalidCredentials, retryUntil } = options
+    if (invalidCredentials && cacheBindings) delete this._bindingsCache[tenant]
+    try {
+      const previousFailures = invalidCredentials ? 0 : (this.failures.has(tenant) ? this.failures.get(tenant).count : 0)
+      const [binding] = await this._bindings4([tenant], { ...options, tenant, failures: previousFailures, retryUntil })
+      if (!binding) cds.error(`Tenant '${tenant}' does not exist`, { status: 404 })
+      const _complete = c => c?.host && c?.port && c?.user && c?.password && c?.schema
+      if (binding.id) {
+        if (!_complete(binding.credentials)) {
+          const deadline = retryUntil ?? (Date.now() + 30_000)
+          for (let attempt = 0; ; attempt++) {
+            const { data } = await this.fetchApi(`service_bindings/${binding.id}/credentials`, { method: 'GET' }, { failures: 0 })
+            binding.credentials = data?.credentials
+            if (_complete(binding.credentials)) break
+            if (Date.now() >= deadline) break
+            await new Promise(resolve => setTimeout(resolve, Math.min(10000, 500 * 2 ** attempt)))
+          }
+        }
+      }
+      if (!_complete(binding?.credentials)) cds.error(`Tenant '${tenant}' does not exist`, { status: 404 })
+      const { credentials } = binding
+      credentials.tenant ??= tenant
+
+      if (invalidCredentials && JSON.stringify(credentials) === JSON.stringify(invalidCredentials)) {
+        LOG.info('previously rejected credentials provided by Service Manager → re-fetching')
+        if (!this.failures.has(tenant)) this.failures.set(tenant, { count: 1 })
+        else this.failures.get(tenant).count++
+        return this.get(tenant, options)
+      }
+      this.failures.delete(tenant)
+      return { name: await this._instanceName4(tenant), tenant_id: tenant, credentials, tags: ['hana'] }
+    } catch (e) {
+      if (e.status) cds.error(ServiceManagerClient._errorMessage(e, 'getting', tenant), { status: e.status ?? 500, cause: e.stack })
+      else throw e
+    }
+  }
+
+  getAll = async (tenants = '*', options) => {
+    return this._bindings4(tenants, options)
+  }
+
+  getAllInstances = async () => {
+    const instances = []
+    const service_plan_id = await this._planId()
+    let page_token
+    do {
+      const params = page_token
+        ? { page_token, service_plan_id, max_items: 100 }
+        : { service_plan_id, max_items: 100 }
+      const response = await this.fetchApi('service_instances', { params })
+      const { data, headers } = response
+      instances.push(...(data?.items ?? []))
+      page_token = this._nextPageToken4(data, headers)
+    } while (page_token)
+    return instances
+  }
+
+  deploy = async (container, tenant, out, options, deployEnv) => {
+    return require('./hdi').deploy(container, tenant, out, options, deployEnv)
+  }
+
+  remove = async (tenant) => {
+    const instance = await this._instance4(tenant, true)
+
+    const bindings = []
+    let page_token
+
+    if (instance) {
+      do {
+        const params = page_token
+          ? { page_token, service_instance_id: instance.id, max_items: 100 }
+          : { service_instance_id: instance.id, max_items: 100 }
+
+        const response = await this.fetchApi('service_bindings', { params })
+        const { data, headers } = response
+        bindings.push(...(data?.items ?? []))
+        page_token = this._nextPageToken4(data, headers)
+      } while (page_token)
+    }
+
+    page_token = undefined
+    do {
+      const labels = `tenant_id=${tenant},service_plan_id=${await this._planId()}`
+      const params = page_token
+        ? { page_token, labels, max_items: 100 }
+        : { labels, max_items: 100 }
+
+      const response = await this.fetchApi('service_bindings', { params })
+      const { data, headers } = response
+      bindings.push(...(data?.items ?? []))
+      page_token = this._nextPageToken4(data, headers)
+    } while (page_token)
+
+    const deduped = [...new Map(bindings.map(b => [b.id, b])).values()]
+    const _deleteBindings = deduped.map(async ({ id }) => {
+      let response
+      try {
+        response = await this.fetchApi(`service_bindings/${id}`, { method: 'DELETE' })
+      } catch (e) {
+        if (e.status === 404) return
+        throw e
+      }
+      const operation = response.data
+      const location = this._pollPath4('service_bindings', id, operation?.id)
+      if (location) return this._poll(location)
+    })
+
+    if (cacheBindings) delete this._bindingsCache[tenant]
+    this.failures.delete(tenant)
+
+    const failedDeletions = (await Promise.allSettled(_deleteBindings)).filter(d => d.status === 'rejected')
+    if (failedDeletions.length > 0) throw new AggregateError(failedDeletions.map(d => d.reason))
+
+    if (instance) {
+      const deadline = Date.now() + 60_000
+      for (let attempt = 0; ; attempt++) {
+        try {
+          const response = await this.fetchApi(`service_instances/${instance.id}`, { method: 'DELETE' })
+          const operation = response.data
+          const location = this._pollPath4('service_instances', instance.id, operation?.id)
+          if (location) await this._poll(location)
+          break
+        } catch (e) {
+          if (e.status === 404) return
+          const isDependentResources = e.status === 422 && (e.message?.includes('DependentResourcesExistError') || e.message?.includes('dependent resources'))
+          const isConcurrency = this._isConcurrencyError(e)
+          if ((!isDependentResources && !isConcurrency) || Date.now() >= deadline) throw e
+          await new Promise(resolve => setTimeout(resolve, Math.min(10000, 500 * 2 ** attempt)))
+        }
+      }
+      const indexDeadline = Date.now() + 10_000
+      for (let attempt = 0; ; attempt++) {
+        if (!await this._instance4(tenant, true)) break
+        if (Date.now() >= indexDeadline) break
+        await new Promise(resolve => setTimeout(resolve, Math.min(2000, 200 * 2 ** attempt)))
+      }
+    }
+  }
+
+  _encryptionParams(data) {
+    return (data?.globalAccountGUID ?? data?.subscriber?.globalAccountId) ? {
+      subscriptionContext: {
+        globalAccountID: data.globalAccountGUID ?? data.subscriber.globalAccountId,
+        subAccountID: data.subscribedSubaccountId ?? data.subscriber.subaccountId,
+        applicationName: data.subscriptionAppName ?? data.rootApplication?.appName
+      }
+    } : {}
+  }
+
+  _createParams = (tenant, params = {}, metadata) => {
+    const hdiParams = this._hdiParams(tenant, params, metadata)
+    // Filter out TMS-specific parameters that the Service Manager rejects
+    for (const key of TMS_ONLY_PARAMS) delete hdiParams?.create?.[key]
+    const final = {}
+    if (Object.keys(hdiParams?.create ?? {}).length > 0) final.provisioning_parameters = hdiParams.create
+    if (Object.keys(hdiParams?.bind ?? {}).length > 0) final.binding_parameters = hdiParams.bind
+    if (hdiParams?.create?.custom_label) {
+      const customLabel = hdiParams.create.custom_label
+      const [k, v] = customLabel.split('=')
+      if (k === 'tenant_id') cds.error('"tenant_id" is not allowed as a custom label key')
+      final.customLabels = { [k]: [v] }
+    }
+    return Object.keys(final).length > 0 ? final : null
+  }
+
+  /* Private helpers */
+
+  async _instance4(tenant, useLabel) {
+    const name = await this._instanceName4(tenant)
+    const service_plan_id = await this._planId()
+    const params = useLabel ? { labels: `tenant_id=${tenant}`, service_plan_id, max_items: 1 } : { name, service_plan_id, max_items: 1 }
+    const instances = await this.fetchApi('service_instances', { params })
+    return instances.data.items?.[0]
+  }
+
+  async _instanceName4(tenant) {
+    if (cds.requires.multitenancy?.humanReadableInstanceName) return tenant
+    return require('crypto').createHash('sha256').update(`${await this._planId()}_${tenant}`).digest('base64')
+  }
+
+  async _bindings4(tenants, { disableCache = false, failures, retryUntil } = {}) {
+    if (failures) disableCache = true
+
+    const isWildcard = tenants === '*'
+    const tenantList = isWildcard ? [] : tenants
+    const useCache = cacheBindings && !disableCache && !isWildcard
+    const uncached = useCache ? tenantList.filter(t => !(t in this._bindingsCache)) : tenantList
+
+    DEBUG?.('retrieving', { tenants }, { uncached })
+    if (!isWildcard && uncached.length === 0) return tenantList.map(t => this._bindingsCache[t])
+
+    const fetched = []
+    const planId = await this._planId()
+
+    const fetchForLabels = async (labels) => {
+      let page_token
+      do {
+        const params = page_token
+          ? { page_token, labels, max_items: 100 }
+          : { labels, max_items: 100 }
+
+        const response = await this.fetchApi('service_bindings', { params }, { failures, retryUntil })
+        const { data, headers } = response
+        fetched.push(...(data?.items ?? []))
+        page_token = this._nextPageToken4(data, headers)
+      } while (page_token)
+    }
+
+    const fetchByTenant = async (tenant) => {
+      let page_token
+      do {
+        const params = page_token
+          ? { page_token, labels: `service_plan_id=${planId},tenant_id=${tenant}`, max_items: 100 }
+          : { labels: `service_plan_id=${planId},tenant_id=${tenant}`, max_items: 100 }
+        const response = await this.fetchApi('service_bindings', { params }, { failures, retryUntil })
+        const { data, headers } = response
+        fetched.push(...(data?.items ?? []))
+        page_token = this._nextPageToken4(data, headers)
+      } while (page_token)
+    }
+
+    if (!isWildcard && uncached.length === 1) {
+      await fetchForLabels(`service_plan_id=${planId},tenant_id=${uncached[0]}`)
+    } else if (!isWildcard && uncached.length > 1 && uncached.length <= 5) {
+      for (const tenant of uncached) await fetchByTenant(tenant)
+    } else {
+      await fetchForLabels(`service_plan_id=${planId}`)
+    }
+
+    const allByTenant = Object.fromEntries(
+      fetched
+        .filter(b => b.labels?.tenant_id?.[0])
+        .map(b => [b.labels.tenant_id[0], b])
+    )
+
+    if (!isWildcard) {
+      const wanted = new Set(tenantList)
+      for (const tenant of Object.keys(allByTenant)) {
+        if (!wanted.has(tenant)) delete allByTenant[tenant]
+      }
+    }
+
+    if (!useCache && !isWildcard && cacheBindings) {
+      for (const tenant of tenantList) delete this._bindingsCache[tenant]
+    }
+    Object.assign(this._bindingsCache, allByTenant)
+
+    if (useCache) return tenantList.map(t => this._bindingsCache[t])
+    if (isWildcard) {
+      const fetchedTenants = new Set(fetched.map(b => b.labels?.tenant_id?.[0]).filter(Boolean))
+      for (const [tenant, binding] of Object.entries(this._bindingsCache)) {
+        if (!fetchedTenants.has(tenant)) fetched.push(binding.labels ? binding : { ...binding, labels: { tenant_id: [tenant] } })
+      }
+      return fetched
+    }
+    return tenantList.map(t => allByTenant[t]).filter(Boolean)
+  }
+
+  async _planId() {
+    if (this._planIdCached) return this._planIdCached
+    const { data } = await this.fetchApi('service_plans', { params: { name: 'hdi-shared', service_offering_id: await this._offeringId() } })
+    const [planId] = data.items
+    if (!planId) cds.error(`Could not find service plan with name 'hdi-shared'. Make sure necessary entitlements are assigned to your BTP subaccount.`)
+    return this._planIdCached = data.items[0].id
+  }
+
+  async _offeringId() {
+    if (this._offeringIdCached) return this._offeringIdCached
+    const { data } = await this.fetchApi('service_offerings', { params: { name: 'hana' } })
+    const [offeringId] = data.items
+    if (!offeringId) cds.error(`Could not find service offering with name 'hana'. Make sure necessary entitlements are assigned to your SAP BTP subaccount.`)
+    return this._offeringIdCached = data.items[0].id
+  }
+
+}
+
+module.exports = ServiceManagerClient

package/node_modules/@sap/cds-mtxs/srv/plugins/hana/srv-mgr.js

@@ -3,6 +3,7 @@ const LOG = cds.log('mtx'), DEBUG = cds.debug('mtx|sm')
 const { uuid } = cds.utils
 const AbstractContainerManagerClient = require('./ctnr-mgr-base')
 const { cacheBindings = true } = cds.env.requires.multitenancy ?? {}
+const { TMS_ONLY_PARAMS } = AbstractContainerManagerClient
 
 /* API */
 class ServiceManagerClient extends AbstractContainerManagerClient {
@@ -64,10 +65,13 @@ class ServiceManagerClient extends AbstractContainerManagerClient {
         const status = e.status ?? 500
         if (status === 409 || e.error === 'Conflict') {
           const instance = await this._instance4(tenant)
-          if (!instance.ready || !instance.usable) {
-            const { type, state, errors, resource_type } = instance?.last_operation ?? {}
+          const { type, state, errors, resource_type } = instance?.last_operation ?? {}
+          // An instance whose last operation was a succeeded deletion is logically gone,
+          // even if it still reports ready/usable as true → treat it as unusable.
+          const deleted = type === 'delete' && state === 'succeeded'
+          if (!instance.ready || !instance.usable || deleted) {
             LOG.info('detected unusable instance for tenant', tenant, 'in state', state, 'for operation type', type)
-            if (state === 'failed' && (type === 'create' || type === 'delete')) {
+            if (deleted || (state === 'failed' && (type === 'create' || type === 'delete'))) {
               LOG.info('removing and recreating faulty instance for tenant', tenant,
                 DEBUG ? `with error: ${e.error}: ${e.description}. Last operation: ${errors?.error} ${errors?.description}` : ''
               )
@@ -153,7 +157,8 @@ class ServiceManagerClient extends AbstractContainerManagerClient {
   }
 
   deploy = async (container, tenant, out, options, deployEnv) => {
-    return require('./hdi').deploy(container, tenant, out, options, deployEnv)
+    const hdi = cds.env.features?.native_hdi_deploy ? require('./hdi-native') : require('./hdi')
+    return hdi.deploy(container, tenant, out, options, deployEnv)
   }
 
   remove = async (tenant) => {
@@ -207,6 +212,8 @@ class ServiceManagerClient extends AbstractContainerManagerClient {
   // merges configured default parameters with parameters derived from metadata and tenant-specific parameters
   _createParams = (tenant, params = {}, metadata) => {
     const hdiParams = this._hdiParams(tenant, params, metadata)
+    // Filter out TMS-specific parameters that the Service Manager rejects
+    for (const key of TMS_ONLY_PARAMS) delete hdiParams?.create?.[key]
     const final = {}
     if (Object.keys(hdiParams?.create ?? {}).length > 0) final.provisioning_parameters = hdiParams.create
     if (Object.keys(hdiParams?.bind ?? {}).length > 0) final.binding_parameters = hdiParams.bind

package/node_modules/@sap/cds-mtxs/srv/plugins/hana/tms.js

@@ -16,6 +16,14 @@ class TMSClient extends AbstractContainerManagerClient {
     super(credentials.uaa)
     this.tmsHost = credentials.baseurl
 
+    // Forward X.509/mTLS material from the binding so token()'s certificate branch can fire
+    // (same as ServiceManagerClient). Without this, an x509 binding falls back to Basic auth
+    // with an empty client secret and fails with 401 invalid_client.
+    const { certurl, certificate, key } = credentials.uaa ?? {}
+    this.certurl = certurl
+    this.certificate = certificate
+    this.key = key
+
     this.credentialCache = {}
 
     // clusterSize > 1 not supported yet for TMS