@xen-orchestra/backups 0.49.0 → 0.50.0

package/_cleanVm.mjs

@@ -3,7 +3,7 @@ import sum from 'lodash/sum.js'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
 import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
-import { dirname, resolve } from 'node:path'
+import { basename, dirname, resolve } from 'node:path'
 import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { mergeVhdChain } from 'vhd-lib/merge.js'
@@ -179,7 +179,8 @@ export async function cleanVm(
   vmDir,
   {
     fixMetadata,
-    remove,
+    remove = false,
+    removeTmp = remove,
     merge,
     mergeBlockConcurrency,
     mergeLimiter = defaultMergeLimiter,
@@ -200,6 +201,10 @@ export async function cleanVm(
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {
+    if(removeTmp && basename(path)[0] === '.'){
+      logInfo('deleting temporary VHD', { path })
+        return VhdAbstract.unlink(handler, path)
+    }
     try {
       await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
         if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {

package/_incrementalVm.mjs

@@ -1,6 +1,5 @@
 import groupBy from 'lodash/groupBy.js'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
-import omit from 'lodash/omit.js'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { CancelToken } from 'promise-toolbox'
 import { compareVersions } from 'compare-versions'
@@ -10,43 +9,16 @@ import { defer } from 'golike-defer'
 import { cancelableMap } from './_cancelableMap.mjs'
 import { Task } from './Task.mjs'
 import pick from 'lodash/pick.js'
-
-// in `other_config` of an incrementally replicated VM, contains the UUID of the source VM
-export const TAG_BASE_DELTA = 'xo:base_delta'
-
-// in `other_config` of an incrementally replicated VM, contains the UUID of the target SR used for replication
-//
-// added after the complete replication
-export const TAG_BACKUP_SR = 'xo:backup:sr'
-
-// in other_config of VDIs of an incrementally replicated VM, contains the UUID of the source VDI
-export const TAG_COPY_SRC = 'xo:copy_of'
+import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
 
 const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
 
 export async function exportIncrementalVm(
   vm,
-  baseVm,
-  {
-    cancelToken = CancelToken.none,
-
-    // Sets of UUIDs of VDIs that must be exported as full.
-    fullVdisRequired = new Set(),
-
-    disableBaseTags = false,
-    nbdConcurrency = 1,
-    preferNbd,
-  } = {}
+  baseVdis = {},
+  { cancelToken = CancelToken.none, nbdConcurrency = 1, preferNbd } = {}
 ) {
   // refs of VM's VDIs → base's VDIs.
-  const baseVdis = {}
-  baseVm &&
-    baseVm.$VBDs.forEach(vbd => {
-      let vdi, snapshotOf
-      if ((vdi = vbd.$VDI) && (snapshotOf = vdi.$snapshot_of) && !fullVdisRequired.has(snapshotOf.uuid)) {
-        baseVdis[vdi.snapshot_of] = vdi
-      }
-    })
 
   const streams = {}
   const vdis = {}
@@ -67,13 +39,16 @@ export async function exportIncrementalVm(
     }
 
     // Look for a snapshot of this vdi in the base VM.
-    const baseVdi = baseVdis[vdi.snapshot_of]
+    const baseVdi = baseVdis[vdi.$snapshot_of.uuid]
 
     vdis[vdiRef] = {
       ...vdi,
       other_config: {
         ...vdi.other_config,
-        [TAG_BASE_DELTA]: baseVdi && !disableBaseTags ? baseVdi.uuid : undefined,
+        [BASE_DELTA_VDI]: baseVdi?.uuid,
+        [VM_UUID]:
+          vm.$snapshot_of?.uuid ?? // vm is a snapshot
+          vm.uuid, // vm is a not snapshot
       },
       $snapshot_of$uuid: vdi.$snapshot_of?.uuid,
       $SR$uuid: vdi.$SR.uuid,
@@ -120,13 +95,6 @@ export async function exportIncrementalVm(
       vifs,
       vm: {
         ...vm,
-        other_config:
-          baseVm && !disableBaseTags
-            ? {
-                ...vm.other_config,
-                [TAG_BASE_DELTA]: baseVm.uuid,
-              }
-            : omit(vm.other_config, TAG_BASE_DELTA),
       },
     },
     'streams',
@@ -205,7 +173,7 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
       newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
       $defer.onFailure(() => newVdi.$destroy())
 
-      await newVdi.update_other_config(TAG_COPY_SRC, vdi.uuid)
+      await newVdi.update_other_config(COPY_OF, vdi.uuid)
       if (vdi.virtual_size > newVdi.virtual_size) {
         await newVdi.$callAsync('resize', vdi.virtual_size)
       }

package/_otherConfig.mjs

@@ -0,0 +1,165 @@
+import { formatDateTime } from '@xen-orchestra/xapi'
+import assert from 'node:assert/strict'
+// in `other_config` of an incrementally replicated VM or VDI
+// contains the UUID of the object used as a base for an incremental export
+// used to search for the replica of the base before applying a incremental replication
+export const BASE_DELTA_VDI = 'xo:base_delta_vdi'
+
+// in `other_config` of an incrementally replicated VM, contains the UUID of the target SR used for replication
+//
+// added after the complete replication
+export const REPLICATED_TO_SR_UUID = 'xo:backup:sr'
+
+// in other_config of VDIs of an incrementally replicated VM
+// contains the UUID of the source  exported object (snapshot or VM)
+
+export const COPY_OF = 'xo:copy_of'
+
+export const DATETIME = 'xo:backup:datetime'
+
+export const JOB_ID = 'xo:backup:job'
+export const SCHEDULE_ID = 'xo:backup:schedule'
+
+// contains the number of delta in a chain, stored as a string
+export const DELTA_CHAIN_LENGTH = 'xo:backup:deltaChainLength'
+// contains the string true if this vdi has been exported successfully
+export const EXPORTED_SUCCESSFULLY = 'xo:backup:exported'
+
+// the VM ( not the snapshot) uuid
+export const VM_UUID = 'xo:backup:vm'
+
+async function listVdiRefs(xapi, vmRef) {
+  return xapi.VM_getDisks(vmRef)
+}
+
+async function applyToVmAndVdis(xapi, vmRef, fn) {
+  const vdiRefs = await listVdiRefs(xapi, vmRef)
+  return Promise.all([fn('VM', vmRef), ...vdiRefs.map(vdiRef => fn('VDI', vdiRef))])
+}
+
+async function getDeltaChainLength(xapi, type, ref) {
+  const otherConfig = await xapi.getField(type, ref, 'other_config')
+  return Number(otherConfig[DELTA_CHAIN_LENGTH] ?? 0)
+}
+
+/**
+ * set the delta chain lenght ( number of delta since last base backup) to a VM and its associated VDIs
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @param {Number} length
+ * @returns {Promise}
+ */
+export async function setVmDeltaChainLength(xapi, vmRef, length) {
+  return applyToVmAndVdis(xapi, vmRef, async (type, ref) => {
+    await xapi.setFieldEntry(type, ref, 'other_config', DELTA_CHAIN_LENGTH, String(length))
+  })
+}
+
+/**
+ * Compute the delta chain length of a VM and its associated VDIs
+ * if there is a discrependcy, use, the highest value
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @returns {Promise}
+ */
+export async function getVmDeltaChainLength(xapi, vmRef) {
+  const lengths = await applyToVmAndVdis(xapi, vmRef, async (type, ref) => getDeltaChainLength(xapi, type, ref))
+  return Math.max(...lengths)
+}
+
+/**
+ *
+ * Reset the other_config field of a VM and its VDIs
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @returns {Promise}
+ */
+export function resetVmOtherConfig(xapi, vmRef) {
+  return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
+    return xapi.setFieldEntries(type, ref, 'other_config', {
+      [DATETIME]: null,
+      [DELTA_CHAIN_LENGTH]: null,
+      [EXPORTED_SUCCESSFULLY]: null,
+      [JOB_ID]: null,
+      [SCHEDULE_ID]: null,
+      [VM_UUID]: null,
+      // REPLICATED_TO_SR_UUID is not reset since we can replicate a replication
+    })
+  })
+}
+
+/**
+ *
+ * used to ensure compatibiliy with the previous snapshots that were having the config stored only into VM
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @returns {Promise}
+ */
+export async function populateVdisOtherConfig(xapi, vmRef) {
+  const otherConfig = await xapi.getField('VM', vmRef, 'other_config')
+  const {
+    [DATETIME]: datetime,
+    [DELTA_CHAIN_LENGTH]: chainLength,
+    [EXPORTED_SUCCESSFULLY]: successfully,
+    [JOB_ID]: jobId,
+    [REPLICATED_TO_SR_UUID]: replicatedTo,
+    [SCHEDULE_ID]: scheduleId,
+    [VM_UUID]: vmUuid,
+  } = otherConfig
+
+  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
+    xapi.setFieldEntries(type, ref, 'other_config', {
+      [DATETIME]: datetime,
+      [DELTA_CHAIN_LENGTH]: chainLength,
+      [EXPORTED_SUCCESSFULLY]: successfully,
+      [JOB_ID]: jobId,
+      [REPLICATED_TO_SR_UUID]: replicatedTo,
+      [SCHEDULE_ID]: scheduleId,
+      [VM_UUID]: vmUuid,
+    })
+  )
+}
+
+/**
+ *
+ * set the other_config key related to a backup of a VM and its associated VDIs
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @param {*} settings
+ * @returns {PRomise}
+ */
+export async function setVmOtherConfig(xapi, vmRef, { timestamp, jobId, scheduleId, vmUuid, srUuid = null, ...other }) {
+  assert.notEqual(timestamp, undefined)
+  assert.notEqual(jobId, undefined)
+  assert.notEqual(scheduleId, undefined)
+  assert.notEqual(vmUuid, undefined)
+  // srUuid is nullish for backup
+  assert.equal(Object.keys(other).length, 0)
+
+  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
+    xapi.setFieldEntries(type, ref, 'other_config', {
+      [REPLICATED_TO_SR_UUID]: srUuid,
+      [DATETIME]: formatDateTime(timestamp),
+      [JOB_ID]: jobId,
+      [SCHEDULE_ID]: scheduleId,
+      [VM_UUID]: vmUuid,
+    })
+  )
+}
+/**
+ *
+ * mark the export of he VM and its VDIs as successfull
+ *
+ * @param {Xapi} xapi
+ * @param {String} vmRef
+ * @returns {Promise}
+ */
+export async function markExportSuccessfull(xapi, vmRef) {
+  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
+    xapi.setFieldEntry(type, ref, 'other_config', EXPORTED_SUCCESSFULLY, 'true')
+  )
+}

package/_runners/_forkStreamUnpipe.mjs

@@ -29,6 +29,10 @@ export function forkStreamUnpipe(source) {
     if (source.forks === 0) {
       debug('no more forks, destroying original stream')
       source.destroy(new Error('no more consumers for this stream'))
+    } else {
+      // a combination of stream.unpipe, onReadable and onData may stall stream here
+      // force it to flow again since we're piping it
+      source.resume()
     }
   })
   return fork

package/_runners/_vmRunners/IncrementalXapi.mjs

@@ -1,8 +1,6 @@
 import { asyncEach } from '@vates/async-each'
-import { asyncMap } from '@xen-orchestra/async-map'
 import { createLogger } from '@xen-orchestra/log'
 import { pipeline } from 'node:stream'
-import findLast from 'lodash/findLast.js'
 import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
 import keyBy from 'lodash/keyBy.js'
 import mapValues from 'lodash/mapValues.js'
@@ -15,9 +13,15 @@ import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs
 import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
 import { Task } from '../../Task.mjs'
 import { watchStreamSize } from '../../_watchStreamSize.mjs'
+import {
+  DATETIME,
+  DELTA_CHAIN_LENGTH,
+  EXPORTED_SUCCESSFULLY,
+  setVmDeltaChainLength,
+  markExportSuccessfull,
+} from '../../_otherConfig.mjs'
 
 const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
-
 const noop = Function.prototype
 
 export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
@@ -30,17 +34,14 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
   }
 
   async _copy() {
-    const baseVm = this._baseVm
+    const baseVdis = this._baseVdis
     const vm = this._vm
     const exportedVm = this._exportedVm
-    const fullVdisRequired = this._fullVdisRequired
-
-    const isFull = fullVdisRequired === undefined || fullVdisRequired.size !== 0
 
+    const isFull = Object.values(baseVdis).length === 0 || Object.values(baseVdis).some(_ => !_)
     await this._callWriters(writer => writer.prepare({ isFull }), 'writer.prepare()')
 
-    const deltaExport = await exportIncrementalVm(exportedVm, baseVm, {
-      fullVdisRequired,
+    const deltaExport = await exportIncrementalVm(exportedVm, baseVdis, {
       nbdConcurrency: this._settings.nbdConcurrency,
       preferNbd: this._settings.preferNbd,
     })
@@ -90,18 +91,13 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       'writer.updateUuidAndChain()'
     )
 
-    this._baseVm = exportedVm
-
-    if (baseVm !== undefined) {
-      await exportedVm.update_other_config(
-        'xo:backup:deltaChainLength',
-        String(+(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1)
-      )
-    }
+    if (!isFull) {
+      await setVmDeltaChainLength(this._xapi, exportedVm.$ref, (this._deltaChainLength ?? 0) + 1)
+    } // on a full the delta chain will be null
 
     // not the case if offlineBackup
     if (exportedVm.is_a_snapshot) {
-      await exportedVm.update_other_config('xo:backup:exported', 'true')
+      await markExportSuccessfull(this._xapi, exportedVm.$ref)
     }
 
     const size = Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0)
@@ -119,43 +115,62 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
   async _selectBaseVm() {
     const xapi = this._xapi
 
-    let baseVm = findLast(this._jobSnapshots, _ => 'xo:backup:exported' in _.other_config)
-    if (baseVm === undefined) {
-      debug('no base VM found')
-      return
+    // filter _jobSnapshotVdis to have only the last successfully exported vdi
+    // compute delta chain length
+    // fill baseUuidToSrcVdi with this data
+
+    const exportedVdis = this._jobSnapshotVdis.filter(
+      _ => EXPORTED_SUCCESSFULLY in _.other_config && DATETIME in _.other_config
+    )
+    let lastSuccessfullBackup
+    let lastExportedVdis = []
+    for (const exportedVdi of exportedVdis) {
+      if (lastSuccessfullBackup === undefined || lastSuccessfullBackup < exportedVdi.other_config[DATETIME]) {
+        lastExportedVdis = []
+        lastSuccessfullBackup = exportedVdi.other_config[DATETIME]
+      }
+      if (lastSuccessfullBackup === exportedVdi.other_config[DATETIME]) {
+        lastExportedVdis.push(exportedVdi)
+      }
     }
 
+    this._baseVdis = {}
+    if (lastExportedVdis.length === 0) {
+      debug('no base VDIS found', {
+        jobLength: this._jobSnapshotVdis.length,
+        lastSuccessfullBackup,
+        exportedLength: exportedVdis.length,
+      })
+      return
+    }
+    const deltaChainLength = Math.max(
+      ...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
+    )
     const fullInterval = this._settings.fullInterval
-    const deltaChainLength = +(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1
-    if (!(fullInterval === 0 || fullInterval > deltaChainLength)) {
-      debug('not using base VM becaust fullInterval reached')
+    if ( fullInterval !== 0 && fullInterval < deltaChainLength + 1) {
+      debug('not using base VM because fullInterval reached', { fullInterval, deltaChainLength, eq: fullInterval < deltaChainLength + 1, dc1: deltaChainLength + 1 })
       return
     }
 
     const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
 
-    // resolve full record
-    baseVm = await xapi.getRecord('VM', baseVm.$ref)
-
-    const baseUuidToSrcVdi = new Map()
-    await asyncMap(await baseVm.$getDisks(), async baseRef => {
-      const [baseUuid, snapshotOf] = await Promise.all([
-        xapi.getField('VDI', baseRef, 'uuid'),
-        xapi.getField('VDI', baseRef, 'snapshot_of'),
-      ])
+    const baseUuidToSrcVdiUuid = new Map()
+    for (const lastExportedVdi of lastExportedVdis) {
+      const baseUuid = lastExportedVdi.uuid
+      const snapshotOf = lastExportedVdi.snapshot_of
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
-        baseUuidToSrcVdi.set(baseUuid, srcVdi.uuid)
+        baseUuidToSrcVdiUuid.set(baseUuid, srcVdi.uuid)
       } else {
         debug('ignore snapshot VDI because no longer present on VM', {
           vdi: baseUuid,
         })
       }
-    })
+    }
 
-    const presentBaseVdis = new Map(baseUuidToSrcVdi)
+    const presentBaseVdis = new Map(baseUuidToSrcVdiUuid)
     await this._callWriters(
-      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis, baseVm),
+      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
       'writer.checkBaseVdis()',
       false
     )
@@ -164,24 +179,22 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       debug('no base VM found')
       return
     }
-
-    const fullVdisRequired = new Set()
-    baseUuidToSrcVdi.forEach((srcVdiUuid, baseUuid) => {
+    baseUuidToSrcVdiUuid.forEach((srcVdiUuid, baseUuid) => {
       if (presentBaseVdis.has(baseUuid)) {
         debug('found base VDI', {
           base: baseUuid,
           vdi: srcVdiUuid,
         })
+        this._baseVdis[srcVdiUuid] = lastExportedVdis.find(vdi => vdi.uuid === baseUuid)
       } else {
         debug('missing base VDI', {
           base: baseUuid,
           vdi: srcVdiUuid,
         })
-        fullVdisRequired.add(srcVdiUuid)
+        // this will force a full for this vdi
+        this._baseVdis[srcVdiUuid] = undefined
       }
     })
-
-    this._baseVm = baseVm
-    this._fullVdisRequired = fullVdisRequired
+    this._deltaChainLength = deltaChainLength
   }
 }

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -4,11 +4,18 @@ import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { decorateMethodsWith } from '@vates/decorate-with'
 import { defer } from 'golike-defer'
-import { formatDateTime } from '@xen-orchestra/xapi'
 
 import { getOldEntries } from '../../_getOldEntries.mjs'
 import { Task } from '../../Task.mjs'
 import { Abstract } from './_Abstract.mjs'
+import {
+  DATETIME,
+  JOB_ID,
+  SCHEDULE_ID,
+  populateVdisOtherConfig,
+  resetVmOtherConfig,
+  setVmOtherConfig,
+} from '../../_otherConfig.mjs'
 
 export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
   constructor({
@@ -25,7 +32,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     vm,
   }) {
     super()
-    if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
+    if (vm.other_config[JOB_ID] === job.id && 'start' in vm.blocked_operations) {
       // don't match replicated VMs created by this very job otherwise they
       // will be replicated again and again
       throw new Error('cannot backup a VM created by this very job')
@@ -49,17 +56,17 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     this._exportedVm = undefined
     this._vm = vm
 
-    this._fullVdisRequired = undefined
+    this._baseVdis = undefined
     this._getSnapshotNameLabel = getSnapshotNameLabel
     this._isIncremental = job.mode === 'delta'
     this._healthCheckSr = healthCheckSr
     this._jobId = job.id
-    this._jobSnapshots = undefined
+    this._jobSnapshotVdis = undefined
     this._throttleStream = throttleStream
     this._xapi = vm.$xapi
 
     // Base VM for the export
-    this._baseVm = undefined
+    this._baseVdis = undefined
 
     // Settings for this specific run (job, schedule, VM)
     if (tags.includes('xo-memory-backup')) {
@@ -123,15 +130,8 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
   // copied on manual snapshots and interfere with the backup jobs
   async _cleanMetadata() {
     const vm = this._vm
-    if ('xo:backup:job' in vm.other_config) {
-      await vm.update_other_config({
-        'xo:backup:datetime': null,
-        'xo:backup:deltaChainLength': null,
-        'xo:backup:exported': null,
-        'xo:backup:job': null,
-        'xo:backup:schedule': null,
-        'xo:backup:vm': null,
-      })
+    if (JOB_ID in vm.other_config) {
+      await resetVmOtherConfig(this._xapi, vm.$ref)
     }
   }
 
@@ -146,6 +146,15 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
         if (!settings.bypassVdiChainsCheck) {
           await vm.$assertHealthyVdiChains()
         }
+        if (settings.preferNbd) {
+          try {
+            // enable CBT on all disks if possible
+            const diskRefs = await xapi.VM_getDisks(vm.$ref)
+            await Promise.all(diskRefs.map(diskRef => xapi.call('VDI.enable_cbt', diskRef)))
+          } catch (error) {
+            Task.info(`couldn't enable CBT`, error)
+          }
+        }
 
         const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
           ignoreNobakVdis: true,
@@ -153,16 +162,13 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
           unplugVusbs: true,
         })
         this.timestamp = Date.now()
-
-        await xapi.setFieldEntries('VM', snapshotRef, 'other_config', {
-          'xo:backup:datetime': formatDateTime(this.timestamp),
-          'xo:backup:job': this._jobId,
-          'xo:backup:schedule': this.scheduleId,
-          'xo:backup:vm': vm.uuid,
+        await setVmOtherConfig(xapi, snapshotRef, {
+          timestamp: this.timestamp,
+          jobId: this._jobId,
+          scheduleId: this.scheduleId,
+          vmUuid: vm.uuid,
         })
-
         this._exportedVm = await xapi.getRecord('VM', snapshotRef)
-
         return this._exportedVm.uuid
       })
     } else {
@@ -176,38 +182,116 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     const vmRef = this._vm.$ref
     const xapi = this._xapi
 
-    const snapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
-    const snapshotsOtherConfig = await asyncMap(snapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
+    // to ensure compatibility with snapshots older than CBT implementation
+    // update vdi data to ensure the vdi are correctly fetched in _jobSnapshotVdis
+    // remove by then end of 2024
+    const vmSnapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
+    const vmSnapshotsOtherConfig = await asyncMap(vmSnapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
 
-    const snapshots = []
-    snapshotsOtherConfig.forEach((other_config, i) => {
-      if (other_config['xo:backup:job'] === jobId) {
-        snapshots.push({ other_config, $ref: snapshotsRef[i] })
+    const vmSnapshots = []
+    vmSnapshotsOtherConfig.forEach((other_config, i) => {
+      if (other_config[JOB_ID] === jobId) {
+        vmSnapshots.push({ other_config, $ref: vmSnapshotsRef[i] })
       }
     })
-    snapshots.sort((a, b) => (a.other_config['xo:backup:datetime'] < b.other_config['xo:backup:datetime'] ? -1 : 1))
-    this._jobSnapshots = snapshots
+    await Promise.all(vmSnapshots.map(snapshot => populateVdisOtherConfig(xapi, snapshot.$ref)))
+    // end of compatibiliy handling
+
+    // handle snapshot by VDI
+    this._jobSnapshotVdis = []
+    const srcVdis = await xapi.getRecords('VDI', await this._vm.$getDisks())
+    for (const srcVdi of srcVdis) {
+      const snapshots = await xapi.getRecords('VDI', srcVdi.snapshots)
+      for (const snapshot of snapshots) {
+        if (snapshot.other_config[JOB_ID] === jobId) {
+          this._jobSnapshotVdis.push(snapshot)
+        }
+      }
+    }
   }
 
   async _removeUnusedSnapshots() {
     const allSettings = this.job.settings
     const baseSettings = this._baseSettings
-    const baseVmRef = this._baseVm?.$ref
 
-    const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
+    const snapshotsPerSchedule = groupBy(this._jobSnapshotVdis, _ => _.other_config[SCHEDULE_ID])
     const xapi = this._xapi
-    await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
+    await asyncMap(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
+      const snapshotPerDatetime = groupBy(snapshots, _ => _.other_config[DATETIME])
+
+      const datetimes = Object.keys(snapshotPerDatetime)
+      datetimes.sort()
+
       const settings = {
         ...baseSettings,
         ...allSettings[scheduleId],
         ...allSettings[this._vm.uuid],
       }
-      return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
-        if ($ref !== baseVmRef) {
-          return xapi.VM_destroy($ref)
+      // ensure we never delete the last one
+      const retention = Math.max(settings.snapshotRetention ?? 0, 1)
+
+      await asyncMap(getOldEntries(retention, datetimes), async datetime => {
+        const vdis = snapshotPerDatetime[datetime]
+
+        let vmRef
+        // if there is an attached VM => destroy the VM (Non CBT backups)
+        for (const vdi of vdis) {
+          if (vdi.$VBDs.length > 0) {
+            const vbds = vdi.$VBDs
+            // only one VM linked to this vdi
+            // this will throw error for VDI still attached to control domain
+            assert.strictEqual(vbds.length, 1, 'VDI must be free or attached to exactly one VM')
+            const vm = vbds[0].$VM
+            assert.strictEqual(vm.is_control_domain, false, `Disk is still attached to DOM0 VM`) // don't delete a VM (especially a control domain)
+            assert.strictEqual(vm.is_a_snapshot, true, `VM must be a snapshot`) // don't delete a VM (especially a control domain)
+
+            const vmRefVdi = vm.$ref
+            // same vm than other vdi of the same batch
+            assert.ok(
+              vmRef === undefined || vmRef === vmRefVdi,
+              '_removeUnusedSnapshots don t handle vdi related to multiple VMs '
+            )
+            vmRef = vmRefVdi
+          }
+        }
+        if (vmRef !== undefined) {
+          return xapi.VM_destroy(vmRef)
+        } else {
+          return asyncMap(
+            vdis.map(async ({ $ref }) => {
+              await xapi.VDI_destroy($ref)
+            })
+          )
         }
       })
     })
+
+    // now that we use CBT, we can destroy the data of the snapshot used for this backup
+    // going back to a previous version of XO not supporting CBT will create a full backup
+    // this will only do something after snapshot and transfer
+    if (
+      // don't modify the VM
+      this._exportedVm?.is_a_snapshot &&
+      // user don't want to keep the snapshot data
+      this._settings.snapshotRetention === 0 &&
+      // preferNbd is not a guarantee that the backup used NBD, depending on the network configuration
+      this._settings.preferNbd &&
+      // only delete snapshost data if the config allows it
+      this.config.purgeSnapshotData
+    ) {
+      Task.info('will delete snapshot data')
+      const vdiRefs = await this._xapi.VM_getDisks(this._exportedVm?.$ref)
+      await xapi.call('VM.destroy', this._exportedVm.$ref)
+      for (const vdiRef of vdiRefs) {
+        try {
+          // data_destroy will fail with a VDI_NO_CBT_METADATA error if CBT is not enabled on this VDI
+          await xapi.VDI_dataDestroy(vdiRef)
+          Task.info(`Snapshot data has been deleted`, { vdiRef })
+        } catch (error) {
+          Task.warning(`Couldn't deleted snapshot data`, { error, vdiRef })
+        }
+      }
+    }
   }
 
   async copy() {

package/_runners/_writers/FullXapiWriter.mjs

@@ -1,6 +1,5 @@
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
-import { formatDateTime } from '@xen-orchestra/xapi'
 
 import { formatFilenameDate } from '../../_filenameDate.mjs'
 import { getOldEntries } from '../../_getOldEntries.mjs'
@@ -9,6 +8,7 @@ import { Task } from '../../Task.mjs'
 import { AbstractFullWriter } from './_AbstractFullWriter.mjs'
 import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
 import { listReplicatedVms } from './_listReplicatedVms.mjs'
+import { setVmOtherConfig } from '../../_otherConfig.mjs'
 
 export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
   constructor(props) {
@@ -76,14 +76,12 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
           'Start operation for this vm is blocked, clone it if you want to use it.'
         )
       ),
-      targetVm.update_other_config({
-        'xo:backup:sr': srUuid,
-
-        // these entries need to be added in case of offline backup
-        'xo:backup:datetime': formatDateTime(timestamp),
-        'xo:backup:job': job.id,
-        'xo:backup:schedule': scheduleId,
-        'xo:backup:vm': vm.uuid,
+      setVmOtherConfig(xapi, targetVmRef, {
+        timestamp,
+        jobId: job.id,
+        scheduleId,
+        srUuid,
+        vmUuid: vm.uuid,
       }),
     ])
 

package/_runners/_writers/IncrementalXapiWriter.mjs

@@ -1,39 +1,34 @@
-import assert from 'node:assert'
 import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
-import { formatDateTime } from '@xen-orchestra/xapi'
 
 import { formatFilenameDate } from '../../_filenameDate.mjs'
 import { getOldEntries } from '../../_getOldEntries.mjs'
-import { importIncrementalVm, TAG_BACKUP_SR, TAG_BASE_DELTA, TAG_COPY_SRC } from '../../_incrementalVm.mjs'
+import { importIncrementalVm } from '../../_incrementalVm.mjs'
 import { Task } from '../../Task.mjs'
 
 import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
 import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
 import { listReplicatedVms } from './_listReplicatedVms.mjs'
-import find from 'lodash/find.js'
+import { COPY_OF, setVmOtherConfig, BASE_DELTA_VDI } from '../../_otherConfig.mjs'
 
+import assert from 'node:assert'
 export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
-  async checkBaseVdis(baseUuidToSrcVdi, baseVm) {
-    assert.notStrictEqual(baseVm, undefined)
+  async checkBaseVdis(baseUuidToSrcVdi) {
     const sr = this._sr
-    const replicatedVm = listReplicatedVms(sr.$xapi, this._job.id, sr.uuid, this._vmUuid).find(
-      vm => vm.other_config[TAG_COPY_SRC] === baseVm.uuid
-    )
-    if (replicatedVm === undefined) {
-      return baseUuidToSrcVdi.clear()
-    }
 
-    const xapi = replicatedVm.$xapi
-    const replicatedVdis = new Set(
-      await asyncMap(await replicatedVm.$getDisks(), async vdiRef => {
-        const otherConfig = await xapi.getField('VDI', vdiRef, 'other_config')
-        return otherConfig[TAG_COPY_SRC]
+    // @todo use an index if possible
+    // @todo : this seems similare to decorateVmMetadata
+
+    const replicatedVdis = sr.$VDIs
+      .filter(({ other_config }) => {
+        // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
+        return baseUuidToSrcVdi.has(other_config?.[COPY_OF])
       })
-    )
+      .map(({ other_config }) => other_config?.[COPY_OF])
+      .filter(_ => !!_)
 
     for (const uuid of baseUuidToSrcVdi.keys()) {
-      if (!replicatedVdis.has(uuid)) {
+      if (!replicatedVdis.includes(uuid)) {
         baseUuidToSrcVdi.delete(uuid)
       }
     }
@@ -89,46 +84,38 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
   #decorateVmMetadata(backup) {
     const { _warmMigration } = this._settings
     const sr = this._sr
-    const xapi = sr.$xapi
     const vm = backup.vm
-    vm.other_config[TAG_COPY_SRC] = vm.uuid
-    const remoteBaseVmUuid = vm.other_config[TAG_BASE_DELTA]
-    let baseVm
-    if (remoteBaseVmUuid) {
-      baseVm = find(
-        xapi.objects.all,
-        obj => (obj = obj.other_config) && obj[TAG_COPY_SRC] === remoteBaseVmUuid && obj[TAG_BACKUP_SR] === sr.$id
-      )
-
-      if (!baseVm) {
-        throw new Error(`could not find the base VM (copy of ${remoteBaseVmUuid})`)
-      }
-    }
-    const baseVdis = {}
-    baseVm?.$VBDs.forEach(vbd => {
-      const vdi = vbd.$VDI
-      if (vdi !== undefined) {
-        baseVdis[vbd.VDI] = vbd.$VDI
-      }
-    })
 
-    vm.other_config[TAG_COPY_SRC] = vm.uuid
+    vm.other_config[COPY_OF] = vm.uuid
     if (!_warmMigration) {
       vm.tags.push('Continuous Replication')
     }
+    // extracting the uuid of each delta vdi on the source
+    // get all in one pass, since there is a lot of objects
+    const sourceVdiUuids = Object.values(backup.vdis)
+      .map(({ other_config }) => other_config[BASE_DELTA_VDI])
+      // full vdi don't have a base
+      .filter(_ => !!_)
+    // @todo use index ?
+
+    const replicatedVdis = sr.$VDIs.filter(({ other_config }) => {
+      // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
+      return sourceVdiUuids.includes(other_config?.[COPY_OF])
+    })
 
     Object.values(backup.vdis).forEach(vdi => {
-      vdi.other_config[TAG_COPY_SRC] = vdi.uuid
-      vdi.SR = sr.$ref
-      // vdi.other_config[TAG_BASE_DELTA] is never defined on a suspend vdi
-      if (vdi.other_config[TAG_BASE_DELTA]) {
-        const remoteBaseVdiUuid = vdi.other_config[TAG_BASE_DELTA]
-        const baseVdi = find(baseVdis, vdi => vdi.other_config[TAG_COPY_SRC] === remoteBaseVdiUuid)
-        if (!baseVdi) {
-          throw new Error(`missing base VDI (copy of ${remoteBaseVdiUuid})`)
-        }
-        vdi.baseVdi = baseVdi
+      vdi.other_config[COPY_OF] = vdi.uuid
+      if (sourceVdiUuids.length > 0) {
+        const baseReplicatedTo = replicatedVdis.find(
+          replicatedVdi => replicatedVdi.other_config[COPY_OF] === vdi.other_config[BASE_DELTA_VDI]
+        )
+        assert.notStrictEqual(baseReplicatedTo, undefined)
+        vdi.baseVdi = baseReplicatedTo
+      } else {
+        vdi.baseVdi = undefined
       }
+      // ensure the VDI are created on the target SR
+      vdi.SR = sr.$ref
     })
 
     return backup
@@ -164,14 +151,12 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
           'Start operation for this vm is blocked, clone it if you want to use it.'
         )
       ),
-      targetVm.update_other_config({
-        [TAG_BACKUP_SR]: srUuid,
-
-        // these entries need to be added in case of offline backup
-        'xo:backup:datetime': formatDateTime(timestamp),
-        'xo:backup:job': job.id,
-        'xo:backup:schedule': scheduleId,
-        [TAG_BASE_DELTA]: vm.uuid,
+      setVmOtherConfig(xapi, targetVmRef, {
+        timestamp,
+        jobId: job.id,
+        scheduleId,
+        vmUuid: vm.uuid,
+        srUuid,
       }),
     ])
   }

package/_runners/_writers/_MixinRemoteWriter.mjs

@@ -38,6 +38,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
             },
             lock: false,
             mergeBlockConcurrency: this._config.mergeBlockConcurrency,
+            removeTmp: true,
           })
         })
       } catch (error) {

package/_runners/_writers/_listReplicatedVms.mjs

@@ -1,5 +1,7 @@
+import { DATETIME, JOB_ID, REPLICATED_TO_SR_UUID, SCHEDULE_ID, VM_UUID } from '../../_otherConfig.mjs'
+
 const getReplicatedVmDatetime = vm => {
-  const { 'xo:backup:datetime': datetime = vm.name_label.slice(-17, -1) } = vm.other_config
+  const { [DATETIME]: datetime = vm.name_label.slice(-17, -1) } = vm.other_config
   return datetime
 }
 
@@ -16,11 +18,11 @@ export function listReplicatedVms(xapi, scheduleOrJobId, srUuid, vmUuid) {
       !object.is_a_snapshot &&
       !object.is_a_template &&
       'start' in object.blocked_operations &&
-      (oc['xo:backup:job'] === scheduleOrJobId || oc['xo:backup:schedule'] === scheduleOrJobId) &&
-      oc['xo:backup:sr'] === srUuid &&
-      (oc['xo:backup:vm'] === vmUuid ||
+      (oc[JOB_ID] === scheduleOrJobId || oc[SCHEDULE_ID] === scheduleOrJobId) &&
+      oc[REPLICATED_TO_SR_UUID] === srUuid &&
+      (oc[VM_UUID] === vmUuid ||
         // 2018-03-28, JFT: to catch VMs replicated before this fix
-        oc['xo:backup:vm'] === undefined)
+        oc[VM_UUID] === undefined)
     ) {
       vms[object.$id] = object
     }

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.49.0",
+  "version": "0.50.0",
   "engines": {
     "node": ">=14.18"
   },
@@ -24,7 +24,7 @@
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.1.0",
     "@vates/disposable": "^0.1.5",
-    "@vates/fuse-vhd": "^2.1.0",
+    "@vates/fuse-vhd": "^2.1.1",
     "@vates/nbd-client": "^3.0.2",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
@@ -46,7 +46,7 @@
     "proper-lockfile": "^4.1.2",
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
-    "vhd-lib": "^4.9.2",
+    "vhd-lib": "^4.10.0",
     "xen-api": "^4.0.0",
     "yazl": "^2.5.1"
   },
@@ -58,7 +58,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^6.0.0"
+    "@xen-orchestra/xapi": "^6.1.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {