@xen-orchestra/backups 0.72.0 → 0.73.0

package/ImportVmBackup.mjs

@@ -10,7 +10,7 @@ import { dirname, join } from 'node:path'
 import pickBy from 'lodash/pickBy.js'
 import { defer } from 'golike-defer'
 import { NegativeDisk } from '@xen-orchestra/disk-transform'
-import { openDiskChain } from './disks/openDiskChain.mjs'
+import { openDiskChain } from '@xen-orchestra/backup-archive/disks'
 import { resetVmOtherConfig } from './_otherConfig.mjs'
 
 const { debug, info, warn } = createLogger('xo:backups:importVmBackup')

package/RemoteAdapter.mjs

@@ -22,7 +22,7 @@ import * as tar from 'tar'
 import zlib from 'zlib'
 
 import { BACKUP_DIR } from './_getVmBackupDir.mjs'
-import { cleanVm } from './_cleanVm.mjs'
+import { VmBackupDirectory } from '@xen-orchestra/backup-archive'
 import { formatFilenameDate } from './_filenameDate.mjs'
 import { getTmpDir } from './_getTmpDir.mjs'
 import { isMetadataFile } from './_backupType.mjs'
@@ -31,8 +31,7 @@ import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from '
 import { lvs, pvs } from './_lvm.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
 
-import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
-import { openDiskChain } from './disks/openDiskChain.mjs'
+import { RemoteVhdDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
 import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
 import { ReadAhead } from '@xen-orchestra/disk-transform'
 
@@ -702,7 +701,7 @@ export class RemoteAdapter {
     const handler = this._handler
 
     if (this.useVhdDirectory()) {
-      await writeToVhdDirectory({
+      return await writeToVhdDirectory({
         disk,
         target: {
           handler,
@@ -714,8 +713,9 @@ export class RemoteAdapter {
       })
     } else {
       const stream = await toVhdStream(disk)
-      await this.outputStream(path, stream, { validator, checksum: false })
+      const size = await this.outputStream(path, stream, { validator, checksum: false })
       await validator(path)
+      return size
     }
   }
 
@@ -878,11 +878,14 @@ export class RemoteAdapter {
 }
 
 Object.assign(RemoteAdapter.prototype, {
-  cleanVm(vmDir, { lock = true } = {}) {
+  cleanVm(vmBackupPath, opts = {}) {
+    const { lock = true, ...cleanOpts } = opts
     if (lock) {
-      return Disposable.use(this._handler.lock(vmDir), () => cleanVm.apply(this, arguments))
+      return Disposable.use(this._handler.lock(vmBackupPath), () => {
+        return VmBackupDirectory.cleanVm(this._handler, vmBackupPath, cleanOpts)
+      })
     } else {
-      return cleanVm.apply(this, arguments)
+      return VmBackupDirectory.cleanVm(this._handler, vmBackupPath, cleanOpts)
     }
   },
   isValidXva,

package/_otherConfig.mjs

@@ -85,6 +85,7 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
 export function resetVmOtherConfig(xapi, vmRef) {
   return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
     return xapi.setFieldEntries(type, ref, 'other_config', {
+      [CONTENT_KEY]: null,
       [COPY_OF]: null,
       [DATETIME]: null,
       [DELTA_CHAIN_LENGTH]: null,
@@ -92,6 +93,7 @@ export function resetVmOtherConfig(xapi, vmRef) {
       [JOB_ID]: null,
       [SCHEDULE_ID]: null,
       [VM_UUID]: null,
+
       // REPLICATED_TO_SR_UUID is not reset since we can replicate a replication
     })
   })

package/_runners/_vmRunners/IncrementalXapi.mjs

@@ -8,6 +8,7 @@ import { exportIncrementalVm } from '../../_incrementalVm.mjs'
 import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
 import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
 import {
+  CONTENT_KEY,
   DATETIME,
   DELTA_CHAIN_LENGTH,
   EXPORTED_SUCCESSFULLY,
@@ -101,63 +102,136 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
     await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
   }
 
+  /**
+   * Groups snapshot VDIs by their DATETIME other_config value and sorts them most-recent-first.
+   *
+   * @param {import('@vates/types').XenApiVdi[]} snapshotVdis
+   * @returns {{ datetime: string, vdis: import('@vates/types').XenApiVdi[] }[]}
+   */
+  _groupSnapshotVdisByDatetime(snapshotVdis) {
+    const byDatetime = new Map()
+    for (const vdi of snapshotVdis) {
+      const datetime = vdi.other_config[DATETIME]
+      if (!byDatetime.has(datetime)) {
+        byDatetime.set(datetime, [])
+      }
+      byDatetime.get(datetime).push(vdi)
+    }
+    // sort them by decreasing datetime
+    return [...byDatetime.entries()]
+      .sort(([a], [b]) => b.localeCompare(a))
+      .map(([datetime, vdis]) => ({ datetime, vdis }))
+  }
+
+  /**
+   * For each snapshot VDI in `snapshotVdis`, maps it to its live source VDI,
+   * attaches its CONTENT_KEY when present, calls checkBaseVdis on all writers,
+   * and returns both the full mapping and the subset confirmed present by writers.
+   *
+   * @param {import('@vates/types').XenApiVdi[]} snapshotVdis - Snapshot VDI records to use as base candidates
+   * @param {Record<string, object>} srcVdisByRef - Live VM VDI records keyed by $ref
+   * @param {{ useContentKey?: boolean }} [options]
+   * @returns {Promise<{ presentBaseVdis: Map<string, string>, baseUuidToSrcVdiUuid: Map<string, string>}>}
+   */
+  async _checkBaseVdis(snapshotVdis, srcVdisByRef, { useContentKey = false } = {}) {
+    const baseUuidToSrcVdiUuid = new Map()
+    const baseUuidToContentKey = new Map()
+    for (const snapshotVdi of snapshotVdis) {
+      const baseUuid = snapshotVdi.uuid
+      const srcVdi = srcVdisByRef[snapshotVdi.snapshot_of]
+      if (srcVdi !== undefined) {
+        baseUuidToSrcVdiUuid.set(baseUuid, srcVdi.uuid)
+        if (useContentKey) {
+          const contentKey = snapshotVdi.other_config[CONTENT_KEY]
+          if (contentKey !== undefined) {
+            baseUuidToContentKey.set(baseUuid, contentKey)
+          }
+        }
+      } else {
+        debug('ignore snapshot VDI because no longer present on VM', { vdi: baseUuid })
+      }
+    }
+
+    const presentBaseVdis = new Map(baseUuidToSrcVdiUuid)
+    await this._callWriters(
+      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis, baseUuidToContentKey),
+      'writer.checkBaseVdis()',
+      false
+    )
+    return { presentBaseVdis, baseUuidToSrcVdiUuid }
+  }
+
   async _selectBaseVm() {
     const xapi = this._xapi
 
-    // filter _jobSnapshotVdis to have only the last successfully exported vdi
-    // compute delta chain length
-    // fill baseUuidToSrcVdi with this data
-
     const exportedVdis = this._jobSnapshotVdis.filter(
       _ => EXPORTED_SUCCESSFULLY in _.other_config && DATETIME in _.other_config
     )
-    let lastSuccessfullBackup
-    let lastExportedVdis = []
-    for (const exportedVdi of exportedVdis) {
-      if (lastSuccessfullBackup === undefined || lastSuccessfullBackup < exportedVdi.other_config[DATETIME]) {
-        lastExportedVdis = []
-        lastSuccessfullBackup = exportedVdi.other_config[DATETIME]
-      }
-      if (lastSuccessfullBackup === exportedVdi.other_config[DATETIME]) {
-        lastExportedVdis.push(exportedVdi)
-      }
-    }
+    const grouped = this._groupSnapshotVdisByDatetime(exportedVdis)
 
     this._baseVdis = {}
-    if (lastExportedVdis.length === 0) {
-      debug('no base VDIS found', {
+    if (grouped.length === 0) {
+      debug('no base VDIs found', {
         jobLength: this._jobSnapshotVdis.length,
-        lastSuccessfullBackup,
         exportedLength: exportedVdis.length,
       })
-      return
     }
-    const deltaChainLength = Math.max(
-      ...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
-    )
-
     const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
+    let lastExportedVdis = []
+    let presentBaseVdis = new Map()
+    let baseUuidToSrcVdiUuid = new Map()
+    let deltaChainLength
 
-    const baseUuidToSrcVdiUuid = new Map()
-    for (const lastExportedVdi of lastExportedVdis) {
-      const baseUuid = lastExportedVdi.uuid
-      const snapshotOf = lastExportedVdi.snapshot_of
-      const srcVdi = srcVdis[snapshotOf]
-      if (srcVdi !== undefined) {
-        baseUuidToSrcVdiUuid.set(baseUuid, srcVdi.uuid)
-      } else {
-        debug('ignore snapshot VDI because no longer present on VM', {
-          vdi: baseUuid,
-        })
-      }
+    if (grouped.length > 0) {
+      lastExportedVdis = grouped[0].vdis
+      // only compute it on the _jobSnapshotVdis , not on the reused snapshot from CONTENT_KEYS
+      deltaChainLength = Math.max(
+        ...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
+      )
+      ;({ presentBaseVdis, baseUuidToSrcVdiUuid } = await this._checkBaseVdis(lastExportedVdis, srcVdis))
     }
 
-    const presentBaseVdis = new Map(baseUuidToSrcVdiUuid)
-    await this._callWriters(
-      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
-      'writer.checkBaseVdis()',
-      false
-    )
+    // if we don't find any candidates
+    // if there is only one writer of type IncrementalXapiWriter
+    // allow the reuse of any matching snapshot of this VM even from another job
+    if (
+      presentBaseVdis.size === 0 &&
+      this._writers.size === 1 &&
+      [...this._writers][0] instanceof IncrementalXapiWriter
+    ) {
+      debug('fallback to content key')
+      const candidates = {}
+      for (const srcVdi of Object.values(srcVdis)) {
+        const snapshots = srcVdi.$snapshots
+        debug(`${srcVdi.uuid} got ${snapshots.length} snapshots`)
+        for (const snapshot of snapshots) {
+          if (DATETIME in snapshot.other_config) {
+            candidates[snapshot.uuid] = snapshot
+          }
+        }
+      }
+
+      debug('got candidates ', Object.keys(candidates).length)
+      this._filterValidSnapshotVdis(candidates)
+      debug('got filtered  ', Object.keys(candidates).length)
+      for (const group of this._groupSnapshotVdisByDatetime(Object.values(candidates))) {
+        debug('check by datetime   ', [...group.vdis].length)
+        debug(group.vdis)
+        const result = await this._checkBaseVdis(group.vdis, srcVdis, { useContentKey: true })
+        debug('result ', result)
+        if (result.presentBaseVdis.size > 0) {
+          lastExportedVdis = group.vdis
+
+          presentBaseVdis = result.presentBaseVdis
+          baseUuidToSrcVdiUuid = result.baseUuidToSrcVdiUuid
+
+          break
+        }
+        debug('try next group if any')
+      }
+    } else {
+      debug(' no fallback ')
+    }
 
     if (presentBaseVdis.size === 0) {
       debug('no base VM found')

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -236,28 +236,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
   //   ensure they are attached to only one vm snapshot
   //   ensure any VM-snapshot harvested by this has all its disk harvested (no mix of vdi snapshot from this job and not)
 
-  async _fetchJobSnapshots() {
-    const jobId = this._jobId
-    const xapi = this._xapi
-
-    const vdiCandidates = {}
-    const vdiUuids = this._vm.$VBDs.map(({ VDI }) => VDI)
-    Object.values(xapi.objects.indexes.type.VDI)
-      .filter(_ => !!_) // filter nullish
-      .filter(({ other_config, snapshot_of }) => {
-        return (
-          vdiUuids.includes(snapshot_of) &&
-          other_config[JOB_ID] === jobId &&
-          other_config[VM_UUID] === this._vm.uuid &&
-          other_config[COPY_OF] === undefined
-        )
-      })
-      .forEach(vdi => {
-        vdiCandidates[vdi.uuid] = vdi
-      })
-
-    // check that user snapshots are clean
-
+  _filterValidSnapshotVdis(vdiCandidates) {
     for (const vdi of Object.values(vdiCandidates)) {
       // cbt metadata are always considered linked to a backup job
       // if they have the right other_config
@@ -279,7 +258,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       const userVms = vbds.map(({ $VM }) => $VM)
       if (vbds.length > 1) {
         warn(
-          `vdi ${vdi.name_label} (${vdi.uuid}) is linked to multipe vms :  ${userVms.map(({ name_label, uuid }) => `${name_label} ${uuid}`).join(', ')}. 
+          `vdi ${vdi.name_label} (${vdi.uuid}) is linked to multipe vms :  ${userVms.map(({ name_label, uuid }) => `${name_label} ${uuid}`).join(', ')}.
               This disk snapshot will be excluded from the backup cleaning`,
           { vdi, userVms }
         )
@@ -294,7 +273,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       // => we exclude these from the backup processing
       if (vm.$snapshot_of === undefined) {
         warn(
-          `vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a non snapshot vm ${vm.name_label} ${vm.uuid}. 
+          `vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a non snapshot vm ${vm.name_label} ${vm.uuid}.
           This disk snapshot will be excluded from the backup cleaning`,
           { vdi, vm }
         )
@@ -311,7 +290,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
         vm.other_config[VM_UUID] !== vdi.other_config[VM_UUID]
       ) {
         warn(
-          `vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a snapshot vm ${vm.name_label} ${vm.uuid} out of this backup job scope. 
+          `vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a snapshot vm ${vm.name_label} ${vm.uuid} out of this backup job scope.
           This disk snapshot will be excluded from the backup cleaning`,
           { vdi, vm }
         )
@@ -328,7 +307,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
         .forEach(({ $VDI: outOfSnapshotsVdi, ...other }) => {
           warn(
             `vdi ${vdi.name_label} ${vdi.uuid} is recognized as a snapshot of the backup job,
-           linked to vm ${vm.name_label} ${vm.uuid} but vdi ${outOfSnapshotsVdi.name_label} ${outOfSnapshotsVdi.uuid} 
+           linked to vm ${vm.name_label} ${vm.uuid} but vdi ${outOfSnapshotsVdi.name_label} ${outOfSnapshotsVdi.uuid}
            is not linked to the job. This disk snapshot will be excluded from the backup cleaning`,
             { vdi, vm, vbds, outOfSnapshotsVdi }
           )
@@ -336,8 +315,47 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
           delete vdiCandidates[vdi.uuid]
         })
     }
+  }
+
+  async _fetchJobSnapshots() {
+    const jobId = this._jobId
+    const xapi = this._xapi
+
+    const vdiCandidates = {}
+    const vdiUuids = this._vm.$VBDs.map(({ VDI }) => VDI)
+    Object.values(xapi.objects.indexes.type.VDI)
+      .filter(_ => !!_) // filter nullish
+      .filter(({ other_config, snapshot_of }) => {
+        return (
+          vdiUuids.includes(snapshot_of) &&
+          other_config[JOB_ID] === jobId &&
+          other_config[VM_UUID] === this._vm.uuid &&
+          other_config[COPY_OF] === undefined
+        )
+      })
+      .forEach(vdi => {
+        vdiCandidates[vdi.uuid] = vdi
+      })
+
+    this._filterValidSnapshotVdis(vdiCandidates)
 
     this._jobSnapshotVdis = Object.values(vdiCandidates)
+
+    // For VMs with no disks, retention must be tracked directly on VM snapshots
+    // since there are no VDIs to anchor the discovery.
+    if (vdiUuids.length === 0) {
+      this._disklessJobSnapshotVms = this._vm.$snapshots
+        .filter(Boolean)
+        .filter(
+          ({ other_config, $snapshot_of }) =>
+            $snapshot_of !== undefined &&
+            other_config[JOB_ID] === jobId &&
+            other_config[VM_UUID] === this._vm.uuid &&
+            other_config[COPY_OF] === undefined
+        )
+    } else {
+      this._disklessJobSnapshotVms = []
+    }
   }
 
   async _removeUnusedSnapshots() {
@@ -349,9 +367,10 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     await xapi.barrier()
     // ensure cached object are up to date
     this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
+    const disklessVmSnapshots = this._disklessJobSnapshotVms.map(vm => xapi.getObject(vm.$ref))
 
-    // get the datetime of the most recent snapshot
-    const lastSnapshotDateTime = this._jobSnapshotVdis
+    // get the datetime of the most recent snapshot across both VDI and diskless VM snapshots
+    const lastSnapshotDateTime = [...this._jobSnapshotVdis, ...disklessVmSnapshots]
       .map(({ other_config }) => other_config[DATETIME])
       .sort()
       .pop()
@@ -428,6 +447,28 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       })
     })
 
+    // Retention for VMs with no disks: VM snapshots are not reachable via VDIs
+    if (disklessVmSnapshots.length > 0) {
+      const snapshotsPerSchedule = groupBy(disklessVmSnapshots, _ => _.other_config[SCHEDULE_ID])
+      await asyncEach(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
+        // we only have one snapshot per date time since it's at the VM level
+        const snapshotPerDatetime = Object.fromEntries(snapshots.map(s => [s.other_config[DATETIME], s.$ref]))
+        const datetimes = Object.keys(snapshotPerDatetime).sort()
+        const settings = {
+          ...baseSettings,
+          ...allSettings[scheduleId],
+          ...allSettings[this._vm.uuid],
+        }
+        const retention = settings.snapshotRetention ?? 0
+        await asyncEach(getOldEntries(retention, datetimes), async datetime => {
+          if (this.job.mode === 'delta' && datetime === lastSnapshotDateTime) {
+            return
+          }
+          await xapi.VM_destroy(snapshotPerDatetime[datetime])
+        })
+      })
+    }
+
     // list and remove the snapshot were the jobs failed between
     // makesnapshot and update_other_config
     const snapshots = this._vm.$snapshots.filter(_ => !!_).filter(({ name_label }) => name_label === TEMP_SNAPSHOT_NAME)

package/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -229,14 +229,13 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
         Object.entries(deltaExport.disks),
         async ([diskRef, disk]) => {
           const path = `${this._vmBackupDir}/${vhds[diskRef]}`
-          await adapter.writeVhd(path, disk, {
+          size += await adapter.writeVhd(path, disk, {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chains
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
             writeBlockConcurrency: this._config.writeBlockConcurrency,
           })
-          size = size + disk.getNbGeneratedBlock() * disk.getBlockSize()
         },
         {
           concurrency: settings.diskPerVmConcurrency,