@xen-orchestra/backups 0.53.1 → 0.54.1

package/HealthCheckVmBackup.mjs

@@ -49,26 +49,25 @@ export class HealthCheckVmBackup {
         }
 
         // wait for the 'Running' event to be really stored in local xapi object cache
+
         restoredVm = await xapi.waitObjectState(restoredVm.$ref, vm => vm.power_state === 'Running', {
           timeout: remainingTimeout,
+          timeoutMessage: refOrUuid =>
+            `local xapi  did not get Running state for VM ${refOrUuid} after ${timeout / 1000} second`,
         })
 
         const running = new Date()
         remainingTimeout -= running - started
 
-        if (remainingTimeout < 0) {
-          throw new Error(`local xapi  did not get Running state for VM ${restoredId} after ${timeout / 1000} second`)
-        }
         // wait for the guest tool version to be defined
         await xapi.waitObjectState(restoredVm.guest_metrics, gm => gm?.PV_drivers_version?.major !== undefined, {
           timeout: remainingTimeout,
+          timeoutMessage: refOrUuid =>
+            `timeout reached while waiting for ${refOrUuid} to report the driver version through the Xen tools. Please check or update the Xen tools.`,
         })
 
         const guestToolsReady = new Date()
         remainingTimeout -= guestToolsReady - running
-        if (remainingTimeout < 0) {
-          throw new Error(`local xapi  did not get he guest tools check ${restoredId} after ${timeout / 1000} second`)
-        }
 
         if (waitForScript) {
           const startedRestoredVm = await xapi.waitObjectState(
@@ -79,19 +78,10 @@ export class HealthCheckVmBackup {
                 vm.xenstore_data['vm-data/xo-backup-health-check'] === 'failure'),
             {
               timeout: remainingTimeout,
+              timeoutMessage: refOrUuid =>
+                `timeout reached while waiting for ${refOrUuid} to report the startup script execution.`,
             }
           )
-          const scriptOk = new Date()
-          remainingTimeout -= scriptOk - guestToolsReady
-          if (remainingTimeout < 0) {
-            throw new Error(
-              `Backup health check script did not update vm-data/xo-backup-health-check of ${restoredId} after ${
-                timeout / 1000
-              } second, got ${
-                startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check']
-              } instead of 'success' or 'failure'`
-            )
-          }
 
           if (startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check'] !== 'success') {
             const message = startedRestoredVm.xenstore_data['vm-data/xo-backup-health-check-error']

package/RemoteAdapter.mjs

@@ -18,6 +18,7 @@ import fromEvent from 'promise-toolbox/fromEvent'
 import groupBy from 'lodash/groupBy.js'
 import pDefer from 'promise-toolbox/defer'
 import pickBy from 'lodash/pickBy.js'
+import reduce from 'lodash/reduce.js'
 import tar from 'tar'
 import zlib from 'zlib'
 
@@ -826,6 +827,29 @@ export class RemoteAdapter {
     }
     return metadata
   }
+
+  #computeTotalBackupSizeRecursively(backups) {
+    return reduce(
+      backups,
+      (prev, backup) => {
+        const _backup = Array.isArray(backup) ? this.#computeTotalBackupSizeRecursively(backup) : backup
+        return {
+          onDisk: prev.onDisk + (_backup.onDisk ?? _backup.size),
+        }
+      },
+      { onDisk: 0 }
+    )
+  }
+
+  async getTotalVmBackupSize() {
+    return this.#computeTotalBackupSizeRecursively(await this.listAllVmBackups())
+  }
+
+  async getTotalBackupSize() {
+    const vmBackupSize = await this.getTotalVmBackupSize()
+    // @TODO: add `getTotalXoBackupSize` and `getTotalPoolBackupSize` once `size` is implemented by fs
+    return vmBackupSize
+  }
 }
 
 Object.assign(RemoteAdapter.prototype, {

package/_cleanVm.mjs

@@ -121,7 +121,19 @@ export async function checkAliases(
 ) {
   const aliasFound = []
   for (const alias of aliasPaths) {
-    const target = await resolveVhdAlias(handler, alias)
+    let target
+    try {
+      target = await resolveVhdAlias(handler, alias)
+    } catch (err) {
+      if (err.code === 'ENOENT') {
+        logWarn('missing target of alias', { alias })
+        if (remove) {
+          logInfo('removing alias and non VHD target', { alias, target })
+          await handler.unlink(target)
+          await handler.unlink(alias)
+        }
+      }
+    }
 
     if (!isVhdFile(target)) {
       logWarn('alias references non VHD target', { alias, target })
@@ -201,9 +213,9 @@ export async function cleanVm(
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {
-    if(removeTmp && basename(path)[0] === '.'){
+    if (removeTmp && basename(path)[0] === '.') {
       logInfo('deleting temporary VHD', { path })
-        return VhdAbstract.unlink(handler, path)
+      return VhdAbstract.unlink(handler, path)
     }
     try {
       await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {

package/_incrementalVm.mjs

@@ -3,7 +3,6 @@ import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { CancelToken } from 'promise-toolbox'
 import { compareVersions } from 'compare-versions'
-import { createVhdStreamWithLength } from 'vhd-lib'
 import { defer } from 'golike-defer'
 
 import { cancelableMap } from './_cancelableMap.mjs'
@@ -53,14 +52,32 @@ export async function exportIncrementalVm(
       $snapshot_of$uuid: vdi.$snapshot_of?.uuid,
       $SR$uuid: vdi.$SR.uuid,
     }
-
-    streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
-      baseRef: baseVdi?.$ref,
-      cancelToken,
-      format: 'vhd',
-      nbdConcurrency,
-      preferNbd,
-    })
+    try {
+      streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
+        baseRef: baseVdi?.$ref,
+        cancelToken,
+        format: 'vhd',
+        nbdConcurrency,
+        preferNbd,
+      })
+    } catch (err) {
+      if (err.code === 'VDI_CANT_DO_DELTA') {
+        // fall back to a base
+        Task.info(`Can't do delta, will try to get a full stream`, { vdi })
+        streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
+          cancelToken,
+          format: 'vhd',
+          nbdConcurrency,
+          preferNbd,
+        })
+        // only warn if the fall back succeed
+        Task.warning(`Can't do delta with this vdi, transfer will be a full`, {
+          vdi,
+        })
+      } else {
+        throw err
+      }
+    }
   })
 
   const suspendVdi = vm.$suspend_VDI
@@ -227,9 +244,6 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
         if (typeof stream === 'function') {
           stream = await stream()
         }
-        if (stream.length === undefined) {
-          stream = await createVhdStreamWithLength(stream)
-        }
         await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
         await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
         await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)

package/_runners/_vmRunners/IncrementalRemote.mjs

@@ -1,3 +1,5 @@
+import { createLogger } from '@xen-orchestra/log'
+
 import { asyncEach } from '@vates/async-each'
 import assert from 'node:assert'
 import * as UUID from 'uuid'
@@ -11,6 +13,7 @@ import { Disposable } from 'promise-toolbox'
 import { openVhd } from 'vhd-lib'
 import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
 
+const { warn } = createLogger('xo:backups:Incrementalremote')
 class IncrementalRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
     return IncrementalRemoteWriter
@@ -46,11 +49,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
     })
 
     const presentBaseVdis = new Map(baseUuidToSrcVdi)
-    await this._callWriters(
-      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis),
-      'writer.checkBaseVdis()',
-      false
-    )
+    await this._callWriters(writer => writer.checkBaseVdis(presentBaseVdis), 'writer.checkBaseVdis()', false)
     // check if the parent vdi are present in all the remotes
     baseUuidToSrcVdi.forEach((srcVdiUuid, baseUuid) => {
       if (!presentBaseVdis.has(baseUuid)) {
@@ -64,17 +63,29 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
 
     for (const metadata of transferList) {
       assert.strictEqual(metadata.mode, 'delta')
-      await this._selectBaseVm(metadata)
-      await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
       const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
         useChain: false,
       })
-
+      // don't trust metadata too much
+      // recompute if it's a base backup
+      // recompute if disks are differencing or not
       const isVhdDifferencing = {}
 
       await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
         isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
       })
+      const hasDifferencingDisk = Object.values(isVhdDifferencing).includes(true)
+      if (metadata.isBase === hasDifferencingDisk) {
+        warn(`Metadata isBase and real disk value are different`, {
+          metadataIsBase: metadata.isBase,
+          diskIsBase: !hasDifferencingDisk,
+          isVhdDifferencing,
+        })
+      }
+      metadata.isBase = !hasDifferencingDisk
+      metadata.isVhdDifferencing = isVhdDifferencing
+      await this._selectBaseVm(metadata)
+      await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
 
       incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
       await this._callWriters(

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -203,7 +203,13 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     for (const srcVdi of srcVdis) {
       const snapshots = await xapi.getRecords('VDI', srcVdi.snapshots)
       for (const snapshot of snapshots) {
-        if (snapshot.other_config[JOB_ID] === jobId) {
+        // only keep the snapshot related to this backup job
+        // and only if the job is still using  purge snapshot data or if the disk
+        // is not a cbt metadata disk ( expect a type: user for normal disks)
+        if (
+          snapshot.other_config[JOB_ID] === jobId &&
+          (this._settings.cbtDestroySnapshotData || snapshot.type !== 'cbt_metadata')
+        ) {
           this._jobSnapshotVdis.push(snapshot)
         }
       }
@@ -219,6 +225,14 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     await xapi.barrier()
     // ensure cached object are up to date
     this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
+
+    // get the datetime of the most recent snapshot
+    const lastSnapshotDateTime = this._jobSnapshotVdis
+      .map(({ other_config }) => other_config[DATETIME])
+      .sort()
+      .pop()
+
+    // remove older snapshot schedule per schedule
     const snapshotsPerSchedule = groupBy(this._jobSnapshotVdis, _ => _.other_config[SCHEDULE_ID])
     await asyncMap(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
       const snapshotPerDatetime = groupBy(snapshots, _ => _.other_config[DATETIME])
@@ -231,10 +245,13 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
         ...allSettings[scheduleId],
         ...allSettings[this._vm.uuid],
       }
-      // ensure we never delete the last one for delta
-      const minRetention = this.job.mode === 'delta' ? 1 : 0
-      const retention = Math.max(settings.snapshotRetention ?? 0, minRetention)
+      const retention = settings.snapshotRetention ?? 0
       await asyncMap(getOldEntries(retention, datetimes), async datetime => {
+        // keep the last snapshot across all schedules for delta
+        // since we'll need it to compute delta for next backup
+        if (this.job.mode === 'delta' && datetime === lastSnapshotDateTime) {
+          return
+        }
         const vdis = snapshotPerDatetime[datetime]
         let vmRef
         // if there is an attached VM => destroy the VM (Non CBT backups)
@@ -293,7 +310,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
           await this._xapi.VDI_dataDestroy(vdiRef)
           Task.info(`Snapshot data has been deleted`, { vdiRef })
         } catch (error) {
-          Task.warning(`Couldn't deleted snapshot data`, { error, vdiRef })
+          Task.warning(`Couldn't delete snapshot data`, { error, vdiRef })
         }
       }
     }

package/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -66,7 +66,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
 
   async beforeBackup() {
     await super.beforeBackup()
-    return this._cleanVm({ merge: true })
+    return this._cleanVm({ merge: true, remove: true })
   }
 
   prepare({ isFull }) {
@@ -149,6 +149,9 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
         assert.notStrictEqual(parentPath, undefined, 'A differential VHD must have a parent')
         // forbid any kind of loop
         assert.ok(basename(parentPath) < basename(path), `vhd must be sorted to be chained`)
+        // re-chainVhd is mandatory
+        // since the parent may be a alias or not
+        // and the child may be the other
         await chainVhd(handler, parentPath, handler, path)
       }
 

package/_runners/_writers/IncrementalXapiWriter.mjs

@@ -15,11 +15,15 @@ import assert from 'node:assert'
 export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
   async checkBaseVdis(baseUuidToSrcVdi) {
     const sr = this._sr
+    if (baseUuidToSrcVdi.size === 0) {
+      // searching for the vdis is expensive
+      // don't do it if there is nothing to find
+      return
+    }
 
     // @todo use an index if possible
     // @todo : this seems similare to decorateVmMetadata
-
-    const replicatedVdis = sr.$VDIs 
+    const replicatedVdis = sr.$VDIs
       .filter(vdi => {
         // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
         return baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
@@ -103,11 +107,10 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
       .filter(_ => !!_)
     // @todo use index ?
 
-    const replicatedVdis = sr.$VDIs
-      .filter(vdi => {
-        // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
-        return sourceVdiUuids.includes(vdi?.other_config[COPY_OF])
-      })
+    const replicatedVdis = sr.$VDIs.filter(vdi => {
+      // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
+      return sourceVdiUuids.includes(vdi?.other_config[COPY_OF])
+    })
 
     Object.values(backup.vdis).forEach(vdi => {
       vdi.other_config[COPY_OF] = vdi.uuid

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.53.1",
+  "version": "0.54.1",
   "engines": {
     "node": ">=14.18"
   },
@@ -23,13 +23,13 @@
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.1.0",
-    "@vates/disposable": "^0.1.5",
-    "@vates/fuse-vhd": "^2.1.1",
-    "@vates/nbd-client": "^3.1.0",
+    "@vates/disposable": "^0.1.6",
+    "@vates/fuse-vhd": "^2.1.2",
+    "@vates/nbd-client": "^3.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^4.1.7",
-    "@xen-orchestra/log": "^0.6.0",
+    "@xen-orchestra/fs": "^4.2.0",
+    "@xen-orchestra/log": "^0.7.0",
     "@xen-orchestra/template": "^0.1.0",
     "app-conf": "^3.0.0",
     "compare-versions": "^6.0.0",
@@ -47,8 +47,8 @@
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^4.11.0",
-    "xen-api": "^4.2.0",
+    "vhd-lib": "^4.11.1",
+    "xen-api": "^4.4.0",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -59,7 +59,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^7.4.0"
+    "@xen-orchestra/xapi": "^7.6.1"
   },
   "license": "AGPL-3.0-or-later",
   "author": {