@xen-orchestra/backups 0.44.1 → 0.44.3

package/ImportVmBackup.mjs

@@ -4,7 +4,14 @@ import { formatFilenameDate } from './_filenameDate.mjs'
 import { importIncrementalVm } from './_incrementalVm.mjs'
 import { Task } from './Task.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
+import { VhdNegative, VhdSynthetic } from 'vhd-lib'
+import { decorateClass } from '@vates/decorate-with'
+import { createLogger } from '@xen-orchestra/log'
+import { dirname, join } from 'node:path'
+import pickBy from 'lodash/pickBy.js'
+import { defer } from 'golike-defer'
 
+const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
 async function resolveUuid(xapi, cache, uuid, type) {
   if (uuid == null) {
     return uuid
@@ -21,17 +28,181 @@ export class ImportVmBackup {
     metadata,
     srUuid,
     xapi,
-    settings: { additionnalVmTag, newMacAddresses, mapVdisSrs = {} } = {},
+    settings: { additionnalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
   }) {
     this._adapter = adapter
-    this._importIncrementalVmSettings = { additionnalVmTag, newMacAddresses, mapVdisSrs }
+    this._importIncrementalVmSettings = { additionnalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
     this._metadata = metadata
     this._srUuid = srUuid
     this._xapi = xapi
   }
 
-  async #decorateIncrementalVmMetadata(backup) {
-    const { additionnalVmTag, mapVdisSrs } = this._importIncrementalVmSettings
+  async #getPathOfVdiSnapshot(snapshotUuid) {
+    const metadata = this._metadata
+    if (this._pathToVdis === undefined) {
+      const backups = await this._adapter.listVmBackups(
+        this._metadata.vm.uuid,
+        ({ mode, timestamp }) => mode === 'delta' && timestamp >= metadata.timestamp
+      )
+      const map = new Map()
+      for (const backup of backups) {
+        for (const [vdiRef, vdi] of Object.entries(backup.vdis)) {
+          map.set(vdi.uuid, backup.vhds[vdiRef])
+        }
+      }
+      this._pathToVdis = map
+    }
+    return this._pathToVdis.get(snapshotUuid)
+  }
+
+  async _reuseNearestSnapshot($defer, ignoredVdis) {
+    const metadata = this._metadata
+    const { mapVdisSrs } = this._importIncrementalVmSettings
+    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
+    const streams = {}
+    const metdataDir = dirname(metadata._filename)
+    const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
+
+    for (const [vdiRef, vdi] of Object.entries(vdis)) {
+      const vhdPath = join(metdataDir, vhds[vdiRef])
+
+      let xapiDisk
+      try {
+        xapiDisk = await this._xapi.getRecordByUuid('VDI', vdi.$snapshot_of$uuid)
+      } catch (err) {
+        // if this disk is not present anymore, fall back to default restore
+        warn(err)
+      }
+
+      let snapshotCandidate, backupCandidate
+      if (xapiDisk !== undefined) {
+        debug('found disks, wlll search its snapshots', { snapshots: xapiDisk.snapshots })
+        for (const snapshotRef of xapiDisk.snapshots) {
+          const snapshot = await this._xapi.getRecord('VDI', snapshotRef)
+          debug('handling snapshot', { snapshot })
+
+          // take only the first snapshot
+          if (snapshotCandidate && snapshotCandidate.snapshot_time < snapshot.snapshot_time) {
+            debug('already got a better candidate')
+            continue
+          }
+
+          // have a corresponding backup more recent than metadata ?
+          const pathToSnapshotData = await this.#getPathOfVdiSnapshot(snapshot.uuid)
+          if (pathToSnapshotData === undefined) {
+            debug('no backup linked to this snaphot')
+            continue
+          }
+          if (snapshot.$SR.uuid !== (mapVdisSrs[vdi.$snapshot_of$uuid] ?? this._srUuid)) {
+            debug('not restored on the same SR', { snapshotSr: snapshot.$SR.uuid, mapVdisSrs, srUuid: this._srUuid })
+            continue
+          }
+
+          debug('got a candidate', pathToSnapshotData)
+
+          snapshotCandidate = snapshot
+          backupCandidate = pathToSnapshotData
+        }
+      }
+
+      let stream
+      const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
+      if (vhdPath === backupWithSnapshotPath) {
+        // all the data are already on the host
+        debug('direct reuse of a snapshot')
+        stream = null
+        vdis[vdiRef].baseVdi = snapshotCandidate
+        // go next disk , we won't use this stream
+        continue
+      }
+
+      let disposableDescendants
+
+      const disposableSynthetic = await VhdSynthetic.fromVhdChain(this._adapter._handler, vhdPath)
+
+      // this will also clean if another disk of this VM backup fails
+      // if user really only need to restore non failing disks he can retry with ignoredVdis
+      let disposed = false
+      const disposeOnce = async () => {
+        if (!disposed) {
+          disposed = true
+          try {
+            await disposableDescendants?.dispose()
+            await disposableSynthetic?.dispose()
+          } catch (error) {
+            warn('openVhd: failed to dispose VHDs', { error })
+          }
+        }
+      }
+      $defer.onFailure(() => disposeOnce())
+
+      const parentVhd = disposableSynthetic.value
+      await parentVhd.readBlockAllocationTable()
+      debug('got vhd synthetic of parents', parentVhd.length)
+
+      if (snapshotCandidate !== undefined) {
+        try {
+          debug('will try to use differential restore', {
+            backupWithSnapshotPath,
+            vhdPath,
+            vdiRef,
+          })
+
+          disposableDescendants = await VhdSynthetic.fromVhdChain(this._adapter._handler, backupWithSnapshotPath, {
+            until: vhdPath,
+          })
+          const descendantsVhd = disposableDescendants.value
+          await descendantsVhd.readBlockAllocationTable()
+          debug('got vhd synthetic of descendants')
+          const negativeVhd = new VhdNegative(parentVhd, descendantsVhd)
+          debug('got vhd negative')
+
+          // update the stream with the negative vhd stream
+          stream = await negativeVhd.stream()
+          vdis[vdiRef].baseVdi = snapshotCandidate
+        } catch (err) {
+          // can be a broken VHD chain, a vhd chain with a key backup, ....
+          // not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
+          warn(`can't use differential restore`, err)
+          disposableDescendants?.dispose()
+        }
+      }
+      // didn't make a negative stream : fallback to classic stream
+      if (stream === undefined) {
+        debug('use legacy restore')
+        stream = await parentVhd.stream()
+      }
+
+      stream.on('end', disposeOnce)
+      stream.on('close', disposeOnce)
+      stream.on('error', disposeOnce)
+      info('everything is ready, will transfer', stream.length)
+      streams[`${vdiRef}.vhd`] = stream
+    }
+    return {
+      streams,
+      vbds,
+      vdis,
+      version: '1.0.0',
+      vifs,
+      vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
+    }
+  }
+
+  async #decorateIncrementalVmMetadata() {
+    const { additionnalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
+    
+    const ignoredVdis = new Set(
+      Object.entries(mapVdisSrs)
+        .filter(([_, srUuid]) => srUuid === null)
+        .map(([vdiUuid]) => vdiUuid)
+    )
+    let backup
+    if (useDifferentialRestore) {
+      backup = await this._reuseNearestSnapshot(ignoredVdis)
+    } else {
+      backup = await this._adapter.readIncrementalVmBackup(this._metadata, ignoredVdis)
+    }
     const xapi = this._xapi
 
     const cache = new Map()
@@ -55,7 +226,7 @@ export class ImportVmBackup {
     const isFull = metadata.mode === 'full'
 
     const sizeContainer = { size: 0 }
-    const { mapVdisSrs, newMacAddresses } = this._importIncrementalVmSettings
+    const { newMacAddresses } = this._importIncrementalVmSettings
     let backup
     if (isFull) {
       backup = await adapter.readFullVmBackup(metadata)
@@ -63,12 +234,7 @@ export class ImportVmBackup {
     } else {
       assert.strictEqual(metadata.mode, 'delta')
 
-      const ignoredVdis = new Set(
-        Object.entries(mapVdisSrs)
-          .filter(([_, srUuid]) => srUuid === null)
-          .map(([vdiUuid]) => vdiUuid)
-      )
-      backup = await this.#decorateIncrementalVmMetadata(await adapter.readIncrementalVmBackup(metadata, ignoredVdis))
+      backup = await this.#decorateIncrementalVmMetadata()
       Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
     }
 
@@ -110,3 +276,5 @@ export class ImportVmBackup {
     )
   }
 }
+
+decorateClass(ImportVmBackup, { _reuseNearestSnapshot: defer })

package/_cleanVm.mjs

@@ -36,34 +36,32 @@ const computeVhdsSize = (handler, vhdPaths) =>
   )
 
 // chain is [ ancestor, child_1, ..., child_n ]
-async function _mergeVhdChain(handler, chain, { logInfo, remove, merge, mergeBlockConcurrency }) {
-  if (merge) {
-    logInfo(`merging VHD chain`, { chain })
-
-    let done, total
-    const handle = setInterval(() => {
-      if (done !== undefined) {
-        logInfo('merge in progress', {
-          done,
-          parent: chain[0],
-          progress: Math.round((100 * done) / total),
-          total,
-        })
-      }
-    }, 10e3)
-    try {
-      return await mergeVhdChain(handler, chain, {
-        logInfo,
-        mergeBlockConcurrency,
-        onProgress({ done: d, total: t }) {
-          done = d
-          total = t
-        },
-        removeUnused: remove,
+async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcurrency }) {
+  logInfo(`merging VHD chain`, { chain })
+
+  let done, total
+  const handle = setInterval(() => {
+    if (done !== undefined) {
+      logInfo('merge in progress', {
+        done,
+        parent: chain[0],
+        progress: Math.round((100 * done) / total),
+        total,
       })
-    } finally {
-      clearInterval(handle)
     }
+  }, 10e3)
+  try {
+    return await mergeVhdChain(handler, chain, {
+      logInfo,
+      mergeBlockConcurrency,
+      onProgress({ done: d, total: t }) {
+        done = d
+        total = t
+      },
+      removeUnused: remove,
+    })
+  } finally {
+    clearInterval(handle)
   }
 }
 
@@ -471,23 +469,20 @@ export async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const merged = await limitedMergeVhdChain(handler, chain, {
+      const { finalVhdSize } = await limitedMergeVhdChain(handler, chain, {
         logInfo,
         logWarn,
         remove,
-        merge,
         mergeBlockConcurrency,
       })
-      if (merged !== undefined) {
-        const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
-        metadataWithMergedVhd[metadataPath] = true
-      }
+      const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
+      metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
     })
   }
 
   await Promise.all([
     ...unusedVhdsDeletion,
-    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
+    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : () => Promise.resolve()),
     asyncMap(unusedXvas, path => {
       logWarn('unused XVA', { path })
       if (remove) {
@@ -509,12 +504,11 @@ export async function cleanVm(
 
   // update size for delta metadata with merged VHD
   // check for the other that the size is the same as the real file size
-
   await asyncMap(jsons, async metadataPath => {
     const metadata = backups.get(metadataPath)
 
     let fileSystemSize
-    const merged = metadataWithMergedVhd[metadataPath] !== undefined
+    const mergedSize = metadataWithMergedVhd[metadataPath]
 
     const { mode, size, vhds, xva } = metadata
 
@@ -524,26 +518,29 @@ export async function cleanVm(
         const linkedXva = resolve('/', vmDir, xva)
         try {
           fileSystemSize = await handler.getSize(linkedXva)
+          if (fileSystemSize !== size && fileSystemSize !== undefined) {
+            logWarn('cleanVm: incorrect backup size in metadata', {
+              path: metadataPath,
+              actual: size ?? 'none',
+              expected: fileSystemSize,
+            })
+          }
         } catch (error) {
           // can fail with encrypted remote
         }
       } else if (mode === 'delta') {
-        const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
-        fileSystemSize = await computeVhdsSize(handler, linkedVhds)
-
-        // the size is not computed in some cases (e.g. VhdDirectory)
-        if (fileSystemSize === undefined) {
-          return
-        }
-
         // don't warn if the size has changed after a merge
-        if (!merged && fileSystemSize !== size) {
-          // FIXME: figure out why it occurs so often and, once fixed, log the real problems with `logWarn`
-          console.warn('cleanVm: incorrect backup size in metadata', {
-            path: metadataPath,
-            actual: size ?? 'none',
-            expected: fileSystemSize,
-          })
+        if (mergedSize === undefined) {
+          const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
+          fileSystemSize = await computeVhdsSize(handler, linkedVhds)
+          // the size is not computed in some cases (e.g. VhdDirectory)
+          if (fileSystemSize !== undefined && fileSystemSize !== size) {
+            logWarn('cleanVm: incorrect backup size in metadata', {
+              path: metadataPath,
+              actual: size ?? 'none',
+              expected: fileSystemSize,
+            })
+          }
         }
       }
     } catch (error) {
@@ -551,9 +548,19 @@ export async function cleanVm(
       return
     }
 
-    // systematically update size after a merge
-    if ((merged || fixMetadata) && size !== fileSystemSize) {
-      metadata.size = fileSystemSize
+    // systematically update size and differentials after a merge
+
+    // @todo : after 2024-04-01 remove the fixmetadata options since the size computation is fixed
+    if (mergedSize || (fixMetadata && fileSystemSize !== size)) {
+      metadata.size = mergedSize ?? fileSystemSize ?? size
+
+      if (mergedSize) {
+        // all disks are now key disk
+        metadata.isVhdDifferencing = {}
+        for (const id of Object.values(metadata.vdis ?? {})) {
+          metadata.isVhdDifferencing[`${id}.vhd`] = false
+        }
+      }
       mustRegenerateCache = true
       try {
         await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })

package/_incrementalVm.mjs

@@ -34,6 +34,7 @@ export async function exportIncrementalVm(
     fullVdisRequired = new Set(),
 
     disableBaseTags = false,
+    nbdConcurrency = 1,
     preferNbd,
   } = {}
 ) {
@@ -82,6 +83,7 @@ export async function exportIncrementalVm(
       baseRef: baseVdi?.$ref,
       cancelToken,
       format: 'vhd',
+      nbdConcurrency,
       preferNbd,
     })
   })
@@ -250,6 +252,10 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     // Import VDI contents.
     cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
       for (let stream of ensureArray(streams[`${id}.vhd`])) {
+        if (stream === null) {
+          // we restore a backup and reuse completly a local snapshot
+          continue
+        }
         if (typeof stream === 'function') {
           stream = await stream()
         }

package/_runners/_vmRunners/IncrementalRemote.mjs

@@ -32,10 +32,10 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
           useChain: false,
         })
 
-        const differentialVhds = {}
+        const isVhdDifferencing = {}
 
         await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
-          differentialVhds[key] = await isVhdDifferencingDisk(stream)
+          isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
         })
 
         incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
@@ -43,7 +43,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
           writer =>
             writer.transfer({
               deltaExport: forkDeltaExport(incrementalExport),
-              differentialVhds,
+              isVhdDifferencing,
               timestamp: metadata.timestamp,
               vm: metadata.vm,
               vmSnapshot: metadata.vmSnapshot,

package/_runners/_vmRunners/IncrementalXapi.mjs

@@ -41,6 +41,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
 
     const deltaExport = await exportIncrementalVm(exportedVm, baseVm, {
       fullVdisRequired,
+      nbdConcurrency: this._settings.nbdConcurrency,
       preferNbd: this._settings.preferNbd,
     })
     // since NBD is network based, if one disk use nbd , all the disk use them
@@ -49,11 +50,11 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       Task.info('Transfer data using NBD')
     }
 
-    const differentialVhds = {}
+    const isVhdDifferencing = {}
     // since isVhdDifferencingDisk is reading and unshifting data in stream
     // it should be done BEFORE any other stream transform
     await asyncEach(Object.entries(deltaExport.streams), async ([key, stream]) => {
-      differentialVhds[key] = await isVhdDifferencingDisk(stream)
+      isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
     })
     const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
 
@@ -68,7 +69,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       writer =>
         writer.transfer({
           deltaExport: forkDeltaExport(deltaExport),
-          differentialVhds,
+          isVhdDifferencing,
           sizeContainers,
           timestamp,
           vm,

package/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -133,7 +133,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     }
   }
 
-  async _transfer($defer, { differentialVhds, timestamp, deltaExport, vm, vmSnapshot }) {
+  async _transfer($defer, { isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot }) {
     const adapter = this._adapter
     const job = this._job
     const scheduleId = this._scheduleId
@@ -161,6 +161,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     )
 
     metadataContent = {
+      isVhdDifferencing,
       jobId,
       mode: job.mode,
       scheduleId,
@@ -180,9 +181,9 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
         async ([id, vdi]) => {
           const path = `${this._vmBackupDir}/${vhds[id]}`
 
-          const isDelta = differentialVhds[`${id}.vhd`]
+          const isDifferencing = isVhdDifferencing[`${id}.vhd`]
           let parentPath
-          if (isDelta) {
+          if (isDifferencing) {
             const vdiDir = dirname(path)
             parentPath = (
               await handler.list(vdiDir, {
@@ -204,16 +205,20 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
             // TODO remove when this has been done before the export
             await checkVhd(handler, parentPath)
           }
-
-          transferSize += await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
+          
+          // don't write it as transferSize += await async function
+          // since i += await asyncFun lead to race condition
+          // as explained : https://eslint.org/docs/latest/rules/require-atomic-updates
+          const transferSizeOneDisk = await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
             writeBlockConcurrency: this._config.writeBlockConcurrency,
           })
+          transferSize += transferSizeOneDisk
 
-          if (isDelta) {
+          if (isDifferencing) {
             await chainVhd(handler, parentPath, handler, path)
           }
 

package/formatVmBackups.mjs

@@ -2,6 +2,8 @@ import mapValues from 'lodash/mapValues.js'
 import { dirname } from 'node:path'
 
 function formatVmBackup(backup) {
+  const { isVhdDifferencing } = backup
+
   return {
     disks:
       backup.vhds === undefined
@@ -25,6 +27,10 @@ function formatVmBackup(backup) {
       name_description: backup.vm.name_description,
       name_label: backup.vm.name_label,
     },
+
+    // isVhdDifferencing is either undefined or an object
+    differencingVhds: isVhdDifferencing && Object.values(isVhdDifferencing).filter(t => t).length,
+    dynamicVhds: isVhdDifferencing && Object.values(isVhdDifferencing).filter(t => !t).length,
   }
 }
 

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.44.1",
+  "version": "0.44.3",
   "engines": {
     "node": ">=14.18"
   },
@@ -25,7 +25,7 @@
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.5",
     "@vates/fuse-vhd": "^2.0.0",
-    "@vates/nbd-client": "^2.0.1",
+    "@vates/nbd-client": "^3.0.0",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/fs": "^4.1.3",
@@ -44,7 +44,7 @@
     "proper-lockfile": "^4.1.2",
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
-    "vhd-lib": "^4.6.1",
+    "vhd-lib": "^4.8.0",
     "xen-api": "^2.0.0",
     "yazl": "^2.5.1"
   },
@@ -56,7 +56,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^4.0.0"
+    "@xen-orchestra/xapi": "^4.1.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {