@xen-orchestra/backups 0.15.1 → 0.17.0

package/RemoteAdapter.js

@@ -3,19 +3,20 @@ const Disposable = require('promise-toolbox/Disposable.js')
 const fromCallback = require('promise-toolbox/fromCallback.js')
 const fromEvent = require('promise-toolbox/fromEvent.js')
 const pDefer = require('promise-toolbox/defer.js')
-const pump = require('pump')
-const { basename, dirname, join, normalize, resolve } = require('path')
+const groupBy = require('lodash/groupBy.js')
+const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createSyntheticStream, mergeVhd, VhdFile } = require('vhd-lib')
+const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
+const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
 const { getTmpDir } = require('./_getTmpDir.js')
-const { isMetadataFile, isVhdFile } = require('./_backupType.js')
+const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
@@ -77,48 +78,6 @@ class RemoteAdapter {
     return this._handler
   }
 
-  async _deleteVhd(path) {
-    const handler = this._handler
-    const vhds = await asyncMapSettled(
-      await handler.list(dirname(path), {
-        filter: isVhdFile,
-        prependDir: true,
-      }),
-      async path => {
-        try {
-          const vhd = new VhdFile(handler, path)
-          await vhd.readHeaderAndFooter()
-          return {
-            footer: vhd.footer,
-            header: vhd.header,
-            path,
-          }
-        } catch (error) {
-          // Do not fail on corrupted VHDs (usually uncleaned temporary files),
-          // they are probably inconsequent to the backup process and should not
-          // fail it.
-          warn(`BackupNg#_deleteVhd ${path}`, { error })
-        }
-      }
-    )
-    const base = basename(path)
-    const child = vhds.find(_ => _ !== undefined && _.header.parentUnicodeName === base)
-    if (child === undefined) {
-      await handler.unlink(path)
-      return 0
-    }
-
-    try {
-      const childPath = child.path
-      const mergedDataSize = await mergeVhd(handler, path, handler, childPath)
-      await handler.rename(path, childPath)
-      return mergedDataSize
-    } catch (error) {
-      handler.unlink(path).catch(warn)
-      throw error
-    }
-  }
-
   async _findPartition(devicePath, partitionId) {
     const partitions = await listPartitions(devicePath)
     const partition = partitions.find(_ => _.id === partitionId)
@@ -255,7 +214,7 @@ class RemoteAdapter {
     const handler = this._handler
 
     // unused VHDs will be detected by `cleanVm`
-    await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
+    await asyncMapSettled(backups, ({ _filename }) => VhdAbstract.unlink(handler, _filename))
   }
 
   async deleteMetadataBackup(backupId) {
@@ -285,17 +244,22 @@ class RemoteAdapter {
     )
   }
 
-  async deleteVmBackup(filename) {
-    const metadata = JSON.parse(String(await this._handler.readFile(filename)))
-    metadata._filename = filename
+  deleteVmBackup(file) {
+    return this.deleteVmBackups([file])
+  }
 
-    if (metadata.mode === 'delta') {
-      await this.deleteDeltaVmBackups([metadata])
-    } else if (metadata.mode === 'full') {
-      await this.deleteFullVmBackups([metadata])
-    } else {
-      throw new Error(`no deleter for backup mode ${metadata.mode}`)
+  async deleteVmBackups(files) {
+    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
+
+    const unsupportedModes = Object.keys(others)
+    if (unsupportedModes.length !== 0) {
+      throw new Error('no deleter for backup modes: ' + unsupportedModes.join(', '))
     }
+
+    await Promise.all([
+      delta !== undefined && this.deleteDeltaVmBackups(delta),
+      full !== undefined && this.deleteFullVmBackups(full),
+    ])
   }
 
   getDisk = Disposable.factory(this.getDisk)
@@ -354,6 +318,17 @@ class RemoteAdapter {
     return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
   }
 
+  // this function will be the one where we plug the logic of the storage format by fs type/user settings
+
+  // if the file is named .vhd => vhd
+  // if the file is named alias.vhd => alias to a vhd
+  getVhdFileName(baseName) {
+    if (this._handler.type === 's3') {
+      return `${baseName}.alias.vhd` // we want an alias to a vhddirectory
+    }
+    return `${baseName}.vhd`
+  }
+
   async listAllVmBackups() {
     const handler = this._handler
 
@@ -498,6 +473,24 @@ class RemoteAdapter {
     return backups.sort(compareTimestamp)
   }
 
+  async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
+    const handler = this._handler
+
+    if (path.endsWith('.alias.vhd')) {
+      const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
+      await createVhdDirectoryFromStream(handler, dataPath, input, {
+        concurrency: 16,
+        async validator() {
+          await input.task
+          return validator.apply(this, arguments)
+        },
+      })
+      await VhdAbstract.createAlias(handler, path, dataPath)
+    } else {
+      await this.outputStream(path, input, { checksum, validator })
+    }
+  }
+
   async outputStream(path, input, { checksum = true, validator = noop } = {}) {
     await this._handler.outputStream(path, input, {
       checksum,
@@ -509,6 +502,52 @@ class RemoteAdapter {
     })
   }
 
+  async _createSyntheticStream(handler, paths) {
+    let disposableVhds = []
+
+    // if it's a path : open all hierarchy of parent
+    if (typeof paths === 'string') {
+      let vhd,
+        vhdPath = paths
+      do {
+        const disposable = await openVhd(handler, vhdPath)
+        vhd = disposable.value
+        disposableVhds.push(disposable)
+        vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
+      } while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
+    } else {
+      // only open the list of path given
+      disposableVhds = paths.map(path => openVhd(handler, path))
+    }
+
+    // I don't want the vhds to be disposed on return
+    // but only when the stream is done ( or failed )
+    const disposables = await Disposable.all(disposableVhds)
+    const vhds = disposables.value
+
+    let disposed = false
+    const disposeOnce = async () => {
+      if (!disposed) {
+        disposed = true
+
+        try {
+          await disposables.dispose()
+        } catch (error) {
+          warn('_createSyntheticStream: failed to dispose VHDs', { error })
+        }
+      }
+    }
+
+    const synthetic = new VhdSynthetic(vhds)
+    await synthetic.readHeaderAndFooter()
+    await synthetic.readBlockAllocationTable()
+    const stream = await synthetic.stream()
+    stream.on('end', disposeOnce)
+    stream.on('close', disposeOnce)
+    stream.on('error', disposeOnce)
+    return stream
+  }
+
   async readDeltaVmBackup(metadata) {
     const handler = this._handler
     const { vbds, vdis, vhds, vifs, vm } = metadata
@@ -516,7 +555,7 @@ class RemoteAdapter {
 
     const streams = {}
     await asyncMapSettled(Object.keys(vdis), async id => {
-      streams[`${id}.vhd`] = await createSyntheticStream(handler, join(dir, vhds[id]))
+      streams[`${id}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[id]))
     })
 
     return {

package/_VmBackup.js

@@ -36,6 +36,11 @@ const forkDeltaExport = deltaExport =>
 
 exports.VmBackup = class VmBackup {
   constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
+    if (vm.other_config['xo:backup:job'] === job.id) {
+      // otherwise replicated VMs would be matched and replicated again and again
+      throw new Error('cannot backup a VM created by this very job')
+    }
+
     this.config = config
     this.job = job
     this.remoteAdapters = remoteAdapters
@@ -333,13 +338,16 @@ exports.VmBackup = class VmBackup {
 
     const baseUuidToSrcVdi = new Map()
     await asyncMap(await baseVm.$getDisks(), async baseRef => {
-      const snapshotOf = await xapi.getField('VDI', baseRef, 'snapshot_of')
+      const [baseUuid, snapshotOf] = await Promise.all([
+        xapi.getField('VDI', baseRef, 'uuid'),
+        xapi.getField('VDI', baseRef, 'snapshot_of'),
+      ])
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
-        baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
+        baseUuidToSrcVdi.set(baseUuid, srcVdi)
       } else {
-        debug('no base VDI found', {
-          vdi: srcVdi.uuid,
+        debug('ignore snapshot VDI because no longer present on VM', {
+          vdi: baseUuid,
         })
       }
     })
@@ -351,6 +359,11 @@ exports.VmBackup = class VmBackup {
       false
     )
 
+    if (presentBaseVdis.size === 0) {
+      debug('no base VM found')
+      return
+    }
+
     const fullVdisRequired = new Set()
     baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
       if (presentBaseVdis.has(baseUuid)) {

package/_cleanVm.js

@@ -1,13 +1,14 @@
 const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { VhdFile, mergeVhd } = require('vhd-lib')
+const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
 const { dirname, resolve } = require('path')
-const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
+const { DISK_TYPES } = Constants
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
 const { Task } = require('./Task.js')
+const { Disposable } = require('promise-toolbox')
 
 // chain is an array of VHDs from child to parent
 //
@@ -65,12 +66,12 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
     clearInterval(handle)
 
     await Promise.all([
-      handler.rename(parent, child),
+      VhdAbstract.rename(handler, parent, child),
       asyncMap(children.slice(0, -1), child => {
         onLog(`the VHD ${child} is unused`)
         if (remove) {
           onLog(`deleting unused VHD ${child}`)
-          return handler.unlink(child)
+          return VhdAbstract.unlink(handler, child)
         }
       }),
     ])
@@ -137,53 +138,55 @@ exports.cleanVm = async function cleanVm(
   // remove broken VHDs
   await asyncMap(vhdsList.vhds, async path => {
     try {
-      const vhd = new VhdFile(handler, path)
-      await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
-      vhds.add(path)
-      if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
-        const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
-        vhdParents[path] = parent
-        if (parent in vhdChildren) {
-          const error = new Error('this script does not support multiple VHD children')
-          error.parent = parent
-          error.child1 = vhdChildren[parent]
-          error.child2 = path
-          throw error // should we throw?
+      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !vhdsList.interruptedVhds.has(path) }), vhd => {
+        vhds.add(path)
+        if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
+          const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
+          vhdParents[path] = parent
+          if (parent in vhdChildren) {
+            const error = new Error('this script does not support multiple VHD children')
+            error.parent = parent
+            error.child1 = vhdChildren[parent]
+            error.child2 = path
+            throw error // should we throw?
+          }
+          vhdChildren[parent] = path
         }
-        vhdChildren[parent] = path
-      }
+      })
     } catch (error) {
       onLog(`error while checking the VHD with path ${path}`, { error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
         onLog(`deleting broken ${path}`)
-        await handler.unlink(path)
+        return VhdAbstract.unlink(handler, path)
       }
     }
   })
 
+  // @todo : add check for data folder of alias not referenced in a valid alias
+
   // remove VHDs with missing ancestors
   {
     const deletions = []
 
     // return true if the VHD has been deleted or is missing
-    const deleteIfOrphan = vhd => {
-      const parent = vhdParents[vhd]
+    const deleteIfOrphan = vhdPath => {
+      const parent = vhdParents[vhdPath]
       if (parent === undefined) {
         return
       }
 
       // no longer needs to be checked
-      delete vhdParents[vhd]
+      delete vhdParents[vhdPath]
 
       deleteIfOrphan(parent)
 
       if (!vhds.has(parent)) {
-        vhds.delete(vhd)
+        vhds.delete(vhdPath)
 
-        onLog(`the parent ${parent} of the VHD ${vhd} is missing`)
+        onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
         if (remove) {
-          onLog(`deleting orphan VHD ${vhd}`)
-          deletions.push(handler.unlink(vhd))
+          onLog(`deleting orphan VHD ${vhdPath}`)
+          deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
     }
@@ -254,16 +257,30 @@ exports.cleanVm = async function cleanVm(
         return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
       })()
 
+      const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
+
       // FIXME: find better approach by keeping as much of the backup as
       // possible (existing disks) even if one disk is missing
-      if (linkedVhds.every(_ => vhds.has(_))) {
+      if (missingVhds.length === 0) {
         linkedVhds.forEach(_ => unusedVhds.delete(_))
 
-        size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
-          onLog(`failed to get size of ${json}`, { error })
-        })
+        // checking the size of a vhd directory is costly
+        // 1 Http Query per 1000 blocks
+        // we only check size of all the vhd are VhdFiles
+
+        const shouldComputeSize = linkedVhds.every(vhd => vhd instanceof VhdFile)
+        if (shouldComputeSize) {
+          try {
+            await Disposable.use(Disposable.all(linkedVhds.map(vhdPath => openVhd(handler, vhdPath))), async vhds => {
+              const sizes = await asyncMap(vhds, vhd => vhd.getSize())
+              size = sum(sizes)
+            })
+          } catch (error) {
+            onLog(`failed to get size of ${json}`, { error })
+          }
+        }
       } else {
-        onLog(`Some VHDs linked to the metadata ${json} are missing`)
+        onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
         if (remove) {
           onLog(`deleting incomplete backup ${json}`)
           await handler.unlink(json)
@@ -324,7 +341,7 @@ exports.cleanVm = async function cleanVm(
       onLog(`the VHD ${vhd} is unused`)
       if (remove) {
         onLog(`deleting unused VHD ${vhd}`)
-        unusedVhdsDeletion.push(handler.unlink(vhd))
+        unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
 

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.15.1",
+  "version": "0.17.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -16,14 +16,14 @@
     "postversion": "npm publish --access public"
   },
   "dependencies": {
-    "@vates/compose": "^2.0.0",
+    "@vates/compose": "^2.1.0",
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/fs": "^0.19.2",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
-    "compare-versions": "^3.6.0",
+    "compare-versions": "^4.0.1",
     "d3-time-format": "^3.0.0",
     "end-of-stream": "^1.4.4",
     "fs-extra": "^10.0.0",
@@ -35,11 +35,12 @@
     "promise-toolbox": "^0.20.0",
     "proper-lockfile": "^4.1.2",
     "pump": "^3.0.0",
-    "vhd-lib": "^1.3.0",
+    "uuid": "^8.3.2",
+    "vhd-lib": "^2.0.4",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.8.0"
+    "@xen-orchestra/xapi": "^0.8.4"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/writers/DeltaBackupWriter.js

@@ -3,7 +3,7 @@ const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { chainVhd, checkVhdChain, VhdFile } = require('vhd-lib')
+const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
 const { dirname } = require('path')
 
@@ -16,6 +16,7 @@ const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
 const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
+const { Disposable } = require('promise-toolbox')
 
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
@@ -37,13 +38,13 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
         await asyncMap(vhds, async path => {
           try {
             await checkVhdChain(handler, path)
-
-            const vhd = new VhdFile(handler, path)
-            await vhd.readHeaderAndFooter()
-            found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
+            await Disposable.use(
+              openVhd(handler, path),
+              vhd => (found = found || vhd.footer.uuid.equals(packUuid(baseUuid)))
+            )
           } catch (error) {
             warn('checkBaseVdis', { error })
-            await ignoreErrors.call(handler.unlink(path))
+            await ignoreErrors.call(VhdAbstract.unlink(handler, path))
           }
         })
       } catch (error) {
@@ -144,7 +145,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
               // don't do delta for it
               vdi.uuid
             : vdi.$snapshot_of$uuid
-        }/${basename}.vhd`
+        }/${adapter.getVhdFileName(basename)}`
     )
 
     const metadataFilename = `${backupDir}/${basename}.json`
@@ -188,7 +189,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
             await checkVhd(handler, parentPath)
           }
 
-          await adapter.outputStream(path, deltaExport.streams[`${id}.vhd`], {
+          await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
@@ -200,11 +201,11 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           }
 
           // set the correct UUID in the VHD
-          const vhd = new VhdFile(handler, path)
-          await vhd.readHeaderAndFooter()
-          vhd.footer.uuid = packUuid(vdi.uuid)
-          await vhd.readBlockAllocationTable() // required by writeFooter()
-          await vhd.writeFooter()
+          await Disposable.use(openVhd(handler, path), async vhd => {
+            vhd.footer.uuid = packUuid(vdi.uuid)
+            await vhd.readBlockAllocationTable() // required by writeFooter()
+            await vhd.writeFooter()
+          })
         })
       )
       return {

package/writers/_MixinBackupWriter.js

@@ -21,10 +21,18 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
       this.#vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
     }
 
-    _cleanVm(options) {
-      return this._adapter
-        .cleanVm(this.#vmBackupDir, { ...options, fixMetadata: true, onLog: warn, lock: false })
-        .catch(warn)
+    async _cleanVm(options) {
+      try {
+        return await this._adapter.cleanVm(this.#vmBackupDir, {
+          ...options,
+          fixMetadata: true,
+          onLog: warn,
+          lock: false,
+        })
+      } catch (error) {
+        warn(error)
+        return {}
+      }
     }
 
     async beforeBackup() {
@@ -43,7 +51,13 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
       // merge worker only compatible with local remotes
       const { handler } = this._adapter
       if (merge && !disableMergeWorker && typeof handler._getRealPath === 'function') {
-        await handler.outputFile(join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())), this._backup.vm.uuid)
+        const taskFile =
+          join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())) +
+          '-' +
+          // add a random suffix to avoid collision in case multiple tasks are created at the same second
+          Math.random().toString(36).slice(2)
+
+        await handler.outputFile(taskFile, this._backup.vm.uuid)
         const remotePath = handler._getRealPath()
         await MergeWorker.run(remotePath)
       }

package/writers/_checkVhd.js

@@ -1,5 +1,6 @@
-const Vhd = require('vhd-lib').VhdFile
+const openVhd = require('vhd-lib').openVhd
+const Disposable = require('promise-toolbox/Disposable')
 
 exports.checkVhd = async function checkVhd(handler, path) {
-  await new Vhd(handler, path).readHeaderAndFooter()
+  await Disposable.use(openVhd(handler, path), () => {})
 }