@xen-orchestra/backups 0.16.1 → 0.18.0

package/RemoteAdapter.js

@@ -3,9 +3,10 @@ const Disposable = require('promise-toolbox/Disposable.js')
 const fromCallback = require('promise-toolbox/fromCallback.js')
 const fromEvent = require('promise-toolbox/fromEvent.js')
 const pDefer = require('promise-toolbox/defer.js')
+const groupBy = require('lodash/groupBy.js')
 const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdSynthetic } = require('vhd-lib')
+const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
@@ -67,10 +68,11 @@ const debounceResourceFactory = factory =>
   }
 
 class RemoteAdapter {
-  constructor(handler, { debounceResource = res => res, dirMode } = {}) {
+  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression = 'brotli' } = {}) {
     this._debounceResource = debounceResource
     this._dirMode = dirMode
     this._handler = handler
+    this._vhdDirectoryCompression = vhdDirectoryCompression
   }
 
   get handler() {
@@ -190,6 +192,22 @@ class RemoteAdapter {
     return files
   }
 
+  // check if we will be allowed to merge a a vhd created in this adapter
+  // with the vhd at path `path`
+  async isMergeableParent(packedParentUid, path) {
+    return await Disposable.use(openVhd(this.handler, path), vhd => {
+      // this baseUuid is not linked with this vhd
+      if (!vhd.footer.uuid.equals(packedParentUid)) {
+        return false
+      }
+
+      const isVhdDirectory = vhd instanceof VhdDirectory
+      return isVhdDirectory
+        ? this.#useVhdDirectory && this.#getCompressionType() === vhd.compressionType
+        : !this.#useVhdDirectory
+    })
+  }
+
   fetchPartitionFiles(diskId, partitionId, paths) {
     const { promise, reject, resolve } = pDefer()
     Disposable.use(
@@ -243,17 +261,34 @@ class RemoteAdapter {
     )
   }
 
-  async deleteVmBackup(filename) {
-    const metadata = JSON.parse(String(await this._handler.readFile(filename)))
-    metadata._filename = filename
+  deleteVmBackup(file) {
+    return this.deleteVmBackups([file])
+  }
+
+  async deleteVmBackups(files) {
+    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
 
-    if (metadata.mode === 'delta') {
-      await this.deleteDeltaVmBackups([metadata])
-    } else if (metadata.mode === 'full') {
-      await this.deleteFullVmBackups([metadata])
-    } else {
-      throw new Error(`no deleter for backup mode ${metadata.mode}`)
+    const unsupportedModes = Object.keys(others)
+    if (unsupportedModes.length !== 0) {
+      throw new Error('no deleter for backup modes: ' + unsupportedModes.join(', '))
     }
+
+    await Promise.all([
+      delta !== undefined && this.deleteDeltaVmBackups(delta),
+      full !== undefined && this.deleteFullVmBackups(full),
+    ])
+  }
+
+  #getCompressionType() {
+    return this._vhdDirectoryCompression
+  }
+
+  #useVhdDirectory() {
+    return this.handler.type === 's3'
+  }
+
+  #useAlias() {
+    return this.#useVhdDirectory()
   }
 
   getDisk = Disposable.factory(this.getDisk)
@@ -312,13 +347,10 @@ class RemoteAdapter {
     return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
   }
 
-  // this function will be the one where we plug the logic of the storage format by fs type/user settings
-
-  // if the file is named .vhd => vhd
-  // if the file is named alias.vhd => alias to a vhd
+  // if we use alias on this remote, we have to name the file alias.vhd
   getVhdFileName(baseName) {
-    if (this._handler.type === 's3') {
-      return `${baseName}.alias.vhd` // we want an alias to a vhddirectory
+    if (this.#useAlias()) {
+      return `${baseName}.alias.vhd`
     }
     return `${baseName}.vhd`
   }
@@ -470,10 +502,11 @@ class RemoteAdapter {
   async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
     const handler = this._handler
 
-    if (path.endsWith('.alias.vhd')) {
+    if (this.#useVhdDirectory()) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
       await createVhdDirectoryFromStream(handler, dataPath, input, {
         concurrency: 16,
+        compression: this.#getCompressionType(),
         async validator() {
           await input.task
           return validator.apply(this, arguments)

package/_VmBackup.js

@@ -36,6 +36,11 @@ const forkDeltaExport = deltaExport =>
 
 exports.VmBackup = class VmBackup {
   constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
+    if (vm.other_config['xo:backup:job'] === job.id) {
+      // otherwise replicated VMs would be matched and replicated again and again
+      throw new Error('cannot backup a VM created by this very job')
+    }
+
     this.config = config
     this.job = job
     this.remoteAdapters = remoteAdapters
@@ -354,6 +359,11 @@ exports.VmBackup = class VmBackup {
       false
     )
 
+    if (presentBaseVdis.size === 0) {
+      debug('no base VM found')
+      return
+    }
+
     const fullVdisRequired = new Set()
     baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
       if (presentBaseVdis.has(baseUuid)) {

package/_backupWorker.js

@@ -70,6 +70,7 @@ class BackupWorker {
       yield new RemoteAdapter(handler, {
         debounceResource: this.debounceResource,
         dirMode: this.#config.dirMode,
+        vhdDirectoryCompression: this.#config.vhdDirectoryCompression,
       })
     } finally {
       await handler.forget()

package/_cleanVm.js

@@ -10,6 +10,24 @@ const { limitConcurrency } = require('limit-concurrency-decorator')
 const { Task } = require('./Task.js')
 const { Disposable } = require('promise-toolbox')
 
+// checking the size of a vhd directory is costly
+// 1 Http Query per 1000 blocks
+// we only check size of all the vhd are VhdFiles
+function shouldComputeVhdsSize(vhds) {
+  return vhds.every(vhd => vhd instanceof VhdFile)
+}
+
+const computeVhdsSize = (handler, vhdPaths) =>
+  Disposable.use(
+    vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
+    async vhds => {
+      if (shouldComputeVhdsSize(vhds)) {
+        const sizes = await asyncMap(vhds, vhd => vhd.getSize())
+        return sum(sizes)
+      }
+    }
+  )
+
 // chain is an array of VHDs from child to parent
 //
 // the whole chain will be merged into parent, parent will be renamed to child
@@ -130,6 +148,7 @@ exports.cleanVm = async function cleanVm(
   const handler = this._handler
 
   const vhds = new Set()
+  const vhdsToJSons = new Set()
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
@@ -202,7 +221,7 @@ exports.cleanVm = async function cleanVm(
     await Promise.all(deletions)
   }
 
-  const jsons = []
+  const jsons = new Set()
   const xvas = new Set()
   const xvaSums = []
   const entries = await handler.list(vmDir, {
@@ -210,7 +229,7 @@ exports.cleanVm = async function cleanVm(
   })
   entries.forEach(path => {
     if (isMetadataFile(path)) {
-      jsons.push(path)
+      jsons.add(path)
     } else if (isXvaFile(path)) {
       xvas.add(path)
     } else if (isXvaSumFile(path)) {
@@ -232,22 +251,25 @@ exports.cleanVm = async function cleanVm(
   // compile the list of unused XVAs and VHDs, and remove backup metadata which
   // reference a missing XVA/VHD
   await asyncMap(jsons, async json => {
-    const metadata = JSON.parse(await handler.readFile(json))
+    let metadata
+    try {
+      metadata = JSON.parse(await handler.readFile(json))
+    } catch (error) {
+      onLog(`failed to read metadata file ${json}`, { error })
+      jsons.delete(json)
+      return
+    }
+
     const { mode } = metadata
-    let size
     if (mode === 'full') {
       const linkedXva = resolve('/', vmDir, metadata.xva)
-
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
-
-        size = await handler.getSize(linkedXva).catch(error => {
-          onLog(`failed to get size of ${json}`, { error })
-        })
       } else {
         onLog(`the XVA linked to the metadata ${json} is missing`)
         if (remove) {
           onLog(`deleting incomplete backup ${json}`)
+          jsons.delete(json)
           await handler.unlink(json)
         }
       }
@@ -256,50 +278,25 @@ exports.cleanVm = async function cleanVm(
         const { vhds } = metadata
         return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
       })()
+
+      const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
+
       // FIXME: find better approach by keeping as much of the backup as
       // possible (existing disks) even if one disk is missing
-      if (linkedVhds.every(_ => vhds.has(_))) {
+      if (missingVhds.length === 0) {
         linkedVhds.forEach(_ => unusedVhds.delete(_))
-
-        // checking the size of a vhd directory is costly
-        // 1 Http Query per 1000 blocks
-        // we only check size of all the vhd are VhdFiles
-
-        const shouldComputeSize = linkedVhds.every(vhd => vhd instanceof VhdFile)
-        if (shouldComputeSize) {
-          try {
-            await Disposable.use(Disposable.all(linkedVhds.map(vhdPath => openVhd(handler, vhdPath))), async vhds => {
-              const sizes = await asyncMap(vhds, vhd => vhd.getSize())
-              size = sum(sizes)
-            })
-          } catch (error) {
-            onLog(`failed to get size of ${json}`, { error })
-          }
-        }
+        linkedVhds.forEach(path => {
+          vhdsToJSons[path] = json
+        })
       } else {
-        onLog(`Some VHDs linked to the metadata ${json} are missing`)
+        onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
         if (remove) {
           onLog(`deleting incomplete backup ${json}`)
+          jsons.delete(json)
           await handler.unlink(json)
         }
       }
     }
-
-    const metadataSize = metadata.size
-    if (size !== undefined && metadataSize !== size) {
-      onLog(`incorrect size in metadata: ${metadataSize ?? 'none'} instead of ${size}`)
-
-      // don't update if the the stored size is greater than found files,
-      // it can indicates a problem
-      if (fixMetadata && (metadataSize === undefined || metadataSize < size)) {
-        try {
-          metadata.size = size
-          await handler.writeFile(json, JSON.stringify(metadata), { flags: 'w' })
-        } catch (error) {
-          onLog(`failed to update size in backup metadata ${json}`, { error })
-        }
-      }
-    }
   })
 
   // TODO: parallelize by vm/job/vdi
@@ -358,9 +355,15 @@ exports.cleanVm = async function cleanVm(
     })
   }
 
-  const doMerge = () => {
-    const promise = asyncMap(toMerge, async chain => limitedMergeVhdChain(chain, { handler, onLog, remove, merge }))
-    return merge ? promise.then(sizes => ({ size: sum(sizes) })) : promise
+  const metadataWithMergedVhd = {}
+  const doMerge = async () => {
+    await asyncMap(toMerge, async chain => {
+      const merged = await limitedMergeVhdChain(chain, { handler, onLog, remove, merge })
+      if (merged !== undefined) {
+        const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
+        metadataWithMergedVhd[metadataPath] = true
+      }
+    })
   }
 
   await Promise.all([
@@ -385,6 +388,47 @@ exports.cleanVm = async function cleanVm(
     }),
   ])
 
+  // update size for delta metadata with merged VHD
+  // check for the other that the size is the same as the real file size
+
+  await asyncMap(jsons, async metadataPath => {
+    const metadata = JSON.parse(await handler.readFile(metadataPath))
+
+    let fileSystemSize
+    const merged = metadataWithMergedVhd[metadataPath] !== undefined
+
+    const { mode, size, vhds, xva } = metadata
+
+    try {
+      if (mode === 'full') {
+        // a full backup : check size
+        const linkedXva = resolve('/', vmDir, xva)
+        fileSystemSize = await handler.getSize(linkedXva)
+      } else if (mode === 'delta') {
+        const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
+        fileSystemSize = await computeVhdsSize(handler, linkedVhds)
+
+        // don't warn if the size has changed after a merge
+        if (!merged && fileSystemSize !== size) {
+          onLog(`incorrect size in metadata: ${size ?? 'none'} instead of ${fileSystemSize}`)
+        }
+      }
+    } catch (error) {
+      onLog(`failed to get size of ${metadataPath}`, { error })
+      return
+    }
+
+    // systematically update size after a merge
+    if ((merged || fixMetadata) && size !== fileSystemSize) {
+      metadata.size = fileSystemSize
+      try {
+        await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
+      } catch (error) {
+        onLog(`failed to update size in backup metadata ${metadataPath} after merge`, { error })
+      }
+    }
+  })
+
   return {
     // boolean whether some VHDs were merged (or should be merged)
     merge: toMerge.length !== 0,

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.16.1",
+  "version": "0.18.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -20,7 +20,7 @@
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.19.1",
+    "@xen-orchestra/fs": "^0.19.3",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
@@ -36,11 +36,11 @@
     "proper-lockfile": "^4.1.2",
     "pump": "^3.0.0",
     "uuid": "^8.3.2",
-    "vhd-lib": "^2.0.2",
+    "vhd-lib": "^2.1.0",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.8.4"
+    "@xen-orchestra/xapi": "^0.8.5"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/writers/DeltaBackupWriter.js

@@ -24,6 +24,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
   async checkBaseVdis(baseUuidToSrcVdi) {
     const { handler } = this._adapter
     const backup = this._backup
+    const adapter = this._adapter
 
     const backupDir = getVmBackupDir(backup.vm.uuid)
     const vdisDir = `${backupDir}/vdis/${backup.job.id}`
@@ -35,13 +36,11 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
           prependDir: true,
         })
+        const packedBaseUuid = packUuid(baseUuid)
         await asyncMap(vhds, async path => {
           try {
             await checkVhdChain(handler, path)
-            await Disposable.use(
-              openVhd(handler, path),
-              vhd => (found = found || vhd.footer.uuid.equals(packUuid(baseUuid)))
-            )
+            found = found || (await adapter.isMergeableParent(packedBaseUuid, path))
           } catch (error) {
             warn('checkBaseVdis', { error })
             await ignoreErrors.call(VhdAbstract.unlink(handler, path))

package/writers/_MixinBackupWriter.js

@@ -21,10 +21,18 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
       this.#vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
     }
 
-    _cleanVm(options) {
-      return this._adapter
-        .cleanVm(this.#vmBackupDir, { ...options, fixMetadata: true, onLog: warn, lock: false })
-        .catch(warn)
+    async _cleanVm(options) {
+      try {
+        return await this._adapter.cleanVm(this.#vmBackupDir, {
+          ...options,
+          fixMetadata: true,
+          onLog: warn,
+          lock: false,
+        })
+      } catch (error) {
+        warn(error)
+        return {}
+      }
     }
 
     async beforeBackup() {
@@ -43,7 +51,13 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
       // merge worker only compatible with local remotes
       const { handler } = this._adapter
       if (merge && !disableMergeWorker && typeof handler._getRealPath === 'function') {
-        await handler.outputFile(join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())), this._backup.vm.uuid)
+        const taskFile =
+          join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())) +
+          '-' +
+          // add a random suffix to avoid collision in case multiple tasks are created at the same second
+          Math.random().toString(36).slice(2)
+
+        await handler.outputFile(taskFile, this._backup.vm.uuid)
         const remotePath = handler._getRealPath()
         await MergeWorker.run(remotePath)
       }