@xen-orchestra/backups 0.14.0 → 0.16.1

package/RemoteAdapter.js

@@ -3,19 +3,19 @@ const Disposable = require('promise-toolbox/Disposable.js')
 const fromCallback = require('promise-toolbox/fromCallback.js')
 const fromEvent = require('promise-toolbox/fromEvent.js')
 const pDefer = require('promise-toolbox/defer.js')
-const pump = require('pump')
-const { basename, dirname, join, normalize, resolve } = require('path')
+const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createSyntheticStream, mergeVhd, default: Vhd } = require('vhd-lib')
+const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
+const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
 const { getTmpDir } = require('./_getTmpDir.js')
-const { isMetadataFile, isVhdFile } = require('./_backupType.js')
+const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
@@ -77,48 +77,6 @@ class RemoteAdapter {
     return this._handler
   }
 
-  async _deleteVhd(path) {
-    const handler = this._handler
-    const vhds = await asyncMapSettled(
-      await handler.list(dirname(path), {
-        filter: isVhdFile,
-        prependDir: true,
-      }),
-      async path => {
-        try {
-          const vhd = new Vhd(handler, path)
-          await vhd.readHeaderAndFooter()
-          return {
-            footer: vhd.footer,
-            header: vhd.header,
-            path,
-          }
-        } catch (error) {
-          // Do not fail on corrupted VHDs (usually uncleaned temporary files),
-          // they are probably inconsequent to the backup process and should not
-          // fail it.
-          warn(`BackupNg#_deleteVhd ${path}`, { error })
-        }
-      }
-    )
-    const base = basename(path)
-    const child = vhds.find(_ => _ !== undefined && _.header.parentUnicodeName === base)
-    if (child === undefined) {
-      await handler.unlink(path)
-      return 0
-    }
-
-    try {
-      const childPath = child.path
-      const mergedDataSize = await mergeVhd(handler, path, handler, childPath)
-      await handler.rename(path, childPath)
-      return mergedDataSize
-    } catch (error) {
-      handler.unlink(path).catch(warn)
-      throw error
-    }
-  }
-
   async _findPartition(devicePath, partitionId) {
     const partitions = await listPartitions(devicePath)
     const partition = partitions.find(_ => _.id === partitionId)
@@ -255,7 +213,7 @@ class RemoteAdapter {
     const handler = this._handler
 
     // unused VHDs will be detected by `cleanVm`
-    await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
+    await asyncMapSettled(backups, ({ _filename }) => VhdAbstract.unlink(handler, _filename))
   }
 
   async deleteMetadataBackup(backupId) {
@@ -354,6 +312,17 @@ class RemoteAdapter {
     return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
   }
 
+  // this function will be the one where we plug the logic of the storage format by fs type/user settings
+
+  // if the file is named .vhd => vhd
+  // if the file is named alias.vhd => alias to a vhd
+  getVhdFileName(baseName) {
+    if (this._handler.type === 's3') {
+      return `${baseName}.alias.vhd` // we want an alias to a vhddirectory
+    }
+    return `${baseName}.vhd`
+  }
+
   async listAllVmBackups() {
     const handler = this._handler
 
@@ -498,6 +467,24 @@ class RemoteAdapter {
     return backups.sort(compareTimestamp)
   }
 
+  async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
+    const handler = this._handler
+
+    if (path.endsWith('.alias.vhd')) {
+      const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
+      await createVhdDirectoryFromStream(handler, dataPath, input, {
+        concurrency: 16,
+        async validator() {
+          await input.task
+          return validator.apply(this, arguments)
+        },
+      })
+      await VhdAbstract.createAlias(handler, path, dataPath)
+    } else {
+      await this.outputStream(path, input, { checksum, validator })
+    }
+  }
+
   async outputStream(path, input, { checksum = true, validator = noop } = {}) {
     await this._handler.outputStream(path, input, {
       checksum,
@@ -509,6 +496,52 @@ class RemoteAdapter {
     })
   }
 
+  async _createSyntheticStream(handler, paths) {
+    let disposableVhds = []
+
+    // if it's a path : open all hierarchy of parent
+    if (typeof paths === 'string') {
+      let vhd,
+        vhdPath = paths
+      do {
+        const disposable = await openVhd(handler, vhdPath)
+        vhd = disposable.value
+        disposableVhds.push(disposable)
+        vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
+      } while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
+    } else {
+      // only open the list of path given
+      disposableVhds = paths.map(path => openVhd(handler, path))
+    }
+
+    // I don't want the vhds to be disposed on return
+    // but only when the stream is done ( or failed )
+    const disposables = await Disposable.all(disposableVhds)
+    const vhds = disposables.value
+
+    let disposed = false
+    const disposeOnce = async () => {
+      if (!disposed) {
+        disposed = true
+
+        try {
+          await disposables.dispose()
+        } catch (error) {
+          warn('_createSyntheticStream: failed to dispose VHDs', { error })
+        }
+      }
+    }
+
+    const synthetic = new VhdSynthetic(vhds)
+    await synthetic.readHeaderAndFooter()
+    await synthetic.readBlockAllocationTable()
+    const stream = await synthetic.stream()
+    stream.on('end', disposeOnce)
+    stream.on('close', disposeOnce)
+    stream.on('error', disposeOnce)
+    return stream
+  }
+
   async readDeltaVmBackup(metadata) {
     const handler = this._handler
     const { vbds, vdis, vhds, vifs, vm } = metadata
@@ -516,7 +549,7 @@ class RemoteAdapter {
 
     const streams = {}
     await asyncMapSettled(Object.keys(vdis), async id => {
-      streams[`${id}.vhd`] = await createSyntheticStream(handler, join(dir, vhds[id]))
+      streams[`${id}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[id]))
     })
 
     return {

package/_VmBackup.js

@@ -1,9 +1,10 @@
 const assert = require('assert')
 const findLast = require('lodash/findLast.js')
+const groupBy = require('lodash/groupBy.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const keyBy = require('lodash/keyBy.js')
 const mapValues = require('lodash/mapValues.js')
-const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
 const { defer } = require('golike-defer')
 const { formatDateTime } = require('@xen-orchestra/xapi')
@@ -284,17 +285,28 @@ exports.VmBackup = class VmBackup {
   }
 
   async _removeUnusedSnapshots() {
-    // TODO: handle all schedules (no longer existing schedules default to 0 retention)
-
-    const { scheduleId } = this
-    const scheduleSnapshots = this._jobSnapshots.filter(_ => _.other_config['xo:backup:schedule'] === scheduleId)
-
+    const jobSettings = this.job.settings
     const baseVmRef = this._baseVm?.$ref
+    const { config } = this
+    const baseSettings = {
+      ...config.defaultSettings,
+      ...config.metadata.defaultSettings,
+      ...jobSettings[''],
+    }
+
+    const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
     const xapi = this._xapi
-    await asyncMap(getOldEntries(this._settings.snapshotRetention, scheduleSnapshots), ({ $ref }) => {
-      if ($ref !== baseVmRef) {
-        return xapi.VM_destroy($ref)
+    await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
+      const settings = {
+        ...baseSettings,
+        ...jobSettings[scheduleId],
+        ...jobSettings[this.vm.uuid],
       }
+      return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
+        if ($ref !== baseVmRef) {
+          return xapi.VM_destroy($ref)
+        }
+      })
     })
   }
 
@@ -321,13 +333,16 @@ exports.VmBackup = class VmBackup {
 
     const baseUuidToSrcVdi = new Map()
     await asyncMap(await baseVm.$getDisks(), async baseRef => {
-      const snapshotOf = await xapi.getField('VDI', baseRef, 'snapshot_of')
+      const [baseUuid, snapshotOf] = await Promise.all([
+        xapi.getField('VDI', baseRef, 'uuid'),
+        xapi.getField('VDI', baseRef, 'snapshot_of'),
+      ])
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
-        baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
+        baseUuidToSrcVdi.set(baseUuid, srcVdi)
       } else {
-        debug('no base VDI found', {
-          vdi: srcVdi.uuid,
+        debug('ignore snapshot VDI because no longer present on VM', {
+          vdi: baseUuid,
         })
       }
     })

package/_cleanVm.js

@@ -1,13 +1,14 @@
 const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { default: Vhd, mergeVhd } = require('vhd-lib')
+const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
 const { dirname, resolve } = require('path')
-const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
+const { DISK_TYPES } = Constants
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
 const { Task } = require('./Task.js')
+const { Disposable } = require('promise-toolbox')
 
 // chain is an array of VHDs from child to parent
 //
@@ -65,12 +66,12 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
     clearInterval(handle)
 
     await Promise.all([
-      handler.rename(parent, child),
+      VhdAbstract.rename(handler, parent, child),
       asyncMap(children.slice(0, -1), child => {
         onLog(`the VHD ${child} is unused`)
         if (remove) {
           onLog(`deleting unused VHD ${child}`)
-          return handler.unlink(child)
+          return VhdAbstract.unlink(handler, child)
         }
       }),
     ])
@@ -124,6 +125,8 @@ exports.cleanVm = async function cleanVm(
   vmDir,
   { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, onLog = noop }
 ) {
+  const limitedMergeVhdChain = mergeLimiter(mergeVhdChain)
+
   const handler = this._handler
 
   const vhds = new Set()
@@ -135,53 +138,55 @@ exports.cleanVm = async function cleanVm(
   // remove broken VHDs
   await asyncMap(vhdsList.vhds, async path => {
     try {
-      const vhd = new Vhd(handler, path)
-      await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
-      vhds.add(path)
-      if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
-        const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
-        vhdParents[path] = parent
-        if (parent in vhdChildren) {
-          const error = new Error('this script does not support multiple VHD children')
-          error.parent = parent
-          error.child1 = vhdChildren[parent]
-          error.child2 = path
-          throw error // should we throw?
+      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !vhdsList.interruptedVhds.has(path) }), vhd => {
+        vhds.add(path)
+        if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
+          const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
+          vhdParents[path] = parent
+          if (parent in vhdChildren) {
+            const error = new Error('this script does not support multiple VHD children')
+            error.parent = parent
+            error.child1 = vhdChildren[parent]
+            error.child2 = path
+            throw error // should we throw?
+          }
+          vhdChildren[parent] = path
         }
-        vhdChildren[parent] = path
-      }
+      })
     } catch (error) {
       onLog(`error while checking the VHD with path ${path}`, { error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
         onLog(`deleting broken ${path}`)
-        await handler.unlink(path)
+        return VhdAbstract.unlink(handler, path)
       }
     }
   })
 
+  // @todo : add check for data folder of alias not referenced in a valid alias
+
   // remove VHDs with missing ancestors
   {
     const deletions = []
 
     // return true if the VHD has been deleted or is missing
-    const deleteIfOrphan = vhd => {
-      const parent = vhdParents[vhd]
+    const deleteIfOrphan = vhdPath => {
+      const parent = vhdParents[vhdPath]
       if (parent === undefined) {
         return
       }
 
       // no longer needs to be checked
-      delete vhdParents[vhd]
+      delete vhdParents[vhdPath]
 
       deleteIfOrphan(parent)
 
       if (!vhds.has(parent)) {
-        vhds.delete(vhd)
+        vhds.delete(vhdPath)
 
-        onLog(`the parent ${parent} of the VHD ${vhd} is missing`)
+        onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
         if (remove) {
-          onLog(`deleting orphan VHD ${vhd}`)
-          deletions.push(handler.unlink(vhd))
+          onLog(`deleting orphan VHD ${vhdPath}`)
+          deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
     }
@@ -251,15 +256,26 @@ exports.cleanVm = async function cleanVm(
         const { vhds } = metadata
         return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
       })()
-
       // FIXME: find better approach by keeping as much of the backup as
       // possible (existing disks) even if one disk is missing
       if (linkedVhds.every(_ => vhds.has(_))) {
         linkedVhds.forEach(_ => unusedVhds.delete(_))
 
-        size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
-          onLog(`failed to get size of ${json}`, { error })
-        })
+        // checking the size of a vhd directory is costly
+        // 1 Http Query per 1000 blocks
+        // we only check size of all the vhd are VhdFiles
+
+        const shouldComputeSize = linkedVhds.every(vhd => vhd instanceof VhdFile)
+        if (shouldComputeSize) {
+          try {
+            await Disposable.use(Disposable.all(linkedVhds.map(vhdPath => openVhd(handler, vhdPath))), async vhds => {
+              const sizes = await asyncMap(vhds, vhd => vhd.getSize())
+              size = sum(sizes)
+            })
+          } catch (error) {
+            onLog(`failed to get size of ${json}`, { error })
+          }
+        }
       } else {
         onLog(`Some VHDs linked to the metadata ${json} are missing`)
         if (remove) {
@@ -322,7 +338,7 @@ exports.cleanVm = async function cleanVm(
       onLog(`the VHD ${vhd} is unused`)
       if (remove) {
         onLog(`deleting unused VHD ${vhd}`)
-        unusedVhdsDeletion.push(handler.unlink(vhd))
+        unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
 
@@ -343,9 +359,7 @@ exports.cleanVm = async function cleanVm(
   }
 
   const doMerge = () => {
-    const promise = asyncMap(toMerge, async chain => {
-      mergeVhdChain(chain, { handler, onLog, remove, merge })
-    })
+    const promise = asyncMap(toMerge, async chain => limitedMergeVhdChain(chain, { handler, onLog, remove, merge }))
     return merge ? promise.then(sizes => ({ size: sum(sizes) })) : promise
   }
 

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.14.0",
+  "version": "0.16.1",
   "engines": {
     "node": ">=14.6"
   },
@@ -16,14 +16,14 @@
     "postversion": "npm publish --access public"
   },
   "dependencies": {
-    "@vates/compose": "^2.0.0",
+    "@vates/compose": "^2.1.0",
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/fs": "^0.19.1",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
-    "compare-versions": "^3.6.0",
+    "compare-versions": "^4.0.1",
     "d3-time-format": "^3.0.0",
     "end-of-stream": "^1.4.4",
     "fs-extra": "^10.0.0",
@@ -35,11 +35,12 @@
     "promise-toolbox": "^0.20.0",
     "proper-lockfile": "^4.1.2",
     "pump": "^3.0.0",
-    "vhd-lib": "^1.2.0",
+    "uuid": "^8.3.2",
+    "vhd-lib": "^2.0.2",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.7.0"
+    "@xen-orchestra/xapi": "^0.8.4"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/writers/DeltaBackupWriter.js

@@ -3,7 +3,7 @@ const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { chainVhd, checkVhdChain, default: Vhd } = require('vhd-lib')
+const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
 const { dirname } = require('path')
 
@@ -16,6 +16,7 @@ const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
 const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
+const { Disposable } = require('promise-toolbox')
 
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
@@ -37,13 +38,13 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
         await asyncMap(vhds, async path => {
           try {
             await checkVhdChain(handler, path)
-
-            const vhd = new Vhd(handler, path)
-            await vhd.readHeaderAndFooter()
-            found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
+            await Disposable.use(
+              openVhd(handler, path),
+              vhd => (found = found || vhd.footer.uuid.equals(packUuid(baseUuid)))
+            )
           } catch (error) {
             warn('checkBaseVdis', { error })
-            await ignoreErrors.call(handler.unlink(path))
+            await ignoreErrors.call(VhdAbstract.unlink(handler, path))
           }
         })
       } catch (error) {
@@ -144,7 +145,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
               // don't do delta for it
               vdi.uuid
             : vdi.$snapshot_of$uuid
-        }/${basename}.vhd`
+        }/${adapter.getVhdFileName(basename)}`
     )
 
     const metadataFilename = `${backupDir}/${basename}.json`
@@ -188,7 +189,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
             await checkVhd(handler, parentPath)
           }
 
-          await adapter.outputStream(path, deltaExport.streams[`${id}.vhd`], {
+          await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
@@ -200,11 +201,11 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           }
 
           // set the correct UUID in the VHD
-          const vhd = new Vhd(handler, path)
-          await vhd.readHeaderAndFooter()
-          vhd.footer.uuid = packUuid(vdi.uuid)
-          await vhd.readBlockAllocationTable() // required by writeFooter()
-          await vhd.writeFooter()
+          await Disposable.use(openVhd(handler, path), async vhd => {
+            vhd.footer.uuid = packUuid(vdi.uuid)
+            await vhd.readBlockAllocationTable() // required by writeFooter()
+            await vhd.writeFooter()
+          })
         })
       )
       return {

package/writers/_checkVhd.js

@@ -1,5 +1,6 @@
-const Vhd = require('vhd-lib').default
+const openVhd = require('vhd-lib').openVhd
+const Disposable = require('promise-toolbox/Disposable')
 
 exports.checkVhd = async function checkVhd(handler, path) {
-  await new Vhd(handler, path).readHeaderAndFooter()
+  await Disposable.use(openVhd(handler, path), () => {})
 }