@xen-orchestra/backups 0.18.2 → 0.19.1

package/Backup.js

@@ -1,6 +1,8 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const Disposable = require('promise-toolbox/Disposable.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const Disposable = require('promise-toolbox/Disposable')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compileTemplate } = require('@xen-orchestra/template')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 

package/DurablePartition.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 exports.DurablePartition = class DurablePartition {

package/ImportVmBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const assert = require('assert')
 
 const { formatFilenameDate } = require('./_filenameDate.js')

package/RemoteAdapter.js

@@ -1,13 +1,17 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const Disposable = require('promise-toolbox/Disposable.js')
-const fromCallback = require('promise-toolbox/fromCallback.js')
-const fromEvent = require('promise-toolbox/fromEvent.js')
-const pDefer = require('promise-toolbox/defer.js')
+const Disposable = require('promise-toolbox/Disposable')
+const fromCallback = require('promise-toolbox/fromCallback')
+const fromEvent = require('promise-toolbox/fromEvent')
+const pDefer = require('promise-toolbox/defer')
 const groupBy = require('lodash/groupBy.js')
 const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
 const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
+const { decorateMethodsWith } = require('@vates/decorate-with')
+const { compose } = require('@vates/compose')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
 const { v4: uuidv4 } = require('uuid')
@@ -88,9 +92,6 @@ class RemoteAdapter {
     return partition
   }
 
-  _getLvmLogicalVolumes = Disposable.factory(this._getLvmLogicalVolumes)
-  _getLvmLogicalVolumes = deduped(this._getLvmLogicalVolumes, (devicePath, pvId, vgName) => [devicePath, pvId, vgName])
-  _getLvmLogicalVolumes = debounceResourceFactory(this._getLvmLogicalVolumes)
   async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
     yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
 
@@ -102,9 +103,6 @@ class RemoteAdapter {
     }
   }
 
-  _getLvmPhysicalVolume = Disposable.factory(this._getLvmPhysicalVolume)
-  _getLvmPhysicalVolume = deduped(this._getLvmPhysicalVolume, (devicePath, partition) => [devicePath, partition?.id])
-  _getLvmPhysicalVolume = debounceResourceFactory(this._getLvmPhysicalVolume)
   async *_getLvmPhysicalVolume(devicePath, partition) {
     const args = []
     if (partition !== undefined) {
@@ -125,9 +123,6 @@ class RemoteAdapter {
     }
   }
 
-  _getPartition = Disposable.factory(this._getPartition)
-  _getPartition = deduped(this._getPartition, (devicePath, partition) => [devicePath, partition?.id])
-  _getPartition = debounceResourceFactory(this._getPartition)
   async *_getPartition(devicePath, partition) {
     const options = ['loop', 'ro']
 
@@ -180,7 +175,6 @@ class RemoteAdapter {
     })
   }
 
-  _usePartitionFiles = Disposable.factory(this._usePartitionFiles)
   async *_usePartitionFiles(diskId, partitionId, paths) {
     const path = yield this.getPartition(diskId, partitionId)
 
@@ -230,8 +224,8 @@ class RemoteAdapter {
   async deleteDeltaVmBackups(backups) {
     const handler = this._handler
 
-    // unused VHDs will be detected by `cleanVm`
-    await asyncMapSettled(backups, ({ _filename }) => VhdAbstract.unlink(handler, _filename))
+    // this will delete the json, unused VHDs will be detected by `cleanVm`
+    await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
   }
 
   async deleteMetadataBackup(backupId) {
@@ -277,6 +271,12 @@ class RemoteAdapter {
       delta !== undefined && this.deleteDeltaVmBackups(delta),
       full !== undefined && this.deleteFullVmBackups(full),
     ])
+
+    const dirs = new Set(files.map(file => dirname(file)))
+    for (const dir of dirs) {
+      // don't merge in main process, unused VHDs will be merged in the next backup run
+      await this.cleanVm(dir, { remove: true, onLog: warn })
+    }
   }
 
   #getCompressionType() {
@@ -291,9 +291,6 @@ class RemoteAdapter {
     return this.#useVhdDirectory()
   }
 
-  getDisk = Disposable.factory(this.getDisk)
-  getDisk = deduped(this.getDisk, diskId => [diskId])
-  getDisk = debounceResourceFactory(this.getDisk)
   async *getDisk(diskId) {
     const handler = this._handler
 
@@ -330,7 +327,6 @@ class RemoteAdapter {
   // - `<partitionId>`: partitioned disk
   // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
   // - `/<vgName>/lvName>`: LVM on a raw disk
-  getPartition = Disposable.factory(this.getPartition)
   async *getPartition(diskId, partitionId) {
     const devicePath = yield this.getDisk(diskId)
     if (partitionId === undefined) {
@@ -359,9 +355,14 @@ class RemoteAdapter {
     const handler = this._handler
 
     const backups = { __proto__: null }
-    await asyncMap(await handler.list(BACKUP_DIR), async vmUuid => {
-      const vmBackups = await this.listVmBackups(vmUuid)
-      backups[vmUuid] = vmBackups
+    await asyncMap(await handler.list(BACKUP_DIR), async entry => {
+      // ignore hidden and lock files
+      if (entry[0] !== '.' && !entry.endsWith('.lock')) {
+        const vmBackups = await this.listVmBackups(entry)
+        if (vmBackups.length !== 0) {
+          backups[entry] = vmBackups
+        }
+      }
     })
 
     return backups
@@ -534,8 +535,8 @@ class RemoteAdapter {
 
     // if it's a path : open all hierarchy of parent
     if (typeof paths === 'string') {
-      let vhd,
-        vhdPath = paths
+      let vhd
+      let vhdPath = paths
       do {
         const disposable = await openVhd(handler, vhdPath)
         vhd = disposable.value
@@ -615,4 +616,30 @@ Object.assign(RemoteAdapter.prototype, {
   isValidXva,
 })
 
+decorateMethodsWith(RemoteAdapter, {
+  _getLvmLogicalVolumes: compose([
+    Disposable.factory,
+    [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
+    debounceResourceFactory,
+  ]),
+
+  _getLvmPhysicalVolume: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _getPartition: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _usePartitionFiles: Disposable.factory,
+
+  getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
+
+  getPartition: Disposable.factory,
+})
+
 exports.RemoteAdapter = RemoteAdapter

package/RestoreMetadataBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
 const { PATH_DB_DUMP } = require('./_PoolMetadataBackup.js')
 

package/Task.js

@@ -1,4 +1,6 @@
-const CancelToken = require('promise-toolbox/CancelToken.js')
+'use strict'
+
+const CancelToken = require('promise-toolbox/CancelToken')
 const Zone = require('node-zone')
 
 const logAfterEnd = () => {
@@ -7,6 +9,8 @@ const logAfterEnd = () => {
 
 const noop = Function.prototype
 
+const serializeErrors = errors => (Array.isArray(errors) ? errors.map(serializeError) : errors)
+
 // Create a serializable object from an error.
 //
 // Otherwise some fields might be non-enumerable and missing from logs.
@@ -15,6 +19,7 @@ const serializeError = error =>
     ? {
         ...error, // Copy enumerable properties.
         code: error.code,
+        errors: serializeErrors(error.errors), // supports AggregateError
         message: error.message,
         name: error.name,
         stack: error.stack,

package/_PoolMetadataBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')

package/_VmBackup.js

@@ -1,11 +1,14 @@
+'use strict'
+
 const assert = require('assert')
 const findLast = require('lodash/findLast.js')
 const groupBy = require('lodash/groupBy.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const keyBy = require('lodash/keyBy.js')
 const mapValues = require('lodash/mapValues.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
+const { decorateMethodsWith } = require('@vates/decorate-with')
 const { defer } = require('golike-defer')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
@@ -21,6 +24,13 @@ const { watchStreamSize } = require('./_watchStreamSize.js')
 
 const { debug, warn } = createLogger('xo:backups:VmBackup')
 
+class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}
+
 const asyncEach = async (iterable, fn, thisArg = iterable) => {
   for (const item of iterable) {
     await fn.call(thisArg, item)
@@ -34,10 +44,11 @@ const forkDeltaExport = deltaExport =>
     },
   })
 
-exports.VmBackup = class VmBackup {
+class VmBackup {
   constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
-    if (vm.other_config['xo:backup:job'] === job.id) {
-      // otherwise replicated VMs would be matched and replicated again and again
+    if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
+      // don't match replicated VMs created by this very job otherwise they
+      // will be replicated again and again
       throw new Error('cannot backup a VM created by this very job')
     }
 
@@ -124,16 +135,18 @@ exports.VmBackup = class VmBackup {
       return
     }
 
+    const errors = []
     await (parallel ? asyncMap : asyncEach)(writers, async function (writer) {
       try {
         await fn(writer)
       } catch (error) {
+        errors.push(error)
         this.delete(writer)
         warn(warnMessage, { error, writer: writer.constructor.name })
       }
     })
     if (writers.size === 0) {
-      throw new Error('all targets have failed, step: ' + warnMessage)
+      throw new AggregateError(errors, 'all targets have failed, step: ' + warnMessage)
     }
   }
 
@@ -384,7 +397,6 @@ exports.VmBackup = class VmBackup {
     this._fullVdisRequired = fullVdisRequired
   }
 
-  run = defer(this.run)
   async run($defer) {
     const settings = this._settings
     assert(
@@ -432,3 +444,8 @@ exports.VmBackup = class VmBackup {
     }
   }
 }
+exports.VmBackup = VmBackup
+
+decorateMethodsWith(VmBackup, {
+  run: defer,
+})

package/_XoMetadataBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 const { DIR_XO_CONFIG_BACKUPS } = require('./RemoteAdapter.js')

package/_backupType.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.isMetadataFile = filename => filename.endsWith('.json')
 exports.isVhdFile = filename => filename.endsWith('.vhd')
 exports.isXvaFile = filename => filename.endsWith('.xva')

package/_backupWorker.js

@@ -1,11 +1,14 @@
+'use strict'
+
 require('@xen-orchestra/log/configure.js').catchGlobalErrors(
   require('@xen-orchestra/log').createLogger('xo:backups:worker')
 )
 
-const Disposable = require('promise-toolbox/Disposable.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const Disposable = require('promise-toolbox/Disposable')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compose } = require('@vates/compose')
 const { createDebounceResource } = require('@vates/disposable/debounceResource.js')
+const { decorateMethodsWith } = require('@vates/decorate-with')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { getHandler } = require('@xen-orchestra/fs')
 const { parseDuration } = require('@vates/parse-duration')
@@ -58,11 +61,6 @@ class BackupWorker {
     }).run()
   }
 
-  getAdapter = Disposable.factory(this.getAdapter)
-  getAdapter = deduped(this.getAdapter, remote => [remote.url])
-  getAdapter = compose(this.getAdapter, function (resource) {
-    return this.debounceResource(resource)
-  })
   async *getAdapter(remote) {
     const handler = getHandler(remote, this.#remoteOptions)
     await handler.sync()
@@ -77,11 +75,6 @@ class BackupWorker {
     }
   }
 
-  getXapi = Disposable.factory(this.getXapi)
-  getXapi = deduped(this.getXapi, ({ url }) => [url])
-  getXapi = compose(this.getXapi, function (resource) {
-    return this.debounceResource(resource)
-  })
   async *getXapi({ credentials: { username: user, password }, ...opts }) {
     const xapi = new Xapi({
       ...this.#xapiOptions,
@@ -103,6 +96,30 @@ class BackupWorker {
   }
 }
 
+decorateMethodsWith(BackupWorker, {
+  getAdapter: compose([
+    Disposable.factory,
+    [deduped, remote => [remote.url]],
+    [
+      compose,
+      function (resource) {
+        return this.debounceResource(resource)
+      },
+    ],
+  ]),
+
+  getXapi: compose([
+    Disposable.factory,
+    [deduped, xapi => [xapi.url]],
+    [
+      compose,
+      function (resource) {
+        return this.debounceResource(resource)
+      },
+    ],
+  ]),
+})
+
 // Received message:
 //
 // Message {

package/_cancelableMap.js

@@ -1,5 +1,7 @@
-const cancelable = require('promise-toolbox/cancelable.js')
-const CancelToken = require('promise-toolbox/CancelToken.js')
+'use strict'
+
+const cancelable = require('promise-toolbox/cancelable')
+const CancelToken = require('promise-toolbox/CancelToken')
 
 // Similar to `Promise.all` + `map` but pass a cancel token to the callback
 //

package/_cleanVm.js

@@ -1,7 +1,10 @@
+'use strict'
+
 const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
+const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
 const { dirname, resolve } = require('path')
 const { DISK_TYPES } = Constants
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
@@ -82,7 +85,6 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
     )
 
     clearInterval(handle)
-
     await Promise.all([
       VhdAbstract.rename(handler, parent, child),
       asyncMap(children.slice(0, -1), child => {
@@ -100,10 +102,11 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
 
 const noop = Function.prototype
 
-const INTERRUPTED_VHDS_REG = /^(?:(.+)\/)?\.(.+)\.merge.json$/
+const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
 const listVhds = async (handler, vmDir) => {
-  const vhds = []
-  const interruptedVhds = new Set()
+  const vhds = new Set()
+  const aliases = {}
+  const interruptedVhds = new Map()
 
   await asyncMap(
     await handler.list(`${vmDir}/vdis`, {
@@ -118,25 +121,77 @@ const listVhds = async (handler, vmDir) => {
         async vdiDir => {
           const list = await handler.list(vdiDir, {
             filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
-            prependDir: true,
           })
-
+          aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
           list.forEach(file => {
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
-              vhds.push(file)
+              vhds.add(`${vdiDir}/${file}`)
             } else {
-              const [, dir, file] = res
-              interruptedVhds.add(`${dir}/${file}`)
+              interruptedVhds.set(`${vdiDir}/${res[1]}`, `${vdiDir}/${file}`)
             }
           })
         }
       )
   )
 
-  return { vhds, interruptedVhds }
+  return { vhds, interruptedVhds, aliases }
 }
 
+async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog = noop, remove = false }) {
+  const aliasFound = []
+  for (const path of aliasPaths) {
+    const target = await resolveVhdAlias(handler, path)
+
+    if (!isVhdFile(target)) {
+      onLog(`Alias ${path} references a non vhd target:  ${target}`)
+      if (remove) {
+        await handler.unlink(target)
+        await handler.unlink(path)
+      }
+      continue
+    }
+
+    try {
+      const { dispose } = await openVhd(handler, target)
+      try {
+        await dispose()
+      } catch (e) {
+        // error during dispose should not trigger a deletion
+      }
+    } catch (error) {
+      onLog(`target ${target} of alias ${path} is missing or broken`, { error })
+      if (remove) {
+        try {
+          await VhdAbstract.unlink(handler, path)
+        } catch (e) {
+          if (e.code !== 'ENOENT') {
+            onLog(`Error while deleting target ${target} of alias ${path}`, { error: e })
+          }
+        }
+      }
+      continue
+    }
+
+    aliasFound.push(resolve('/', target))
+  }
+
+  const entries = await handler.list(targetDataRepository, {
+    ignoreMissing: true,
+    prependDir: true,
+  })
+
+  entries.forEach(async entry => {
+    if (!aliasFound.includes(entry)) {
+      onLog(`the Vhd  ${entry} is not referenced by a an alias`)
+      if (remove) {
+        await VhdAbstract.unlink(handler, entry)
+      }
+    }
+  })
+}
+exports.checkAliases = checkAliases
+
 const defaultMergeLimiter = limitConcurrency(1)
 
 exports.cleanVm = async function cleanVm(
@@ -147,18 +202,16 @@ exports.cleanVm = async function cleanVm(
 
   const handler = this._handler
 
-  const vhds = new Set()
   const vhdsToJSons = new Set()
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
-  const vhdsList = await listVhds(handler, vmDir)
+  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir)
 
   // remove broken VHDs
-  await asyncMap(vhdsList.vhds, async path => {
+  await asyncMap(vhds, async path => {
     try {
-      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !vhdsList.interruptedVhds.has(path) }), vhd => {
-        vhds.add(path)
+      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
         if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
           const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
           vhdParents[path] = parent
@@ -173,6 +226,7 @@ exports.cleanVm = async function cleanVm(
         }
       })
     } catch (error) {
+      vhds.delete(path)
       onLog(`error while checking the VHD with path ${path}`, { error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
         onLog(`deleting broken ${path}`)
@@ -181,7 +235,28 @@ exports.cleanVm = async function cleanVm(
     }
   })
 
-  // @todo : add check for data folder of alias not referenced in a valid alias
+  // remove interrupted merge states for missing VHDs
+  for (const interruptedVhd of interruptedVhds.keys()) {
+    if (!vhds.has(interruptedVhd)) {
+      const statePath = interruptedVhds.get(interruptedVhd)
+      interruptedVhds.delete(interruptedVhd)
+
+      onLog('orphan merge state', {
+        mergeStatePath: statePath,
+        missingVhdPath: interruptedVhd,
+      })
+      if (remove) {
+        onLog(`deleting orphan merge state ${statePath}`)
+        await handler.unlink(statePath)
+      }
+    }
+  }
+
+  // check if alias are correct
+  // check if all vhd in data subfolder have a corresponding alias
+  await asyncMap(Object.keys(aliases), async dir => {
+    await checkAliases(aliases[dir], `${dir}/data`, { handler, onLog, remove })
+  })
 
   // remove VHDs with missing ancestors
   {
@@ -344,9 +419,9 @@ exports.cleanVm = async function cleanVm(
     })
 
     // merge interrupted VHDs
-    vhdsList.interruptedVhds.forEach(parent => {
+    for (const parent of interruptedVhds.keys()) {
       vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
-    })
+    }
 
     Object.values(vhdChainsToMerge).forEach(chain => {
       if (chain !== undefined) {

package/_deltaVm.js

@@ -1,7 +1,9 @@
+'use strict'
+
 const compareVersions = require('compare-versions')
 const find = require('lodash/find.js')
 const groupBy = require('lodash/groupBy.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const omit = require('lodash/omit.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { CancelToken } = require('promise-toolbox')

package/_extractIdsFromSimplePattern.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.extractIdsFromSimplePattern = function extractIdsFromSimplePattern(pattern) {
   if (pattern === undefined) {
     return []

package/_filenameDate.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { utcFormat, utcParse } = require('d3-time-format')
 
 // Format a date in ISO 8601 in a safe way to be used in filenames

package/_forkStreamUnpipe.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const eos = require('end-of-stream')
 const { PassThrough } = require('stream')
 

package/_getOldEntries.js

@@ -1,3 +1,5 @@
+'use strict'
+
 // returns all entries but the last retention-th
 exports.getOldEntries = function getOldEntries(retention, entries) {
   return entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries

package/_getTmpDir.js

@@ -1,4 +1,6 @@
-const Disposable = require('promise-toolbox/Disposable.js')
+'use strict'
+
+const Disposable = require('promise-toolbox/Disposable')
 const { join } = require('path')
 const { mkdir, rmdir } = require('fs-extra')
 const { tmpdir } = require('os')

package/_getVmBackupDir.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const BACKUP_DIR = 'xo-vm-backups'
 exports.BACKUP_DIR = BACKUP_DIR
 

package/_isValidXva.js

@@ -1,11 +1,26 @@
+'use strict'
+
 const assert = require('assert')
 
-const isGzipFile = async (handler, fd) => {
+const COMPRESSED_MAGIC_NUMBERS = [
   // https://tools.ietf.org/html/rfc1952.html#page-5
-  const magicNumber = Buffer.allocUnsafe(2)
+  Buffer.from('1F8B', 'hex'),
+
+  // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
+  Buffer.from('28B52FFD', 'hex'),
+]
+const MAGIC_NUMBER_MAX_LENGTH = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(_ => _.length))
+
+const isCompressedFile = async (handler, fd) => {
+  const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
+  assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
 
-  assert.strictEqual((await handler.read(fd, magicNumber, 0)).bytesRead, magicNumber.length)
-  return magicNumber[0] === 31 && magicNumber[1] === 139
+  for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
+    if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
+      return true
+    }
+  }
+  return false
 }
 
 // TODO: better check?
@@ -43,8 +58,8 @@ async function isValidXva(path) {
         return false
       }
 
-      return (await isGzipFile(handler, fd))
-        ? true // gzip files cannot be validated at this time
+      return (await isCompressedFile(handler, fd))
+        ? true // compressed files cannot be validated at this time
         : await isValidTar(handler, size, fd)
     } finally {
       handler.closeFile(fd).catch(noop)

package/_listPartitions.js

@@ -1,4 +1,6 @@
-const fromCallback = require('promise-toolbox/fromCallback.js')
+'use strict'
+
+const fromCallback = require('promise-toolbox/fromCallback')
 const { createLogger } = require('@xen-orchestra/log')
 const { createParser } = require('parse-pairs')
 const { execFile } = require('child_process')

package/_lvm.js

@@ -1,4 +1,6 @@
-const fromCallback = require('promise-toolbox/fromCallback.js')
+'use strict'
+
+const fromCallback = require('promise-toolbox/fromCallback')
 const { createParser } = require('parse-pairs')
 const { execFile } = require('child_process')
 

package/_watchStreamSize.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.watchStreamSize = function watchStreamSize(stream, container = { size: 0 }) {
   stream.on('data', data => {
     container.size += data.length

package/formatVmBackups.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const mapValues = require('lodash/mapValues.js')
 const { dirname } = require('path')
 

package/merge-worker/cli.js

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+'use strict'
+
 const { catchGlobalErrors } = require('@xen-orchestra/log/configure.js')
 const { createLogger } = require('@xen-orchestra/log')
 const { getSyncedHandler } = require('@xen-orchestra/fs')
@@ -41,13 +43,32 @@ const main = Disposable.wrap(async function* main(args) {
   let taskFiles
   while ((taskFiles = await listRetry()) !== undefined) {
     const taskFileBasename = min(taskFiles)
+    const previousTaskFile = join(CLEAN_VM_QUEUE, taskFileBasename)
     const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
 
     // move this task to the end
-    await handler.rename(join(CLEAN_VM_QUEUE, taskFileBasename), taskFile)
+    try {
+      await handler.rename(previousTaskFile, taskFile)
+    } catch (error) {
+      // this error occurs if the task failed too many times (i.e. too many `_` prefixes)
+      // there is nothing more that can be done
+      if (error.code === 'ENAMETOOLONG') {
+        await handler.unlink(previousTaskFile)
+      }
+
+      throw error
+    }
+
     try {
       const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
-      await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+      try {
+        await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+      } catch (error) {
+        // consider the clean successful if the VM dir is missing
+        if (error.code !== 'ENOENT') {
+          throw error
+        }
+      }
 
       handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
     } catch (error) {

package/merge-worker/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { join, resolve } = require('path')
 const { spawn } = require('child_process')
 const { check } = require('proper-lockfile')

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.18.2",
+  "version": "0.19.1",
   "engines": {
     "node": ">=14.6"
   },
@@ -17,10 +17,11 @@
   },
   "dependencies": {
     "@vates/compose": "^2.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.19.3",
+    "@xen-orchestra/fs": "^0.20.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
@@ -32,15 +33,14 @@
     "lodash": "^4.17.20",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.1.0",
-    "promise-toolbox": "^0.20.0",
+    "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
-    "pump": "^3.0.0",
     "uuid": "^8.3.2",
-    "vhd-lib": "^2.1.0",
+    "vhd-lib": "^3.1.0",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.8.5"
+    "@xen-orchestra/xapi": "^0.9.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/parseMetadataBackupId.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { DIR_XO_CONFIG_BACKUPS, DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
 
 exports.parseMetadataBackupId = function parseMetadataBackupId(backupId) {

package/runBackupWorker.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const path = require('path')
 const { createLogger } = require('@xen-orchestra/log')
 const { fork } = require('child_process')

package/writers/DeltaBackupWriter.js

@@ -1,7 +1,9 @@
+'use strict'
+
 const assert = require('assert')
 const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
@@ -40,7 +42,14 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
         await asyncMap(vhds, async path => {
           try {
             await checkVhdChain(handler, path)
-            found = found || (await adapter.isMergeableParent(packedBaseUuid, path))
+            // Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
+            //
+            // since all the checks of a path are done in parallel, found would be containing
+            // only the last answer of isMergeableParent which is probably not the right one
+            // this led to the support tickets  https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
+
+            const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
+            found = found || isMergeable
           } catch (error) {
             warn('checkBaseVdis', { error })
             await ignoreErrors.call(VhdAbstract.unlink(handler, path))

package/writers/DeltaReplicationWriter.js

@@ -1,5 +1,7 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
 const { formatFilenameDate } = require('../_filenameDate.js')

package/writers/FullBackupWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { formatFilenameDate } = require('../_filenameDate.js')
 const { getOldEntries } = require('../_getOldEntries.js')
 const { getVmBackupDir } = require('../_getVmBackupDir.js')

package/writers/FullReplicationWriter.js

@@ -1,4 +1,6 @@
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+'use strict'
+
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 

package/writers/_AbstractDeltaWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { AbstractWriter } = require('./_AbstractWriter.js')
 
 exports.AbstractDeltaWriter = class AbstractDeltaWriter extends AbstractWriter {

package/writers/_AbstractFullWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { AbstractWriter } = require('./_AbstractWriter.js')
 
 exports.AbstractFullWriter = class AbstractFullWriter extends AbstractWriter {

package/writers/_AbstractWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.AbstractWriter = class AbstractWriter {
   constructor({ backup, settings }) {
     this._backup = backup

package/writers/_MixinBackupWriter.js

@@ -1,7 +1,9 @@
+'use strict'
+
 const { createLogger } = require('@xen-orchestra/log')
 const { join } = require('path')
 
-const { BACKUP_DIR, getVmBackupDir } = require('../_getVmBackupDir.js')
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
 const MergeWorker = require('../merge-worker/index.js')
 const { formatFilenameDate } = require('../_filenameDate.js')
 
@@ -44,13 +46,14 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
 
     async afterBackup() {
       const { disableMergeWorker } = this._backup.config
+      // merge worker only compatible with local remotes
+      const { handler } = this._adapter
+      const willMergeInWorker = !disableMergeWorker && typeof handler._getRealPath === 'function'
 
-      const { merge } = await this._cleanVm({ remove: true, merge: disableMergeWorker })
+      const { merge } = await this._cleanVm({ remove: true, merge: !willMergeInWorker })
       await this.#lock.dispose()
 
-      // merge worker only compatible with local remotes
-      const { handler } = this._adapter
-      if (merge && !disableMergeWorker && typeof handler._getRealPath === 'function') {
+      if (merge && willMergeInWorker) {
         const taskFile =
           join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())) +
           '-' +

package/writers/_MixinReplicationWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.MixinReplicationWriter = (BaseClass = Object) =>
   class MixinReplicationWriter extends BaseClass {
     constructor({ sr, ...rest }) {

package/writers/_checkVhd.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const openVhd = require('vhd-lib').openVhd
 const Disposable = require('promise-toolbox/Disposable')
 

package/writers/_listReplicatedVms.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const getReplicatedVmDatetime = vm => {
   const { 'xo:backup:datetime': datetime = vm.name_label.slice(-17, -1) } = vm.other_config
   return datetime

package/writers/_packUuid.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const PARSE_UUID_RE = /-/g
 
 exports.packUuid = function packUuid(uuid) {