@xen-orchestra/backups 0.19.0 → 0.19.1

package/Backup.js

@@ -1,6 +1,8 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const Disposable = require('promise-toolbox/Disposable.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const Disposable = require('promise-toolbox/Disposable')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compileTemplate } = require('@xen-orchestra/template')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 

package/DurablePartition.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 exports.DurablePartition = class DurablePartition {

package/ImportVmBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const assert = require('assert')
 
 const { formatFilenameDate } = require('./_filenameDate.js')

package/RemoteAdapter.js

@@ -1,13 +1,17 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const Disposable = require('promise-toolbox/Disposable.js')
-const fromCallback = require('promise-toolbox/fromCallback.js')
-const fromEvent = require('promise-toolbox/fromEvent.js')
-const pDefer = require('promise-toolbox/defer.js')
+const Disposable = require('promise-toolbox/Disposable')
+const fromCallback = require('promise-toolbox/fromCallback')
+const fromEvent = require('promise-toolbox/fromEvent')
+const pDefer = require('promise-toolbox/defer')
 const groupBy = require('lodash/groupBy.js')
 const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
 const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
+const { decorateMethodsWith } = require('@vates/decorate-with')
+const { compose } = require('@vates/compose')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
 const { v4: uuidv4 } = require('uuid')
@@ -88,9 +92,6 @@ class RemoteAdapter {
     return partition
   }
 
-  _getLvmLogicalVolumes = Disposable.factory(this._getLvmLogicalVolumes)
-  _getLvmLogicalVolumes = deduped(this._getLvmLogicalVolumes, (devicePath, pvId, vgName) => [devicePath, pvId, vgName])
-  _getLvmLogicalVolumes = debounceResourceFactory(this._getLvmLogicalVolumes)
   async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
     yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
 
@@ -102,9 +103,6 @@ class RemoteAdapter {
     }
   }
 
-  _getLvmPhysicalVolume = Disposable.factory(this._getLvmPhysicalVolume)
-  _getLvmPhysicalVolume = deduped(this._getLvmPhysicalVolume, (devicePath, partition) => [devicePath, partition?.id])
-  _getLvmPhysicalVolume = debounceResourceFactory(this._getLvmPhysicalVolume)
   async *_getLvmPhysicalVolume(devicePath, partition) {
     const args = []
     if (partition !== undefined) {
@@ -125,9 +123,6 @@ class RemoteAdapter {
     }
   }
 
-  _getPartition = Disposable.factory(this._getPartition)
-  _getPartition = deduped(this._getPartition, (devicePath, partition) => [devicePath, partition?.id])
-  _getPartition = debounceResourceFactory(this._getPartition)
   async *_getPartition(devicePath, partition) {
     const options = ['loop', 'ro']
 
@@ -180,7 +175,6 @@ class RemoteAdapter {
     })
   }
 
-  _usePartitionFiles = Disposable.factory(this._usePartitionFiles)
   async *_usePartitionFiles(diskId, partitionId, paths) {
     const path = yield this.getPartition(diskId, partitionId)
 
@@ -297,9 +291,6 @@ class RemoteAdapter {
     return this.#useVhdDirectory()
   }
 
-  getDisk = Disposable.factory(this.getDisk)
-  getDisk = deduped(this.getDisk, diskId => [diskId])
-  getDisk = debounceResourceFactory(this.getDisk)
   async *getDisk(diskId) {
     const handler = this._handler
 
@@ -336,7 +327,6 @@ class RemoteAdapter {
   // - `<partitionId>`: partitioned disk
   // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
   // - `/<vgName>/lvName>`: LVM on a raw disk
-  getPartition = Disposable.factory(this.getPartition)
   async *getPartition(diskId, partitionId) {
     const devicePath = yield this.getDisk(diskId)
     if (partitionId === undefined) {
@@ -545,8 +535,8 @@ class RemoteAdapter {
 
     // if it's a path : open all hierarchy of parent
     if (typeof paths === 'string') {
-      let vhd,
-        vhdPath = paths
+      let vhd
+      let vhdPath = paths
       do {
         const disposable = await openVhd(handler, vhdPath)
         vhd = disposable.value
@@ -626,4 +616,30 @@ Object.assign(RemoteAdapter.prototype, {
   isValidXva,
 })
 
+decorateMethodsWith(RemoteAdapter, {
+  _getLvmLogicalVolumes: compose([
+    Disposable.factory,
+    [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
+    debounceResourceFactory,
+  ]),
+
+  _getLvmPhysicalVolume: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _getPartition: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _usePartitionFiles: Disposable.factory,
+
+  getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
+
+  getPartition: Disposable.factory,
+})
+
 exports.RemoteAdapter = RemoteAdapter

package/RestoreMetadataBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
 const { PATH_DB_DUMP } = require('./_PoolMetadataBackup.js')
 

package/Task.js

@@ -1,4 +1,6 @@
-const CancelToken = require('promise-toolbox/CancelToken.js')
+'use strict'
+
+const CancelToken = require('promise-toolbox/CancelToken')
 const Zone = require('node-zone')
 
 const logAfterEnd = () => {
@@ -7,6 +9,8 @@ const logAfterEnd = () => {
 
 const noop = Function.prototype
 
+const serializeErrors = errors => (Array.isArray(errors) ? errors.map(serializeError) : errors)
+
 // Create a serializable object from an error.
 //
 // Otherwise some fields might be non-enumerable and missing from logs.
@@ -15,6 +19,7 @@ const serializeError = error =>
     ? {
         ...error, // Copy enumerable properties.
         code: error.code,
+        errors: serializeErrors(error.errors), // supports AggregateError
         message: error.message,
         name: error.name,
         stack: error.stack,

package/_PoolMetadataBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')

package/_VmBackup.js

@@ -1,11 +1,14 @@
+'use strict'
+
 const assert = require('assert')
 const findLast = require('lodash/findLast.js')
 const groupBy = require('lodash/groupBy.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const keyBy = require('lodash/keyBy.js')
 const mapValues = require('lodash/mapValues.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
+const { decorateMethodsWith } = require('@vates/decorate-with')
 const { defer } = require('golike-defer')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
@@ -21,6 +24,13 @@ const { watchStreamSize } = require('./_watchStreamSize.js')
 
 const { debug, warn } = createLogger('xo:backups:VmBackup')
 
+class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}
+
 const asyncEach = async (iterable, fn, thisArg = iterable) => {
   for (const item of iterable) {
     await fn.call(thisArg, item)
@@ -34,7 +44,7 @@ const forkDeltaExport = deltaExport =>
     },
   })
 
-exports.VmBackup = class VmBackup {
+class VmBackup {
   constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
     if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
       // don't match replicated VMs created by this very job otherwise they
@@ -125,16 +135,18 @@ exports.VmBackup = class VmBackup {
       return
     }
 
+    const errors = []
     await (parallel ? asyncMap : asyncEach)(writers, async function (writer) {
       try {
         await fn(writer)
       } catch (error) {
+        errors.push(error)
         this.delete(writer)
         warn(warnMessage, { error, writer: writer.constructor.name })
       }
     })
     if (writers.size === 0) {
-      throw new Error('all targets have failed, step: ' + warnMessage)
+      throw new AggregateError(errors, 'all targets have failed, step: ' + warnMessage)
     }
   }
 
@@ -385,7 +397,6 @@ exports.VmBackup = class VmBackup {
     this._fullVdisRequired = fullVdisRequired
   }
 
-  run = defer(this.run)
   async run($defer) {
     const settings = this._settings
     assert(
@@ -433,3 +444,8 @@ exports.VmBackup = class VmBackup {
     }
   }
 }
+exports.VmBackup = VmBackup
+
+decorateMethodsWith(VmBackup, {
+  run: defer,
+})

package/_XoMetadataBackup.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 const { DIR_XO_CONFIG_BACKUPS } = require('./RemoteAdapter.js')

package/_backupType.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.isMetadataFile = filename => filename.endsWith('.json')
 exports.isVhdFile = filename => filename.endsWith('.vhd')
 exports.isXvaFile = filename => filename.endsWith('.xva')

package/_backupWorker.js

@@ -1,11 +1,14 @@
+'use strict'
+
 require('@xen-orchestra/log/configure.js').catchGlobalErrors(
   require('@xen-orchestra/log').createLogger('xo:backups:worker')
 )
 
-const Disposable = require('promise-toolbox/Disposable.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const Disposable = require('promise-toolbox/Disposable')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compose } = require('@vates/compose')
 const { createDebounceResource } = require('@vates/disposable/debounceResource.js')
+const { decorateMethodsWith } = require('@vates/decorate-with')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { getHandler } = require('@xen-orchestra/fs')
 const { parseDuration } = require('@vates/parse-duration')
@@ -58,11 +61,6 @@ class BackupWorker {
     }).run()
   }
 
-  getAdapter = Disposable.factory(this.getAdapter)
-  getAdapter = deduped(this.getAdapter, remote => [remote.url])
-  getAdapter = compose(this.getAdapter, function (resource) {
-    return this.debounceResource(resource)
-  })
   async *getAdapter(remote) {
     const handler = getHandler(remote, this.#remoteOptions)
     await handler.sync()
@@ -77,11 +75,6 @@ class BackupWorker {
     }
   }
 
-  getXapi = Disposable.factory(this.getXapi)
-  getXapi = deduped(this.getXapi, ({ url }) => [url])
-  getXapi = compose(this.getXapi, function (resource) {
-    return this.debounceResource(resource)
-  })
   async *getXapi({ credentials: { username: user, password }, ...opts }) {
     const xapi = new Xapi({
       ...this.#xapiOptions,
@@ -103,6 +96,30 @@ class BackupWorker {
   }
 }
 
+decorateMethodsWith(BackupWorker, {
+  getAdapter: compose([
+    Disposable.factory,
+    [deduped, remote => [remote.url]],
+    [
+      compose,
+      function (resource) {
+        return this.debounceResource(resource)
+      },
+    ],
+  ]),
+
+  getXapi: compose([
+    Disposable.factory,
+    [deduped, xapi => [xapi.url]],
+    [
+      compose,
+      function (resource) {
+        return this.debounceResource(resource)
+      },
+    ],
+  ]),
+})
+
 // Received message:
 //
 // Message {

package/_cancelableMap.js

@@ -1,5 +1,7 @@
-const cancelable = require('promise-toolbox/cancelable.js')
-const CancelToken = require('promise-toolbox/CancelToken.js')
+'use strict'
+
+const cancelable = require('promise-toolbox/cancelable')
+const CancelToken = require('promise-toolbox/CancelToken')
 
 // Similar to `Promise.all` + `map` but pass a cancel token to the callback
 //

package/_cleanVm.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')

package/_deltaVm.js

@@ -1,7 +1,9 @@
+'use strict'
+
 const compareVersions = require('compare-versions')
 const find = require('lodash/find.js')
 const groupBy = require('lodash/groupBy.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const omit = require('lodash/omit.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { CancelToken } = require('promise-toolbox')

package/_extractIdsFromSimplePattern.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.extractIdsFromSimplePattern = function extractIdsFromSimplePattern(pattern) {
   if (pattern === undefined) {
     return []

package/_filenameDate.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { utcFormat, utcParse } = require('d3-time-format')
 
 // Format a date in ISO 8601 in a safe way to be used in filenames

package/_forkStreamUnpipe.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const eos = require('end-of-stream')
 const { PassThrough } = require('stream')
 

package/_getOldEntries.js

@@ -1,3 +1,5 @@
+'use strict'
+
 // returns all entries but the last retention-th
 exports.getOldEntries = function getOldEntries(retention, entries) {
   return entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries

package/_getTmpDir.js

@@ -1,4 +1,6 @@
-const Disposable = require('promise-toolbox/Disposable.js')
+'use strict'
+
+const Disposable = require('promise-toolbox/Disposable')
 const { join } = require('path')
 const { mkdir, rmdir } = require('fs-extra')
 const { tmpdir } = require('os')

package/_getVmBackupDir.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const BACKUP_DIR = 'xo-vm-backups'
 exports.BACKUP_DIR = BACKUP_DIR
 

package/_isValidXva.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const assert = require('assert')
 
 const COMPRESSED_MAGIC_NUMBERS = [

package/_listPartitions.js

@@ -1,4 +1,6 @@
-const fromCallback = require('promise-toolbox/fromCallback.js')
+'use strict'
+
+const fromCallback = require('promise-toolbox/fromCallback')
 const { createLogger } = require('@xen-orchestra/log')
 const { createParser } = require('parse-pairs')
 const { execFile } = require('child_process')

package/_lvm.js

@@ -1,4 +1,6 @@
-const fromCallback = require('promise-toolbox/fromCallback.js')
+'use strict'
+
+const fromCallback = require('promise-toolbox/fromCallback')
 const { createParser } = require('parse-pairs')
 const { execFile } = require('child_process')
 

package/_watchStreamSize.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.watchStreamSize = function watchStreamSize(stream, container = { size: 0 }) {
   stream.on('data', data => {
     container.size += data.length

package/formatVmBackups.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const mapValues = require('lodash/mapValues.js')
 const { dirname } = require('path')
 

package/merge-worker/cli.js

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+'use strict'
+
 const { catchGlobalErrors } = require('@xen-orchestra/log/configure.js')
 const { createLogger } = require('@xen-orchestra/log')
 const { getSyncedHandler } = require('@xen-orchestra/fs')
@@ -41,13 +43,32 @@ const main = Disposable.wrap(async function* main(args) {
   let taskFiles
   while ((taskFiles = await listRetry()) !== undefined) {
     const taskFileBasename = min(taskFiles)
+    const previousTaskFile = join(CLEAN_VM_QUEUE, taskFileBasename)
     const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
 
     // move this task to the end
-    await handler.rename(join(CLEAN_VM_QUEUE, taskFileBasename), taskFile)
+    try {
+      await handler.rename(previousTaskFile, taskFile)
+    } catch (error) {
+      // this error occurs if the task failed too many times (i.e. too many `_` prefixes)
+      // there is nothing more that can be done
+      if (error.code === 'ENAMETOOLONG') {
+        await handler.unlink(previousTaskFile)
+      }
+
+      throw error
+    }
+
     try {
       const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
-      await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+      try {
+        await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+      } catch (error) {
+        // consider the clean successful if the VM dir is missing
+        if (error.code !== 'ENOENT') {
+          throw error
+        }
+      }
 
       handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
     } catch (error) {

package/merge-worker/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { join, resolve } = require('path')
 const { spawn } = require('child_process')
 const { check } = require('proper-lockfile')

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.19.0",
+  "version": "0.19.1",
   "engines": {
     "node": ">=14.6"
   },
@@ -17,6 +17,7 @@
   },
   "dependencies": {
     "@vates/compose": "^2.1.0",
+    "@vates/decorate-with": "^1.0.0",
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
@@ -32,15 +33,14 @@
     "lodash": "^4.17.20",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.1.0",
-    "promise-toolbox": "^0.20.0",
+    "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
-    "pump": "^3.0.0",
     "uuid": "^8.3.2",
     "vhd-lib": "^3.1.0",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.8.5"
+    "@xen-orchestra/xapi": "^0.9.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/parseMetadataBackupId.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { DIR_XO_CONFIG_BACKUPS, DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
 
 exports.parseMetadataBackupId = function parseMetadataBackupId(backupId) {

package/runBackupWorker.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const path = require('path')
 const { createLogger } = require('@xen-orchestra/log')
 const { fork } = require('child_process')

package/writers/DeltaBackupWriter.js

@@ -1,7 +1,9 @@
+'use strict'
+
 const assert = require('assert')
 const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')

package/writers/DeltaReplicationWriter.js

@@ -1,5 +1,7 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
 const { formatFilenameDate } = require('../_filenameDate.js')

package/writers/FullBackupWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { formatFilenameDate } = require('../_filenameDate.js')
 const { getOldEntries } = require('../_getOldEntries.js')
 const { getVmBackupDir } = require('../_getVmBackupDir.js')

package/writers/FullReplicationWriter.js

@@ -1,4 +1,6 @@
-const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+'use strict'
+
+const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 

package/writers/_AbstractDeltaWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { AbstractWriter } = require('./_AbstractWriter.js')
 
 exports.AbstractDeltaWriter = class AbstractDeltaWriter extends AbstractWriter {

package/writers/_AbstractFullWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { AbstractWriter } = require('./_AbstractWriter.js')
 
 exports.AbstractFullWriter = class AbstractFullWriter extends AbstractWriter {

package/writers/_AbstractWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.AbstractWriter = class AbstractWriter {
   constructor({ backup, settings }) {
     this._backup = backup

package/writers/_MixinBackupWriter.js

@@ -1,7 +1,9 @@
+'use strict'
+
 const { createLogger } = require('@xen-orchestra/log')
 const { join } = require('path')
 
-const { BACKUP_DIR, getVmBackupDir } = require('../_getVmBackupDir.js')
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
 const MergeWorker = require('../merge-worker/index.js')
 const { formatFilenameDate } = require('../_filenameDate.js')
 

package/writers/_MixinReplicationWriter.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.MixinReplicationWriter = (BaseClass = Object) =>
   class MixinReplicationWriter extends BaseClass {
     constructor({ sr, ...rest }) {

package/writers/_checkVhd.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const openVhd = require('vhd-lib').openVhd
 const Disposable = require('promise-toolbox/Disposable')
 

package/writers/_listReplicatedVms.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const getReplicatedVmDatetime = vm => {
   const { 'xo:backup:datetime': datetime = vm.name_label.slice(-17, -1) } = vm.other_config
   return datetime

package/writers/_packUuid.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const PARSE_UUID_RE = /-/g
 
 exports.packUuid = function packUuid(uuid) {