@xen-orchestra/backups 0.31.0 → 0.33.0

package/Backup.js

@@ -12,6 +12,7 @@ const { PoolMetadataBackup } = require('./_PoolMetadataBackup.js')
 const { Task } = require('./Task.js')
 const { VmBackup } = require('./_VmBackup.js')
 const { XoMetadataBackup } = require('./_XoMetadataBackup.js')
+const createStreamThrottle = require('./_createStreamThrottle.js')
 
 const noop = Function.prototype
 
@@ -40,6 +41,7 @@ const DEFAULT_VM_SETTINGS = {
   fullInterval: 0,
   healthCheckSr: undefined,
   healthCheckVmsWithTags: [],
+  maxExportRate: 0,
   maxMergedDeltasPerRun: Infinity,
   offlineBackup: false,
   offlineSnapshot: false,
@@ -226,9 +228,11 @@ exports.Backup = class Backup {
     // FIXME: proper SimpleIdPattern handling
     const getSnapshotNameLabel = this._getSnapshotNameLabel
     const schedule = this._schedule
+    const settings = this._settings
+
+    const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
-    const settings = this._settings
     await Disposable.use(
       Disposable.all(
         extractIdsFromSimplePattern(job.srs).map(id =>
@@ -265,23 +269,35 @@ exports.Backup = class Backup {
         const allSettings = this._job.settings
         const baseSettings = this._baseSettings
 
-        const handleVm = vmUuid =>
-          runTask({ name: 'backup VM', data: { type: 'VM', id: vmUuid } }, () =>
-            Disposable.use(this._getRecord('VM', vmUuid), vm =>
-              new VmBackup({
-                baseSettings,
-                config,
-                getSnapshotNameLabel,
-                healthCheckSr,
-                job,
-                remoteAdapters,
-                schedule,
-                settings: { ...settings, ...allSettings[vm.uuid] },
-                srs,
-                vm,
-              }).run()
-            )
+        const handleVm = vmUuid => {
+          const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
+
+          return this._getRecord('VM', vmUuid).then(
+            disposableVm =>
+              Disposable.use(disposableVm, vm => {
+                taskStart.data.name_label = vm.name_label
+                return runTask(taskStart, () =>
+                  new VmBackup({
+                    baseSettings,
+                    config,
+                    getSnapshotNameLabel,
+                    healthCheckSr,
+                    job,
+                    remoteAdapters,
+                    schedule,
+                    settings: { ...settings, ...allSettings[vm.uuid] },
+                    srs,
+                    throttleStream,
+                    vm,
+                  }).run()
+                )
+              }),
+            error =>
+              runTask(taskStart, () => {
+                throw error
+              })
           )
+        }
         const { concurrency } = settings
         await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
       }

package/RemoteAdapter.js

@@ -10,7 +10,14 @@ const groupBy = require('lodash/groupBy.js')
 const pickBy = require('lodash/pickBy.js')
 const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
+const {
+  createVhdDirectoryFromStream,
+  createVhdStreamWithLength,
+  openVhd,
+  VhdAbstract,
+  VhdDirectory,
+  VhdSynthetic,
+} = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { decorateMethodsWith } = require('@vates/decorate-with')
 const { compose } = require('@vates/compose')
@@ -32,6 +39,7 @@ const { watchStreamSize } = require('./_watchStreamSize')
 // @todo : this import is marked extraneous , sould be fixed when lib is published
 const { mount } = require('@vates/fuse-vhd')
 const { asyncEach } = require('@vates/async-each')
+const { strictEqual } = require('assert')
 
 const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
@@ -209,8 +217,8 @@ class RemoteAdapter {
 
       const isVhdDirectory = vhd instanceof VhdDirectory
       return isVhdDirectory
-        ? this.#useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
-        : !this.#useVhdDirectory()
+        ? this.useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
+        : !this.useVhdDirectory()
     })
   }
 
@@ -321,12 +329,12 @@ class RemoteAdapter {
     return this._vhdDirectoryCompression
   }
 
-  #useVhdDirectory() {
+  useVhdDirectory() {
     return this.handler.useVhdDirectory()
   }
 
   #useAlias() {
-    return this.#useVhdDirectory()
+    return this.useVhdDirectory()
   }
 
   async *#getDiskLegacy(diskId) {
@@ -660,7 +668,7 @@ class RemoteAdapter {
 
   async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency, nbdClient } = {}) {
     const handler = this._handler
-    if (this.#useVhdDirectory()) {
+    if (this.useVhdDirectory()) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
       const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
         concurrency: writeBlockConcurrency,
@@ -674,17 +682,37 @@ class RemoteAdapter {
       await VhdAbstract.createAlias(handler, path, dataPath)
       return size
     } else {
-      return this.outputStream(path, input, { checksum, validator })
+      const inputWithSize = await createVhdStreamWithLength(input)
+      return this.outputStream(path, inputWithSize, { checksum, validator, expectedSize: inputWithSize.length })
     }
   }
 
-  async outputStream(path, input, { checksum = true, validator = noop } = {}) {
+  async outputStream(path, input, { checksum = true, validator = noop, expectedSize } = {}) {
     const container = watchStreamSize(input)
+
     await this._handler.outputStream(path, input, {
       checksum,
       dirMode: this._dirMode,
       async validator() {
         await input.task
+        if (expectedSize !== undefined) {
+          // check that we read all the stream
+          strictEqual(
+            container.size,
+            expectedSize,
+            `transferred size ${container.size}, expected file size : ${expectedSize}`
+          )
+        }
+        let size
+        try {
+          size = await this._handler.getSize(path)
+        } catch (err) {
+          // can fail is the remote is encrypted
+        }
+        if (size !== undefined) {
+          // check that everything is written to disk
+          strictEqual(size, container.size, `written size ${size}, transfered size : ${container.size}`)
+        }
         return validator.apply(this, arguments)
       },
     })

package/_VmBackup.js

@@ -55,6 +55,7 @@ class VmBackup {
     schedule,
     settings,
     srs,
+    throttleStream,
     vm,
   }) {
     if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
@@ -82,6 +83,7 @@ class VmBackup {
     this._healthCheckSr = healthCheckSr
     this._jobId = job.id
     this._jobSnapshots = undefined
+    this._throttleStream = throttleStream
     this._xapi = vm.$xapi
 
     // Base VM for the export
@@ -244,6 +246,7 @@ class VmBackup {
       fullVdisRequired,
     })
     const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
+    deltaExport.streams = mapValues(deltaExport.streams, this._throttleStream)
 
     const timestamp = Date.now()
 
@@ -285,10 +288,12 @@ class VmBackup {
 
   async _copyFull() {
     const { compression } = this.job
-    const stream = await this._xapi.VM_export(this.exportedVm.$ref, {
-      compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
-      useSnapshot: false,
-    })
+    const stream = this._throttleStream(
+      await this._xapi.VM_export(this.exportedVm.$ref, {
+        compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
+        useSnapshot: false,
+      })
+    )
     const sizeContainer = watchStreamSize(stream)
 
     const timestamp = Date.now()

package/_cleanVm.js

@@ -541,7 +541,8 @@ exports.cleanVm = async function cleanVm(
 
         // don't warn if the size has changed after a merge
         if (!merged && fileSystemSize !== size) {
-          logWarn('incorrect backup size in metadata', {
+          // FIXME: figure out why it occurs so often and, once fixed, log the real problems with `logWarn`
+          console.warn('cleanVm: incorrect backup size in metadata', {
             path: metadataPath,
             actual: size ?? 'none',
             expected: fileSystemSize,

package/_createStreamThrottle.js

@@ -0,0 +1,17 @@
+'use strict'
+
+const { pipeline } = require('node:stream')
+const { ThrottleGroup } = require('@kldzj/stream-throttle')
+const identity = require('lodash/identity.js')
+
+const noop = Function.prototype
+
+module.exports = function createStreamThrottle(rate) {
+  if (rate === 0) {
+    return identity
+  }
+  const group = new ThrottleGroup({ rate })
+  return function throttleStream(stream) {
+    return pipeline(stream, group.createThrottle(), noop)
+  }
+}

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.31.0",
+  "version": "0.33.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -17,6 +17,7 @@
     "test": "node--test"
   },
   "dependencies": {
+    "@kldzj/stream-throttle": "^1.1.1",
     "@vates/async-each": "^1.0.0",
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
@@ -26,7 +27,7 @@
     "@vates/nbd-client": "^1.0.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^3.3.2",
+    "@xen-orchestra/fs": "^3.3.4",
     "@xen-orchestra/log": "^0.6.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^5.0.1",
@@ -51,7 +52,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^1.6.1"
+    "@xen-orchestra/xapi": "^2.0.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/writers/DeltaBackupWriter.js

@@ -203,7 +203,7 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
           const vdiRef = vm.$xapi.getObject(vdi.uuid).$ref
 
           let nbdClient
-          if (this._backup.config.useNbd) {
+          if (this._backup.config.useNbd && adapter.useVhdDirectory()) {
             debug('useNbd is enabled', { vdi: id, path })
             // get nbd if possible
             try {

package/writers/DeltaReplicationWriter.js

@@ -45,6 +45,7 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
       data: {
         id: this._sr.uuid,
         isFull,
+        name_label: this._sr.name_label,
         type: 'SR',
       },
     })

package/writers/FullReplicationWriter.js

@@ -21,6 +21,7 @@ exports.FullReplicationWriter = class FullReplicationWriter extends MixinReplica
         name: 'export',
         data: {
           id: props.sr.uuid,
+          name_label: this._sr.name_label,
           type: 'SR',
 
           // necessary?