@xen-orchestra/backups 0.64.2 → 0.65.0

package/_incrementalVm.mjs

@@ -94,11 +94,13 @@ export async function exportIncrementalVm(
     }
   })
 
+  // Get a fresh list of VM's VTPM to avoid `vm.VTPMs: [undefined]`
+  const vmVtpms = await vm.$xapi.getField('VM', vm.$ref, 'VTPMs')
   const vtpms = await Promise.all(
-    vm.$VTPMs.map(async vtpm => {
+    vmVtpms.map(async vtpmRef => {
       let content
       try {
-        content = await vm.$xapi.call('VTPM.get_contents', vtpm.$ref)
+        content = await vm.$xapi.call('VTPM.get_contents', vtpmRef)
       } catch (err) {
         console.error(err)
       }

package/_runners/VmsRemote.mjs

@@ -9,6 +9,7 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { FullRemote } from './_vmRunners/FullRemote.mjs'
 import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
 import { Throttle } from '@vates/generator-toolbox'
+import createStreamThrottle from './_createStreamThrottle.mjs'
 
 const noop = Function.prototype
 
@@ -41,7 +42,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
     const schedule = this._schedule
     const settings = this._settings
 
-    const throttleGenerator = new Throttle()
+    const throttleGenerator = new Throttle(settings.maxExportRate)
+    const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
 
@@ -89,7 +91,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             schedule,
             settings: vmSettings,
             sourceRemoteAdapter,
-            throttleGenerator,
+            throttleGenerator, // for incrementals
+            throttleStream, // for full
             vmUuid,
           }
           let vmBackup

package/_runners/VmsXapi.mjs

@@ -10,6 +10,7 @@ import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
 import { FullXapi } from './_vmRunners/FullXapi.mjs'
 import { Throttle } from '@vates/generator-toolbox'
+import createStreamThrottle from './_createStreamThrottle.mjs'
 
 const noop = Function.prototype
 
@@ -55,7 +56,8 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
     const schedule = this._schedule
     const settings = this._settings
 
-    const throttleGenerator = new Throttle()
+    const throttleGenerator = new Throttle(settings.maxExportRate)
+    const throttleStream = createStreamThrottle(settings.maxExportRate)
 
     const config = this._config
 
@@ -148,6 +150,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
                       settings: vmSettings,
                       srs,
                       throttleGenerator,
+                      throttleStream,
                       vm,
                     }
 

package/_runners/_vmRunners/FullRemote.mjs

@@ -16,7 +16,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
     const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
 
     for (const metadata of transferList) {
-      const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
+      const stream = this._throttleStream(await this._sourceRemoteAdapter.readFullVmBackup(metadata))
       const sizeContainer = watchStreamSize(stream)
 
       // @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)

package/_runners/_vmRunners/FullXapi.mjs

@@ -49,16 +49,14 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
     const { compression } = this.job
     const vm = this._vm
     const exportedVm = this._exportedVm
-    // @todo put back throttle for full backup/Replication
-    const stream =
-      /* this._throttleStream( */
+    const stream = this._throttleStream(
       (
         await this._xapi.VM_export(exportedVm.$ref, {
           compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
           useSnapshot: false,
         })
       ).body
-    /* ) */
+    )
 
     const vdis = await exportedVm.$getDisks()
     let maxStreamLength = 1024 * 1024 // Ovf file and tar headers are a few KB, let's stay safe

package/_runners/_vmRunners/IncrementalRemote.mjs

@@ -10,7 +10,7 @@ import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs
 import { Disposable } from 'promise-toolbox'
 import { openVhd } from 'vhd-lib'
 import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
-import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
+import { SynchronizedDisk, ThrottledDisk } from '@xen-orchestra/disk-transform'
 
 const { warn } = createLogger('xo:backups:Incrementalremote')
 class IncrementalRemoteVmBackupRunner extends AbstractRemote {
@@ -71,8 +71,9 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
       const isVhdDifferencing = {}
 
       for (const key in incrementalExport.disks) {
-        const disk = incrementalExport.disks[key]
+        let disk = incrementalExport.disks[key]
         isVhdDifferencing[key] = disk.isDifferencing()
+        disk = new ThrottledDisk(disk, this._throttleGenerator)
         incrementalExport.disks[key] = new SynchronizedDisk(disk)
       }
 

package/_runners/_vmRunners/IncrementalXapi.mjs

@@ -14,7 +14,7 @@ import {
   setVmDeltaChainLength,
   markExportSuccessfull,
 } from '../../_otherConfig.mjs'
-import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
+import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
 
 const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
 
@@ -43,13 +43,14 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
     const isVhdDifferencing = {}
     let useNbd = false
     for (const key in deltaExport.disks) {
-      const disk = deltaExport.disks[key]
+      let disk = deltaExport.disks[key]
       isVhdDifferencing[key] = disk.isDifferencing()
       if (!isFull && !isVhdDifferencing[key] && key !== exportedVm.$suspend_VDI?.$ref) {
         Task.warning('Backup fell back to a full')
       }
-      deltaExport.disks[key] = new SynchronizedDisk(disk)
       useNbd = useNbd || disk.useNbd()
+      disk = new ThrottledDisk(disk, this._throttleGenerator)
+      deltaExport.disks[key] = new SynchronizedDisk(disk)
     }
     if (useNbd) {
       Task.info('Transfer data using NBD')

package/_runners/_vmRunners/_AbstractRemote.mjs

@@ -24,6 +24,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
     settings,
     sourceRemoteAdapter,
     throttleGenerator,
+    throttleStream,
     vmUuid,
   }) {
     super()
@@ -37,6 +38,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
     this._healthCheckSr = healthCheckSr
     this._sourceRemoteAdapter = sourceRemoteAdapter
     this._throttleGenerator = throttleGenerator
+    this._throttleStream = throttleStream
     this._vmUuid = vmUuid
 
     const allSettings = job.settings

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -29,6 +29,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     settings,
     srs,
     throttleGenerator,
+    throttleStream,
     vm,
   }) {
     super()
@@ -63,6 +64,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     this._jobId = job.id
     this._jobSnapshotVdis = undefined
     this._throttleGenerator = throttleGenerator
+    this._throttleStream = throttleStream
     this._xapi = vm.$xapi
 
     // Base VM for the export

package/_runners/_writers/IncrementalXapiWriter.mjs

@@ -10,6 +10,7 @@ import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
 import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
 import { listReplicatedVms } from './_listReplicatedVms.mjs'
 import { COPY_OF, setVmOtherConfig, BASE_DELTA_VDI } from '../../_otherConfig.mjs'
+import assert from 'node:assert'
 
 export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
   async checkBaseVdis(baseUuidToSrcVdi) {
@@ -25,7 +26,11 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
     const replicatedVdis = sr.$VDIs
       .filter(vdi => {
         // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
-        return baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
+        return (
+          vdi?.managed &&
+          !vdi?.is_a_snapshot /* only look for real vdi */ &&
+          baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
+        )
       })
       .map(({ other_config }) => other_config?.[COPY_OF])
       .filter(_ => !!_)
@@ -108,17 +113,21 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
 
     const replicatedVdis = sr.$VDIs.filter(vdi => {
       // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
-      return sourceVdiUuids.includes(vdi?.other_config[COPY_OF])
+      return vdi?.managed && !vdi?.is_a_snapshot && sourceVdiUuids.includes(vdi?.other_config[COPY_OF])
     })
 
     Object.values(backup.vdis).forEach(vdi => {
       vdi.other_config[COPY_OF] = vdi.uuid
       if (sourceVdiUuids.length > 0) {
-        const baseReplicatedTo = replicatedVdis.find(
+        const baseReplicatedTo = replicatedVdis.filter(
           replicatedVdi => replicatedVdi.other_config[COPY_OF] === vdi.other_config[BASE_DELTA_VDI]
         )
+        assert.ok(
+          baseReplicatedTo.length <= 1,
+          `Target of a replication must be unique, got ${baseReplicatedTo.length} candidates`
+        )
         // baseReplicatedTo can be undefined if a new disk is added and other are already replicated
-        vdi.baseVdi = baseReplicatedTo
+        vdi.baseVdi = baseReplicatedTo[0]
       } else {
         // first replication of this disk
         vdi.baseVdi = undefined

package/formatVmBackups.mjs

@@ -18,6 +18,8 @@ function formatVmBackup(backup) {
     }
   }
   return {
+    type: 'xo-vm-backup',
+    backupRepository: backup.backupRepositoryId,
     disks:
       backup.vhds === undefined
         ? []
@@ -38,6 +40,7 @@ function formatVmBackup(backup) {
     size: backup.size,
     timestamp: backup.timestamp,
     vm: {
+      uuid: backup.vm.uuid,
       name_description: backup.vm.name_description,
       name_label: backup.vm.name_label,
     },
@@ -48,7 +51,12 @@ function formatVmBackup(backup) {
   }
 }
 
-// format all backups as returned by RemoteAdapter#listAllVmBackups()
-export function formatVmBackups(backupsByVM) {
-  return mapValues(backupsByVM, backups => backups.map(formatVmBackup))
+/**
+ * format all backups as returned by RemoteAdapter#listAllVmBackups()
+ * @param {Record<string, object[]>} backupsByVM
+ * @param {string} backupRepositoryId
+ * @returns {Record<string, object[]>}
+ */
+export function formatVmBackups(backupsByVM, backupRepositoryId) {
+  return mapValues(backupsByVM, backups => backups.map(backup => formatVmBackup({ ...backup, backupRepositoryId })))
 }

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.64.2",
+  "version": "0.65.0",
   "engines": {
     "node": ">=14.18"
   },
@@ -19,18 +19,18 @@
   "dependencies": {
     "@iarna/toml": "^2.2.5",
     "@kldzj/stream-throttle": "^1.1.1",
-    "@vates/async-each": "^1.0.0",
+    "@vates/async-each": "^1.0.1",
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.1.0",
     "@vates/disposable": "^0.1.6",
     "@vates/fuse-vhd": "^2.1.2",
-    "@vates/generator-toolbox": "^1.0.4",
+    "@vates/generator-toolbox": "^1.1.0",
     "@vates/nbd-client": "^3.2.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/disk-transform": "^1.1.0",
-    "@xen-orchestra/fs": "^4.6.3",
+    "@xen-orchestra/disk-transform": "^1.2.0",
+    "@xen-orchestra/fs": "^4.6.4",
     "@xen-orchestra/log": "^0.7.1",
     "@xen-orchestra/qcow2": "^1.0.0",
     "@xen-orchestra/template": "^0.1.0",
@@ -51,18 +51,18 @@
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^4.14.1",
+    "vhd-lib": "^4.14.2",
     "xen-api": "^4.7.4",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
     "fs-extra": "^11.1.0",
-    "rimraf": "^5.0.1",
+    "rimraf": "^6.0.1",
     "sinon": "^18.0.0",
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^8.4.1"
+    "@xen-orchestra/xapi": "^8.4.2"
   },
   "license": "AGPL-3.0-or-later",
   "author": {