@xen-orchestra/backups 0.52.2 → 0.53.0

package/_runners/_vmRunners/FullRemote.mjs

@@ -2,39 +2,39 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
 import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
 import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
 import { watchStreamSize } from '../../_watchStreamSize.mjs'
-import { Task } from '../../Task.mjs'
 
 export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
     return FullRemoteWriter
   }
+
+  _filterTransferList(transferList) {
+    return transferList.filter(this._filterPredicate)
+  }
+
   async _run() {
     const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
 
-    if (transferList.length > 0) {
-      for (const metadata of transferList) {
-        const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
-        const sizeContainer = watchStreamSize(stream)
+    for (const metadata of transferList) {
+      const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
+      const sizeContainer = watchStreamSize(stream)
 
-        // @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
-        await this._callWriters(
-          writer =>
-            writer.run({
-              stream: forkStreamUnpipe(stream),
-              // stream will be forked and transformed, it's not safe to attach additionnal properties to it
-              streamLength: stream.length,
-              timestamp: metadata.timestamp,
-              vm: metadata.vm,
-              vmSnapshot: metadata.vmSnapshot,
-              sizeContainer,
-            }),
-          'writer.run()'
-        )
-        // for healthcheck
-        this._tags = metadata.vm.tags
-      }
-    } else {
-      Task.info('No new data to upload for this VM')
+      // @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
+      await this._callWriters(
+        writer =>
+          writer.run({
+            stream: forkStreamUnpipe(stream),
+            // stream will be forked and transformed, it's not safe to attach additionnal properties to it
+            streamLength: stream.length,
+            timestamp: metadata.timestamp,
+            vm: metadata.vm,
+            vmSnapshot: metadata.vmSnapshot,
+            sizeContainer,
+          }),
+        'writer.run()'
+      )
+      // for healthcheck
+      this._tags = metadata.vm.tags
     }
   }
 }

package/_runners/_vmRunners/IncrementalRemote.mjs

@@ -7,7 +7,6 @@ import mapValues from 'lodash/mapValues.js'
 import { AbstractRemote } from './_AbstractRemote.mjs'
 import { forkDeltaExport } from './_forkDeltaExport.mjs'
 import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
-import { Task } from '../../Task.mjs'
 import { Disposable } from 'promise-toolbox'
 import { openVhd } from 'vhd-lib'
 import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
@@ -16,6 +15,16 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
     return IncrementalRemoteWriter
   }
+
+  // we'll transfer the full list if at least one backup should be transfered
+  // to ensure we don't cut the delta chain
+  _filterTransferList(transferList) {
+    if (transferList.some(vmBackupMetadata => this._filterPredicate(vmBackupMetadata))) {
+      return transferList
+    }
+    return []
+  }
+
   async _selectBaseVm(metadata) {
     // for each disk , get the parent
     const baseUuidToSrcVdi = new Map()
@@ -53,50 +62,46 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
   async _run() {
     const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
 
-    if (transferList.length > 0) {
-      for (const metadata of transferList) {
-        assert.strictEqual(metadata.mode, 'delta')
-        await this._selectBaseVm(metadata)
-        await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
-        const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
-          useChain: false,
-        })
+    for (const metadata of transferList) {
+      assert.strictEqual(metadata.mode, 'delta')
+      await this._selectBaseVm(metadata)
+      await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
+      const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
+        useChain: false,
+      })
 
-        const isVhdDifferencing = {}
+      const isVhdDifferencing = {}
 
-        await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
-          isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
-        })
+      await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
+        isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
+      })
 
-        incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
-        await this._callWriters(
-          writer =>
-            writer.transfer({
-              deltaExport: forkDeltaExport(incrementalExport),
-              isVhdDifferencing,
-              timestamp: metadata.timestamp,
-              vm: metadata.vm,
-              vmSnapshot: metadata.vmSnapshot,
-            }),
-          'writer.transfer()'
-        )
-        // this will update parent name with the needed alias
-        await this._callWriters(
-          writer =>
-            writer.updateUuidAndChain({
-              isVhdDifferencing,
-              timestamp: metadata.timestamp,
-              vdis: incrementalExport.vdis,
-            }),
-          'writer.updateUuidAndChain()'
-        )
+      incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
+      await this._callWriters(
+        writer =>
+          writer.transfer({
+            deltaExport: forkDeltaExport(incrementalExport),
+            isVhdDifferencing,
+            timestamp: metadata.timestamp,
+            vm: metadata.vm,
+            vmSnapshot: metadata.vmSnapshot,
+          }),
+        'writer.transfer()'
+      )
+      // this will update parent name with the needed alias
+      await this._callWriters(
+        writer =>
+          writer.updateUuidAndChain({
+            isVhdDifferencing,
+            timestamp: metadata.timestamp,
+            vdis: incrementalExport.vdis,
+          }),
+        'writer.updateUuidAndChain()'
+      )
 
-        await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
-        // for healthcheck
-        this._tags = metadata.vm.tags
-      }
-    } else {
-      Task.info('No new data to upload for this VM')
+      await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
+      // for healthcheck
+      this._tags = metadata.vm.tags
     }
   }
 }

package/_runners/_vmRunners/_AbstractRemote.mjs

@@ -1,14 +1,18 @@
-import { asyncEach } from '@vates/async-each'
+import groupBy from 'lodash/groupBy.js'
+
 import { decorateMethodsWith } from '@vates/decorate-with'
 import { defer } from 'golike-defer'
 import { Disposable } from 'promise-toolbox'
+import { createPredicate } from 'value-matcher'
 
 import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
 
 import { Abstract } from './_Abstract.mjs'
 import { extractIdsFromSimplePattern } from '../../extractIdsFromSimplePattern.mjs'
+import { Task } from '../../Task.mjs'
 
 export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstract {
+  _filterPredicate
   constructor({
     config,
     job,
@@ -57,39 +61,74 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
         })
       )
     })
+    const { filter } = job
+    if (filter === undefined) {
+      this._filterPredicate = () => true
+    } else {
+      this._filterPredicate = createPredicate(filter)
+    }
   }
 
-  async _computeTransferList(predicate) {
-    const vmBackups = await this._sourceRemoteAdapter.listVmBackups(this._vmUuid, predicate)
+  async #computeTransferListPerJob(sourceBackups, remotesBackups) {
     const localMetada = new Map()
-    Object.values(vmBackups).forEach(metadata => {
+    sourceBackups.forEach(metadata => {
       const timestamp = metadata.timestamp
       localMetada.set(timestamp, metadata)
     })
-    const nbRemotes = Object.keys(this.remoteAdapters).length
+    const nbRemotes = remotesBackups.length
     const remoteMetadatas = {}
-    await asyncEach(Object.values(this.remoteAdapters), async remoteAdapter => {
-      const remoteMetadata = await remoteAdapter.listVmBackups(this._vmUuid, predicate)
-      remoteMetadata.forEach(metadata => {
+    remotesBackups.forEach(async remoteBackups => {
+      remoteBackups.forEach(metadata => {
         const timestamp = metadata.timestamp
         remoteMetadatas[timestamp] = (remoteMetadatas[timestamp] ?? 0) + 1
       })
     })
 
-    let chain = []
+    let transferList = []
     const timestamps = [...localMetada.keys()]
     timestamps.sort()
     for (const timestamp of timestamps) {
       if (remoteMetadatas[timestamp] !== nbRemotes) {
         // this backup is not present in all the remote
         // should be retransfered if not found later
-        chain.push(localMetada.get(timestamp))
+        transferList.push(localMetada.get(timestamp))
       } else {
         // backup is present in local and remote : the chain has already been transferred
-        chain = []
+        transferList = []
+      }
+    }
+    if (transferList.length > 0) {
+      const filteredTransferList = this._filterTransferList(transferList)
+      if (filteredTransferList.length > 0) {
+        return filteredTransferList
+      } else {
+        Task.info('This VM is excluded by the job filter')
+        return []
       }
+    } else {
+      Task.info('No new data to upload for this VM')
     }
-    return chain
+
+    return []
+  }
+
+  /**
+   *
+   * @param {*} vmPredicate a callback checking if backup is eligible for transfer. This filter MUST NOT cut delta chains
+   * @returns
+   */
+  async _computeTransferList(vmPredicate) {
+    const sourceBackups = Object.values(await this._sourceRemoteAdapter.listVmBackups(this._vmUuid, vmPredicate))
+    const remotesBackups = await Promise.all(
+      Object.values(this.remoteAdapters).map(remoteAdapter => remoteAdapter.listVmBackups(this._vmUuid, vmPredicate))
+    )
+    const sourceBackupByJobId = groupBy(sourceBackups, 'jobId')
+    const transferByJobs = await Promise.all(
+      Object.values(sourceBackupByJobId).map(vmBackupsByJob =>
+        this.#computeTransferListPerJob(vmBackupsByJob, remotesBackups)
+      )
+    )
+    return transferByJobs.flat(1)
   }
 
   async run($defer) {

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -214,8 +214,12 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     const allSettings = this.job.settings
     const baseSettings = this._baseSettings
 
-    const snapshotsPerSchedule = groupBy(this._jobSnapshotVdis, _ => _.other_config[SCHEDULE_ID])
     const xapi = this._xapi
+    // ensure all the event has been processed by the xapi
+    await xapi.barrier()
+    // ensure cached object are up to date
+    this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
+    const snapshotsPerSchedule = groupBy(this._jobSnapshotVdis, _ => _.other_config[SCHEDULE_ID])
     await asyncMap(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
       const snapshotPerDatetime = groupBy(snapshots, _ => _.other_config[DATETIME])
 
@@ -235,13 +239,12 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
         let vmRef
         // if there is an attached VM => destroy the VM (Non CBT backups)
         for (const vdi of vdis) {
-          if (vdi.$VBDs.length > 0) {
-            const vbds = vdi.$VBDs
+          const vbds = vdi.$VBDs.filter(({ $VM }) => $VM.is_control_domain === false)
+          if (vbds.length > 0) {
             // only one VM linked to this vdi
             // this will throw error for VDI still attached to control domain
             assert.strictEqual(vbds.length, 1, 'VDI must be free or attached to exactly one VM')
             const vm = vbds[0].$VM
-            assert.strictEqual(vm.is_control_domain, false, `Disk is still attached to DOM0 VM`) // don't delete a VM (especially a control domain)
             assert.strictEqual(vm.is_a_snapshot, true, `VM must be a snapshot`) // don't delete a VM (especially a control domain)
 
             const vmRefVdi = vm.$ref
@@ -335,15 +338,22 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       const reason = 'VM migration is blocked during backup'
       await vm.update_blocked_operations({ pool_migrate: reason, migrate_send: reason })
 
-      $defer(() =>
+      $defer(async () => {
         // delete the entries if they did not exist previously or if they were
         // equal to reason (which happen if a previous backup was interrupted
         // before resetting them)
-        vm.update_blocked_operations({
+        await vm.update_blocked_operations({
           migrate_send: migrate_send === undefined || migrate_send === reason ? null : migrate_send,
           pool_migrate: pool_migrate === undefined || pool_migrate === reason ? null : pool_migrate,
         })
-      )
+
+        // 2024-08-19 - Work-around a XAPI bug where allowed_operations are not properly computed when blocked_operations is updated
+        //
+        // this is a problem because some clients (e.g. XenCenter) use this field to allow operations.
+        //
+        // internal source: https://team.vates.fr/vates/pl/mjmxnce9qfdx587r3qpe4z91ho
+        await vm.$call('update_allowed_operations')
+      })
     }
 
     await this._fetchJobSnapshots()

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.52.2",
+  "version": "0.53.0",
   "engines": {
     "node": ">=14.18"
   },
@@ -46,6 +46,7 @@
     "proper-lockfile": "^4.1.2",
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
+    "value-matcher": "^0.2.0",
     "vhd-lib": "^4.11.0",
     "xen-api": "^4.2.0",
     "yazl": "^2.5.1"
@@ -58,7 +59,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^7.2.1"
+    "@xen-orchestra/xapi": "^7.3.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {