@xen-orchestra/backups 0.67.0 → 0.67.2

package/ImportVmBackup.mjs

@@ -11,6 +11,7 @@ import pickBy from 'lodash/pickBy.js'
 import { defer } from 'golike-defer'
 import { NegativeDisk } from '@xen-orchestra/disk-transform'
 import { openDiskChain } from './disks/openDiskChain.mjs'
+import { resetVmOtherConfig } from './_otherConfig.mjs'
 
 const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
 async function resolveUuid(xapi, cache, uuid, type) {
@@ -270,6 +271,7 @@ export class ImportVmBackup {
             `${metadata.vm.name_label} (${formatFilenameDate(metadata.timestamp)})`
           ),
           xapi.call('VM.set_name_description', vmRef, desc),
+          resetVmOtherConfig(xapi, vmRef),
         ])
 
         return {

package/RemoteAdapter.mjs

@@ -709,7 +709,7 @@ export class RemoteAdapter {
       })
     } else {
       const stream = await toVhdStream(disk)
-      await this.outputStream(path, stream, { validator })
+      await this.outputStream(path, stream, { validator, checksum: false })
       await validator(path)
     }
   }

package/_backupType.mjs

@@ -1,4 +1,5 @@
 export const isMetadataFile = filename => filename.endsWith('.json')
 export const isVhdFile = filename => filename.endsWith('.vhd')
+export const isVhdSumFile = filename => filename.endsWith('.vhd.checksum')
 export const isXvaFile = filename => filename.endsWith('.xva')
 export const isXvaSumFile = filename => filename.endsWith('.xva.checksum')

package/_cleanVm.mjs

@@ -4,7 +4,7 @@ import { asyncMap } from '@xen-orchestra/async-map'
 import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
 import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
 import { basename, dirname, resolve } from 'node:path'
-import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
+import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { mergeVhdChain } from 'vhd-lib/merge.js'
 
@@ -68,10 +68,12 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
 const noop = Function.prototype
 
 const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
+
 const listVhds = async (handler, vmDir, logWarn) => {
   const vhds = new Set()
   const aliases = {}
   const interruptedVhds = new Map()
+  const checksums = new Set()
 
   await asyncMap(
     await handler.list(`${vmDir}/vdis`, {
@@ -85,11 +87,15 @@ const listVhds = async (handler, vmDir, logWarn) => {
         }),
         async vdiDir => {
           const list = await handler.list(vdiDir, {
-            filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
+            filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file) || isVhdSumFile(file),
           })
           aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
 
           await asyncMap(list, async file => {
+            if (isVhdSumFile(file)) {
+              checksums.add(`${vdiDir}/${file}`)
+              return
+            }
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
               vhds.add(`${vdiDir}/${file}`)
@@ -111,7 +117,7 @@ const listVhds = async (handler, vmDir, logWarn) => {
       )
   )
 
-  return { vhds, interruptedVhds, aliases }
+  return { vhds, interruptedVhds, aliases, checksums }
 }
 
 export async function checkAliases(
@@ -228,7 +234,16 @@ export async function cleanVm(
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
-  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir, logWarn)
+  const { vhds, interruptedVhds, aliases, checksums } = await listVhds(handler, vmDir, logWarn)
+
+  // from 5.110 to 5.113 we computed checksum for vhd file
+  // but never used nor removed them
+  await asyncMap(checksums, async path => {
+    if (remove) {
+      logInfo('deleting checksum file ', { path })
+      return handler.unlink(path)
+    }
+  })
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {

package/_otherConfig.mjs

@@ -70,7 +70,8 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
 
 /**
  *
- * Reset the other_config field of a VM and its VDIs
+ * Reset the other_config field related to backups of a VM and its VDIs
+ *
  *
  * @param {Xapi} xapi
  * @param {String} vmRef

package/_runners/_vmRunners/_AbstractRemote.mjs

@@ -103,17 +103,10 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
     }
     if (transferList.length > 0) {
       const filteredTransferList = this._filterTransferList(transferList)
-      if (filteredTransferList.length > 0) {
-        return filteredTransferList
-      } else {
-        Task.info('This VM is excluded by the job filter')
-        return []
-      }
+      return { transferList: filteredTransferList, filtered: true }
     } else {
-      Task.info('No new data to upload for this VM')
+      return { transferList, filtered: false }
     }
-
-    return []
   }
 
   /**
@@ -127,12 +120,21 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
       Object.values(this.remoteAdapters).map(remoteAdapter => remoteAdapter.listVmBackups(this._vmUuid, vmPredicate))
     )
     const sourceBackupByJobId = groupBy(sourceBackups, 'jobId')
-    const transferByJobs = await Promise.all(
+    const transferObjectsList = await Promise.all(
       Object.values(sourceBackupByJobId).map(vmBackupsByJob =>
         this.#computeTransferListPerJob(vmBackupsByJob, remotesBackups)
       )
     )
-    return transferByJobs.flat(1)
+    const transferList = transferObjectsList.map(transferObject => transferObject.transferList).flat(1)
+    if (transferList.length === 0 && Object.values(sourceBackupByJobId).length > 0) {
+      if (transferObjectsList.some(transferObject => transferObject.filtered)) {
+        // Happens if any job transfer list wasn't empty before filter and is empty after filter.
+        Task.info('This VM is excluded by the job filter')
+      } else {
+        Task.info('No new data to upload for this VM')
+      }
+    }
+    return transferList
   }
 
   async run($defer) {

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -1,5 +1,6 @@
 import assert from 'node:assert'
 import groupBy from 'lodash/groupBy.js'
+import { createLogger } from '@xen-orchestra/log'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { decorateMethodsWith } from '@vates/decorate-with'
@@ -17,6 +18,8 @@ import {
   setVmOtherConfig,
 } from '../../_otherConfig.mjs'
 
+const { warn } = createLogger('xo:backups:AbstractXapi')
+
 export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
   constructor({
     config,
@@ -255,28 +258,47 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
           return
         }
         const vdis = snapshotPerDatetime[datetime]
-        let vmRef
+        let vm
         // if there is an attached VM => destroy the VM (Non CBT backups)
         for (const vdi of vdis) {
           const vbds = vdi.$VBDs.filter(({ $VM }) => $VM.is_control_domain === false)
           if (vbds.length > 0) {
             // only one VM linked to this vdi
             // this will throw error for VDI still attached to control domain
+            // since we won't be able to remove an attached VDI
             assert.strictEqual(vbds.length, 1, 'VDI must be free or attached to exactly one VM')
-            const vm = vbds[0].$VM
-            assert.strictEqual(vm.is_a_snapshot, true, `VM must be a snapshot`) // don't delete a VM (especially a control domain)
-
-            const vmRefVdi = vm.$ref
-            // same vm than other vdi of the same batch
-            assert.ok(
-              vmRef === undefined || vmRef === vmRefVdi,
-              '_removeUnusedSnapshots don t handle vdi related to multiple VMs '
-            )
-            vmRef = vmRefVdi
+            const vdiVm = vbds[0].$VM
+            if (!vdiVm.is_a_snapshot) {
+              // don't delete a VM (especially a control domain)
+              warn(
+                `VM ${vdiVm.uuid} (${vdiVm.name_label}) linked to VDI ${vdi.uuid} (${vdi.name_label}) should be a snapshot`
+              )
+              throw new Error(`VM must be a snapshot`)
+            }
+            if (vm !== undefined && vm.$ref !== vdiVm.$ref) {
+              // this VDI is attached to another VM than the other vdi of
+              // this batch
+              // in doubt, do not delete anything
+              warn("_removeUnusedSnapshots don't handle vdi related to multiple VMs ", {
+                vm1: {
+                  label: vm.name_label,
+                  id: vm.$id,
+                },
+                vm2: {
+                  label: vdiVm.name_label,
+                  id: vdiVm.$id,
+                },
+                vdis: vdis.map(({ name_label, $id }) => ({ name_label, $id })),
+              })
+              throw new Error(
+                `_removeUnusedSnapshots don't handle vdi related to multiple VMs ${vm.name_label} and ${vdiVm.name_label}`
+              )
+            }
+            vm = vdiVm
           }
         }
-        if (vmRef !== undefined) {
-          return xapi.VM_destroy(vmRef)
+        if (vm?.$ref !== undefined) {
+          return xapi.VM_destroy(vm.$ref)
         } else {
           return asyncMap(
             vdis.map(async ({ $ref }) => {

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.67.0",
+  "version": "0.67.2",
   "engines": {
     "node": ">=14.18"
   },
@@ -26,13 +26,13 @@
     "@vates/disposable": "^0.1.6",
     "@vates/fuse-vhd": "^2.1.2",
     "@vates/generator-toolbox": "^1.1.0",
-    "@vates/nbd-client": "^3.2.1",
+    "@vates/nbd-client": "^3.2.2",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/disk-transform": "^1.2.0",
-    "@xen-orchestra/fs": "^4.6.4",
+    "@xen-orchestra/disk-transform": "^1.2.1",
+    "@xen-orchestra/fs": "^4.6.5",
     "@xen-orchestra/log": "^0.7.1",
-    "@xen-orchestra/qcow2": "^1.1.0",
+    "@xen-orchestra/qcow2": "^1.1.1",
     "@xen-orchestra/template": "^0.1.0",
     "app-conf": "^3.0.0",
     "compare-versions": "^6.0.0",
@@ -51,8 +51,8 @@
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^4.14.4",
-    "xen-api": "^4.7.5",
+    "vhd-lib": "^4.14.6",
+    "xen-api": "^4.7.6",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -62,7 +62,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^8.6.1"
+    "@xen-orchestra/xapi": "^8.6.3"
   },
   "license": "AGPL-3.0-or-later",
   "author": {