@xen-orchestra/backups 0.72.0 → 0.72.1

package/RemoteAdapter.mjs

@@ -702,7 +702,7 @@ export class RemoteAdapter {
     const handler = this._handler
 
     if (this.useVhdDirectory()) {
-      await writeToVhdDirectory({
+      return await writeToVhdDirectory({
         disk,
         target: {
           handler,
@@ -714,8 +714,9 @@ export class RemoteAdapter {
       })
     } else {
       const stream = await toVhdStream(disk)
-      await this.outputStream(path, stream, { validator, checksum: false })
+      const size = await this.outputStream(path, stream, { validator, checksum: false })
       await validator(path)
+      return size
     }
   }
 

package/_cleanVm.mjs

@@ -586,18 +586,13 @@ export async function cleanVm(
       return
     }
 
-    // systematically update size and differentials after a merge
-
-    // @todo : after 2024-04-01 remove the fixmetadata options since the size computation is fixed
-    if (mergedSize || (fixMetadata && fileSystemSize !== size)) {
-      metadata.size = mergedSize ?? fileSystemSize ?? size
-
-      if (mergedSize) {
-        // all disks are now key disk
-        metadata.isVhdDifferencing = {}
-        for (const id of Object.keys(metadata.vdis ?? {})) {
-          metadata.isVhdDifferencing[id] = false
-        }
+    // Rewrite metadata when a merge changed the backup layout.
+    if (mergedSize) {
+      metadata.size = mergedSize
+      // all disks are now key disk
+      metadata.isVhdDifferencing = {}
+      for (const id of Object.keys(metadata.vdis ?? {})) {
+        metadata.isVhdDifferencing[id] = false
       }
       mustRegenerateCache = true
       try {

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -338,6 +338,20 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     }
 
     this._jobSnapshotVdis = Object.values(vdiCandidates)
+
+    // For VMs with no disks, retention must be tracked directly on VM snapshots
+    // since there are no VDIs to anchor the discovery.
+    if (vdiUuids.length === 0) {
+      this._disklessJobSnapshotVms = this._vm.$snapshots.filter(Boolean).filter(
+        ({ other_config, $snapshot_of }) =>
+          $snapshot_of !== undefined &&
+          other_config[JOB_ID] === jobId &&
+          other_config[VM_UUID] === this._vm.uuid &&
+          other_config[COPY_OF] === undefined
+      )
+    } else {
+      this._disklessJobSnapshotVms = []
+    }
   }
 
   async _removeUnusedSnapshots() {
@@ -349,9 +363,10 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     await xapi.barrier()
     // ensure cached object are up to date
     this._jobSnapshotVdis = this._jobSnapshotVdis.map(vdi => xapi.getObject(vdi.$ref))
+    const disklessVmSnapshots = this._disklessJobSnapshotVms.map(vm => xapi.getObject(vm.$ref))
 
-    // get the datetime of the most recent snapshot
-    const lastSnapshotDateTime = this._jobSnapshotVdis
+    // get the datetime of the most recent snapshot across both VDI and diskless VM snapshots
+    const lastSnapshotDateTime = [...this._jobSnapshotVdis, ...disklessVmSnapshots]
       .map(({ other_config }) => other_config[DATETIME])
       .sort()
       .pop()
@@ -428,6 +443,28 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       })
     })
 
+    // Retention for VMs with no disks: VM snapshots are not reachable via VDIs
+    if (disklessVmSnapshots.length > 0) {
+      const snapshotsPerSchedule = groupBy(disklessVmSnapshots, _ => _.other_config[SCHEDULE_ID])
+      await asyncEach(Object.entries(snapshotsPerSchedule), async ([scheduleId, snapshots]) => {
+        // we only have one snapshot per date time since it's at the VM level 
+        const snapshotPerDatetime = Object.fromEntries(snapshots.map(s => [s.other_config[DATETIME], s.$ref]))
+        const datetimes = Object.keys(snapshotPerDatetime).sort()
+        const settings = {
+          ...baseSettings,
+          ...allSettings[scheduleId],
+          ...allSettings[this._vm.uuid],
+        }
+        const retention = settings.snapshotRetention ?? 0
+        await asyncEach(getOldEntries(retention, datetimes), async datetime => {
+          if (this.job.mode === 'delta' && datetime === lastSnapshotDateTime) {
+            return
+          }
+          await xapi.VM_destroy(snapshotPerDatetime[datetime])
+        })
+      })
+    }
+
     // list and remove the snapshot were the jobs failed between
     // makesnapshot and update_other_config
     const snapshots = this._vm.$snapshots.filter(_ => !!_).filter(({ name_label }) => name_label === TEMP_SNAPSHOT_NAME)

package/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -229,14 +229,13 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
         Object.entries(deltaExport.disks),
         async ([diskRef, disk]) => {
           const path = `${this._vmBackupDir}/${vhds[diskRef]}`
-          await adapter.writeVhd(path, disk, {
+          size += await adapter.writeVhd(path, disk, {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chains
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
             writeBlockConcurrency: this._config.writeBlockConcurrency,
           })
-          size = size + disk.getNbGeneratedBlock() * disk.getBlockSize()
         },
         {
           concurrency: settings.diskPerVmConcurrency,

package/_runners/_writers/IncrementalXapiWriter.mjs

@@ -57,71 +57,14 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
     debug('checkBaseVdis, got snapshot candidates,', snapshotCandidates.length)
 
     if (snapshotCandidates.length > 0) {
-      // New snapshot-based flow (6.3+): verify no data was written between
-      // the target snapshot and its active VDI.
-      let targetVmRef
-      let canChainToTargetVm = true
-      await asyncEach(
-        snapshotCandidates,
-        async snapshot => {
-          let diffDisk
-          let activeVdi
-          try {
-            activeVdi = sr.$xapi.getObject(snapshot.$snapshot_of)
-            const userVbds = activeVdi.$VBDs?.filter(vbd => vbd.$VM && !vbd.$VM.is_control_domain) ?? []
-            if (userVbds.length !== 1) {
-              debug('checkBaseVdis, share vbd ', { ref: snapshot.$ref, userVbds })
-              // shared vdi ignore
-              return
-            }
-            const vm = userVbds[0].$VM
-            if (!('start' in vm.blocked_operations)) {
-              debug('checkBaseVdis, vm not blocked', { vmRef: vm.$ref })
-              // vm start unlocked
-              // not really an issue since we have check the delta
-              // but it indicates the users played with the blocked operations
-              return
-            }
-            diffDisk = new XapiDiskSource({
-              xapi: sr.$xapi,
-              vdiRef: activeVdi.$ref,
-              baseRef: snapshot.$ref,
-              onlyListChangedBlocks: true,
-            })
-            await diffDisk.init()
-            if (diffDisk.getBlockIndexes().length === 0) {
-              const sourceUuid = snapshot.other_config?.[COPY_OF]
-              if (sourceUuid) {
-                this.#baseVdisBySourceUuid.set(sourceUuid, activeVdi)
-              }
-              // Track the target VM (the replicated VM to update on the next transfer).
-              targetVmRef = vm.$ref
-            } else {
-              // not empty, we will create a new VM
-              canChainToTargetVm = false
-              debug('checkBaseVdis, data between snapshot and active disk', {
-                vdiRef: snapshot.$ref,
-                nbBlocks: diffDisk.getBlockIndexes().length,
-              })
-            }
-          } catch (error) {
-            debug('checkBaseVdis, skipping snapshot', { ref: snapshot.$ref, error })
-            return
-          } finally {
-            await diffDisk?.close().catch(error => debug('checkBaseVdis, error closing', error))
-            await sr.$xapi.VDI_disconnectFromControlDomain(snapshot.$ref)
-            if (activeVdi !== undefined) {
-              await sr.$xapi.VDI_disconnectFromControlDomain(activeVdi.$ref)
-            }
-          }
-        },
-        {
-          concurrency: 4,
-        }
-      )
-
-      if (canChainToTargetVm && targetVmRef !== undefined) {
-        debug('checkBaseVdis,got a valid vm target', targetVmRef)
+      // reset before searching for candidates
+      this.#baseVdisBySourceUuid = new Map()
+      this._targetVmRef = undefined
+      const { baseVdisBySourceUuid, targetVmRef } = await this.#validateSnapshotCandidates(snapshotCandidates)
+      for (const [sourceUuid, vdi] of baseVdisBySourceUuid) {
+        this.#baseVdisBySourceUuid.set(sourceUuid, vdi)
+      }
+      if (targetVmRef !== undefined) {
         this._targetVmRef = targetVmRef
       }
     } else {
@@ -145,6 +88,93 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
       }
     }
   }
+  /**
+   * 6.3+ snapshot-based validation: for each snapshot candidate, check whether
+   * the active VDI has diverged from the snapshot. Returns a baseVdisBySourceUuid
+   * map and, when all disks are clean, the targetVmRef to reuse.
+   */
+  async #validateSnapshotCandidates(snapshotCandidates) {
+    const sr = this._sr
+    const baseVdisBySourceUuid = new Map()
+    let targetVmRef
+    let canChainToTargetVm = true
+
+    await asyncEach(
+      snapshotCandidates,
+      async snapshot => {
+        let diffDisk
+        let activeVdi
+        try {
+          activeVdi = sr.$xapi.getObject(snapshot.$snapshot_of)
+          const userVbds = activeVdi.$VBDs?.filter(vbd => vbd.$VM && !vbd.$VM.is_control_domain) ?? []
+          if (userVbds.length !== 1) {
+            debug('checkBaseVdis, shared vbd ', { ref: snapshot.$ref, userVbds })
+            // shared vdi ignore
+            return
+          }
+          const vm = userVbds[0].$VM
+          if (!('start' in vm.blocked_operations)) {
+            debug('checkBaseVdis, vm not blocked', { vmRef: vm.$ref })
+            // vm start unlocked
+            // not really an issue since we have check the delta
+            // but it indicates the users played with the blocked operations
+            return
+          }
+          diffDisk = new XapiDiskSource({
+            xapi: sr.$xapi,
+            vdiRef: activeVdi.$ref,
+            baseRef: snapshot.$ref,
+            onlyListChangedBlocks: true,
+          })
+          await diffDisk.init()
+          const sourceUuid = snapshot.other_config?.[COPY_OF]
+          if (diffDisk.getBlockIndexes().length === 0) {
+            // no block modification since the common snapshot, we can chain VM and disk
+            if (sourceUuid) {
+              baseVdisBySourceUuid.set(sourceUuid, activeVdi)
+            }
+            // Track the target VM (the replicated VM to update on the next transfer).
+            targetVmRef = vm.$ref
+          } else {
+            if (sourceUuid) {
+              baseVdisBySourceUuid.set(sourceUuid, snapshot)
+            }
+            // there are changed block since the snapshot
+            // we can reuse it to transfer a delta, but we will
+            // create a new VM
+            canChainToTargetVm = false
+            debug('checkBaseVdis, data between snapshot and active disk', {
+              vdiRef: snapshot.$ref,
+              nbBlocks: diffDisk.getBlockIndexes().length,
+            })
+          }
+        } catch (error) {
+          debug('checkBaseVdis, skipping snapshot', { ref: snapshot.$ref, error })
+          return
+        } finally {
+          await diffDisk?.close().catch(error => debug('checkBaseVdis, error closing', error))
+          await sr.$xapi.VDI_disconnectFromControlDomain(snapshot.$ref)
+          if (activeVdi !== undefined) {
+            await sr.$xapi.VDI_disconnectFromControlDomain(activeVdi.$ref)
+          }
+        }
+      },
+      {
+        concurrency: 4,
+      }
+    )
+
+    if (!canChainToTargetVm) {
+      // if at least one disk has new data, create a new VM
+      // instead of updating it
+      targetVmRef = undefined
+    } else if (targetVmRef !== undefined) {
+      debug('checkBaseVdis,got a valid vm target', targetVmRef)
+    }
+
+    return { baseVdisBySourceUuid, targetVmRef }
+  }
+
   updateUuidAndChain() {
     // nothing to do, the chaining is not modified in this case
   }

package/disks/MergeRemoteDisk.mjs

@@ -222,6 +222,7 @@ export class MergeRemoteDisk {
     await this.#mergeBlocks(parentDisk, childDisk)
     await parentDisk.flushMetadata(childDisk)
     await parentDisk.mergeMetadata(childDisk)
+    this.#state.diskSize = parentDisk.getSizeOnDisk()
   }
 
   /**
@@ -266,8 +267,6 @@ export class MergeRemoteDisk {
 
     await this.#writeState()
 
-    this.#state.diskSize = childDisk.getSizeOnDisk()
-
     this.#onProgress({ total: nBlocks, done: nBlocks })
   }
 

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.72.0",
+  "version": "0.72.1",
   "engines": {
     "node": ">=14.18"
   },