@xen-orchestra/backups 0.69.4 → 0.71.0

package/ImportVmBackup.mjs

@@ -255,6 +255,14 @@ export class ImportVmBackup {
           : await importIncrementalVm(backup, await xapi.getRecord('SR', srRef), {
               newMacAddresses,
             })
+        let size = 0
+        if (isFull) {
+          size = sizeContainer.size
+        } else {
+          for (const disk of Object.values(backup.disks)) {
+            size += disk.getNbGeneratedBlock() * disk.getBlockSize()
+          }
+        }
         const remoteName = adapter._handler._remote.name
         let desc = `Restored on ${formatFilenameDate(+new Date())}`
         if (remoteName !== undefined) {
@@ -275,7 +283,7 @@ export class ImportVmBackup {
         ])
 
         return {
-          size: sizeContainer.size,
+          size,
           id: await xapi.getField('VM', vmRef, 'uuid'),
         }
       }

package/_incrementalVm.mjs

@@ -8,7 +8,7 @@ import { defer } from 'golike-defer'
 import { cancelableMap } from './_cancelableMap.mjs'
 import { Task } from './Task.mjs'
 import pick from 'lodash/pick.js'
-import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
+import { BASE_DELTA_VDI, CONTENT_KEY, COPY_OF, VM_UUID } from './_otherConfig.mjs'
 
 import { VHD_MAX_SIZE, XapiDiskSource } from '@xen-orchestra/xapi'
 import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
@@ -124,7 +124,7 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
   $defer,
   incrementalVm,
   sr,
-  { cancelToken = CancelToken.none, newMacAddresses = false } = {}
+  { cancelToken = CancelToken.none, newMacAddresses = false, targetRef = undefined } = {}
 ) {
   const { version } = incrementalVm
   if (compareVersions(version, '1.0.0') < 0) {
@@ -136,44 +136,106 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
 
   const vdiRecords = incrementalVm.vdis
 
-  // 0. Create suspend_VDI
+  // When targetRef is provided, update the existing VM instead of creating a new one.
+  const targetVm = targetRef !== undefined ? xapi.getObjectByRef(targetRef, undefined) : undefined
+  const isUpdate = targetVm !== undefined && !targetVm.is_a_snapshot && !targetVm.is_a_template
+
+  // 0. Create suspend_VDI (only when creating a new VM).
   let suspendVdi
-  if (vmRecord.suspend_VDI !== undefined && vmRecord.suspend_VDI !== 'OpaqueRef:NULL') {
-    const vdi = vdiRecords[vmRecord.suspend_VDI]
-    if (vdi === undefined) {
-      Task.warning('Suspend VDI not available for this suspended VM', {
-        vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
-      })
-    } else {
-      suspendVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
-      $defer.onFailure(() => suspendVdi.$destroy())
+  if (!isUpdate) {
+    if (vmRecord.suspend_VDI !== undefined && vmRecord.suspend_VDI !== 'OpaqueRef:NULL') {
+      const vdi = vdiRecords[vmRecord.suspend_VDI]
+      if (vdi === undefined) {
+        Task.warning('Suspend VDI not available for this suspended VM', {
+          vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
+        })
+      } else {
+        suspendVdi = await xapi.getRecord('VDI', await xapi.VDI_create(vdi))
+        $defer.onFailure(() => suspendVdi.$destroy())
+      }
     }
   }
 
-  // 1. Create the VM.
-  const vmRef = await xapi.VM_create(
-    {
-      ...vmRecord,
-      affinity: undefined,
-      blocked_operations: {
-        ...vmRecord.blocked_operations,
+  // 1. Create the VM or update the existing one.
+  let vmRef
+  if (isUpdate) {
+    vmRef = targetRef
+    await Promise.all([
+      xapi.setFields('VM', vmRef, {
+        actions_after_crash: vmRecord.actions_after_crash,
+        actions_after_reboot: vmRecord.actions_after_reboot,
+        actions_after_shutdown: vmRecord.actions_after_shutdown,
+        domain_type: vmRecord.domain_type,
+        ha_restart_priority: vmRecord.ha_restart_priority,
+        has_vendor_device: vmRecord.has_vendor_device,
+        HVM_boot_params: vmRecord.HVM_boot_params,
+        HVM_boot_policy: vmRecord.HVM_boot_policy,
+        HVM_shadow_multiplier: vmRecord.HVM_shadow_multiplier,
+        memory_dynamic_max: vmRecord.memory_dynamic_max,
+        memory_dynamic_min: vmRecord.memory_dynamic_min,
+        memory_static_max: vmRecord.memory_static_max,
+        memory_static_min: vmRecord.memory_static_min,
+        name_label: vmRecord.name_label,
+        name_description: vmRecord.name_description,
+        order: vmRecord.order,
+        other_config: vmRecord.other_config,
+        platform: vmRecord.platform,
+        PV_args: vmRecord.PV_args,
+        PV_bootloader: vmRecord.PV_bootloader,
+        PV_bootloader_args: vmRecord.PV_bootloader_args,
+        PV_kernel: vmRecord.PV_kernel,
+        PV_legacy_args: vmRecord.PV_legacy_args,
+        PV_ramdisk: vmRecord.PV_ramdisk,
+        recommendations: vmRecord.recommendations,
+        shutdown_delay: vmRecord.shutdown_delay,
+        start_delay: vmRecord.start_delay,
+        suspend_SR: vmRecord.suspend_SR,
+        tags: vmRecord.tags,
+        user_version: vmRecord.user_version,
+        VCPUs_at_startup: vmRecord.VCPUs_at_startup,
+        VCPUs_max: vmRecord.VCPUs_max,
+        VCPUs_params: vmRecord.VCPUs_params,
+        xenstore_data: vmRecord.xenstore_data,
+      }),
+      targetVm.update_blocked_operations({
         start: 'Importing…',
         start_on: 'Importing…',
+      }),
+    ])
+  } else {
+    vmRef = await xapi.VM_create(
+      {
+        ...vmRecord,
+        affinity: undefined,
+        blocked_operations: {
+          ...vmRecord.blocked_operations,
+          start: 'Importing…',
+          start_on: 'Importing…',
+        },
+        ha_always_run: false,
+        is_a_template: false,
+        name_label: '[Importing…] ' + vmRecord.name_label,
       },
-      ha_always_run: false,
-      is_a_template: false,
-      name_label: '[Importing…] ' + vmRecord.name_label,
-    },
-    {
-      bios_strings: vmRecord.bios_strings,
-      generateMacSeed: newMacAddresses,
-      suspend_VDI: suspendVdi?.$ref,
-    }
-  )
-  $defer.onFailure.call(xapi, 'VM_destroy', vmRef)
+      {
+        bios_strings: vmRecord.bios_strings,
+        generateMacSeed: newMacAddresses,
+        suspend_VDI: suspendVdi?.$ref,
+      }
+    )
+    $defer.onFailure.call(xapi, 'VM_destroy', vmRef)
+  }
 
   // 2. Delete all VBDs which may have been created by the import.
-  await asyncMap(await xapi.getField('VM', vmRef, 'VBDs'), ref => ignoreErrors.call(xapi.call('VBD.destroy', ref)))
+  // In update mode, also collect the VDI refs so orphaned VDIs can be destroyed after new ones are created.
+  const existingVbdRefs = await xapi.getField('VM', vmRef, 'VBDs')
+  const oldVdiRefs = new Set(
+    isUpdate
+      ? (await asyncMap(existingVbdRefs, ref => xapi.getField('VBD', ref, 'VDI'))).filter(
+          ref => ref !== undefined && ref !== 'OpaqueRef:NULL'
+        )
+      : []
+  )
+  await asyncMap(existingVbdRefs, ref => ignoreErrors.call(xapi.call('VBD.destroy', ref)))
 
   // 3. Create VDIs & VBDs.
   const vbdRecords = incrementalVm.vbds
@@ -184,10 +246,18 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     let newVdi
 
     if (vdi.baseVdi?.$ref !== undefined) {
-      newVdi = await xapi.getRecord('VDI', await xapi.VDI_clone(vdi.baseVdi.$ref))
-      $defer.onFailure(() => newVdi.$destroy())
-
+      if (isUpdate) {
+        // In update mode, reuse the existing target VDI directly — no clone needed.
+        newVdi = vdi.baseVdi
+        oldVdiRefs.delete(newVdi.$ref)
+      } else {
+        newVdi = await xapi.getRecord('VDI', await xapi.VDI_clone(vdi.baseVdi.$ref))
+        $defer.onFailure(() => newVdi.$destroy())
+      }
       await newVdi.update_other_config(COPY_OF, vdi.uuid)
+      if (vdi.other_config[CONTENT_KEY] !== undefined) {
+        await newVdi.update_other_config(CONTENT_KEY, vdi.other_config[CONTENT_KEY])
+      }
       if (vdi.virtual_size > newVdi.virtual_size) {
         await newVdi.$callAsync('resize', vdi.virtual_size)
       }
@@ -213,6 +283,16 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     newVdis[vdiRef] = newVdi
   })
 
+  // 3.5. Destroy old VDIs that are no longer attached to the VM.
+  // Uses ignoreErrors because some storage backends refuse to destroy a VDI that still has snapshot children;
+  // those VDIs will become truly orphaned once the old snapshot VMs are cleaned up by _deleteOldEntries.
+  await asyncMap([...oldVdiRefs], ref => ignoreErrors.call(xapi.call('VDI.destroy', ref)))
+
+  // 4. For updates, destroy existing VIFs before recreating them.
+  if (isUpdate) {
+    await asyncMap(await xapi.getField('VM', vmRef, 'VIFs'), ref => ignoreErrors.call(xapi.call('VIF.destroy', ref)))
+  }
+
   const networksByNameLabelByVlan = {}
   let defaultNetwork
   Object.values(xapi.objects.all).forEach(object => {

package/_otherConfig.mjs

@@ -28,6 +28,11 @@ export const EXPORTED_SUCCESSFULLY = 'xo:backup:exported'
 // the VM ( not the snapshot) uuid
 export const VM_UUID = 'xo:backup:vm'
 
+// in `other_config` of source snapshot VDIs and replicated target VDIs
+// contains the UUID of the source snapshot VDI whose content this VDI represents
+// allows direction-agnostic discovery of common base VDIs (e.g. for failback/reverse replication)
+export const CONTENT_KEY = 'xo:backup:contentKey'
+
 async function listVdiRefs(xapi, vmRef) {
   return xapi.VM_getDisks(vmRef)
 }
@@ -80,6 +85,7 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
 export function resetVmOtherConfig(xapi, vmRef) {
   return applyToVmAndVdis(xapi, vmRef, (type, ref) => {
     return xapi.setFieldEntries(type, ref, 'other_config', {
+      [COPY_OF]: null,
       [DATETIME]: null,
       [DELTA_CHAIN_LENGTH]: null,
       [EXPORTED_SUCCESSFULLY]: null,
@@ -131,3 +137,24 @@ export async function markExportSuccessfull(xapi, vmRef) {
     xapi.setFieldEntry(type, ref, 'other_config', EXPORTED_SUCCESSFULLY, 'true')
   )
 }
+
+/**
+ * Set CONTENT_KEY on each VDI of a VM snapshot to its own UUID.
+ *
+ * This marks each snapshot VDI with a unique content identifier that can be
+ * propagated to replicated copies for direction-agnostic base VDI discovery
+ * (e.g. for failback / reverse replication).
+ *
+ * @param {Xapi} xapi
+ * @param {String} snapshotRef
+ * @returns {Promise}
+ */
+export async function setVmSnapshotContentKeys(xapi, snapshotRef) {
+  return applyToVmAndVdis(xapi, snapshotRef, async (type, ref) => {
+    if (type !== 'VDI') {
+      return
+    }
+    const uuid = await xapi.getField(type, ref, 'uuid')
+    return xapi.setFieldEntry(type, ref, 'other_config', CONTENT_KEY, uuid)
+  })
+}

package/_runners/_vmRunners/IncrementalXapi.mjs

@@ -135,16 +135,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
     const deltaChainLength = Math.max(
       ...lastExportedVdis.map(({ other_config }) => Number(other_config[DELTA_CHAIN_LENGTH] ?? 0))
     )
-    const fullInterval = this._settings.fullInterval
-    if (fullInterval !== 0 && fullInterval <= deltaChainLength + 1) {
-      debug('not using base VM because fullInterval reached', {
-        fullInterval,
-        deltaChainLength,
-        eq: fullInterval < deltaChainLength + 1,
-        dc1: deltaChainLength + 1,
-      })
-      return
-    }
 
     const srcVdis = keyBy(await xapi.getRecords('VDI', await this._vm.$getDisks()), '$ref')
 
@@ -173,6 +163,20 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       debug('no base VM found')
       return
     }
+
+    // we do tafter checkbasevdis because we want the writer to know the target VM
+    // especially on replication when we alwayr update the same VM
+    const fullInterval = this._settings.fullInterval
+    if (fullInterval !== 0 && fullInterval <= deltaChainLength + 1) {
+      debug('not using base VM because fullInterval reached', {
+        fullInterval,
+        deltaChainLength,
+        eq: fullInterval < deltaChainLength + 1,
+        dc1: deltaChainLength + 1,
+      })
+      return
+    }
+
     baseUuidToSrcVdiUuid.forEach((srcVdiUuid, baseUuid) => {
       if (presentBaseVdis.has(baseUuid)) {
         debug('found base VDI', {

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -10,7 +10,16 @@ import { defer } from 'golike-defer'
 import { getOldEntries } from '../../_getOldEntries.mjs'
 import { Task } from '../../Task.mjs'
 import { Abstract } from './_Abstract.mjs'
-import { DATETIME, JOB_ID, SCHEDULE_ID, VM_UUID, resetVmOtherConfig, setVmOtherConfig } from '../../_otherConfig.mjs'
+import {
+  COPY_OF,
+  DATETIME,
+  JOB_ID,
+  SCHEDULE_ID,
+  VM_UUID,
+  resetVmOtherConfig,
+  setVmOtherConfig,
+  setVmSnapshotContentKeys,
+} from '../../_otherConfig.mjs'
 
 const { warn, info } = createLogger('xo:backups:AbstractXapi')
 
@@ -96,7 +105,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
               config,
               healthCheckSr,
               job,
-              scheduleId: schedule.id,
+              schedule,
               vmUuid: vm.uuid,
               settings,
             })
@@ -114,7 +123,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
                   config,
                   healthCheckSr,
                   job,
-                  scheduleId: schedule.id,
+                  schedule,
                   vmUuid: vm.uuid,
                   remoteId,
                   settings: targetSettings,
@@ -132,7 +141,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
               healthCheckSr,
               job,
               ReplicationWriter,
-              scheduleId: schedule.id,
+              schedule,
               vmUuid: vm.uuid,
               srs,
               settings,
@@ -150,7 +159,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
                   config,
                   healthCheckSr,
                   job,
-                  scheduleId: schedule.id,
+                  schedule,
                   vmUuid: vm.uuid,
                   sr,
                   settings: targetSettings,
@@ -165,11 +174,9 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
 
   // ensure the VM itself does not have any backup metadata which would be
   // copied on manual snapshots and interfere with the backup jobs
+
   async _cleanMetadata() {
-    const vm = this._vm
-    if (JOB_ID in vm.other_config) {
-      await resetVmOtherConfig(this._xapi, vm.$ref)
-    }
+    await resetVmOtherConfig(this._xapi, this._vm.$ref)
   }
 
   async _snapshot() {
@@ -205,6 +212,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
           scheduleId: this.scheduleId,
           vmUuid: vm.uuid,
         })
+        await setVmSnapshotContentKeys(xapi, snapshotRef)
         const snapshot = await xapi.getRecord('VM', snapshotRef)
         await snapshot.set_name_label(this._getSnapshotNameLabel(vm))
         // reload data to ensure it is up to date with the new name label
@@ -233,11 +241,16 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     const xapi = this._xapi
 
     const vdiCandidates = {}
-
+    const vdiUuids = this._vm.$VBDs.map(({ VDI }) => VDI)
     Object.values(xapi.objects.indexes.type.VDI)
       .filter(_ => !!_) // filter nullish
-      .filter(({ other_config, $snapshot_of }) => {
-        return $snapshot_of !== undefined && other_config[JOB_ID] === jobId && other_config[VM_UUID] === this._vm.uuid
+      .filter(({ other_config, snapshot_of }) => {
+        return (
+          vdiUuids.includes(snapshot_of) &&
+          other_config[JOB_ID] === jobId &&
+          other_config[VM_UUID] === this._vm.uuid &&
+          other_config[COPY_OF] === undefined
+        )
       })
       .forEach(vdi => {
         vdiCandidates[vdi.uuid] = vdi

package/_runners/_writers/FullRemoteWriter.mjs

@@ -27,7 +27,7 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
   async _run({ maxStreamLength, timestamp, sizeContainer, stream, streamLength, vm, vmSnapshot }) {
     const settings = this._settings
     const job = this._job
-    const scheduleId = this._scheduleId
+    const scheduleId = this._schedule.id
 
     const adapter = this._adapter
     let metadata = await this._isAlreadyTransferred(timestamp)

package/_runners/_writers/FullXapiWriter.mjs

@@ -34,7 +34,7 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
     const sr = this._sr
     const settings = this._settings
     const job = this._job
-    const scheduleId = this._scheduleId
+    const scheduleId = this._schedule.id
 
     const { uuid: srUuid, $xapi: xapi } = sr
 

package/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -90,7 +90,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
   async _prepare() {
     const adapter = this._adapter
     const settings = this._settings
-    const scheduleId = this._scheduleId
+    const scheduleId = this._schedule.id
     const vmUuid = this._vmUuid
 
     const oldEntries = getOldEntries(
@@ -180,7 +180,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
   async _transfer($defer, { isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot }) {
     const adapter = this._adapter
     const job = this._job
-    const scheduleId = this._scheduleId
+    const scheduleId = this._schedule.id
     const settings = this._settings
     const jobId = job.id
     const handler = adapter.handler

package/_runners/_writers/IncrementalXapiWriter.mjs

@@ -1,3 +1,5 @@
+import humanFormat from 'human-format'
+
 import { asyncMapSettled } from '@xen-orchestra/async-map'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
 
@@ -7,7 +9,7 @@ import { Task } from '../../Task.mjs'
 
 import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
 import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
-import { listReplicatedVms } from './_listReplicatedVms.mjs'
+import { compareReplicatedVmDatetime, listReplicatedVms } from './_listReplicatedVms.mjs'
 import {
   COPY_OF,
   setVmOtherConfig,
@@ -18,34 +20,102 @@ import {
   DATETIME,
   VM_UUID,
 } from '../../_otherConfig.mjs'
-import assert from 'node:assert'
 import { formatFilenameDate } from '../../_filenameDate.mjs'
+import { XapiDiskSource } from '@xen-orchestra/xapi'
+import { asyncEach } from '@vates/async-each'
+import { createLogger } from '@xen-orchestra/log'
+
+const { debug } = createLogger('xo:backups:IncrementalXapiWriter')
 
 export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
+  // Map of source VDI UUID (COPY_OF) → validated active VDI on the target SR.
+  // Built by checkBaseVdis, consumed by #decorateVmMetadata to set baseVdi.
+  #baseVdisBySourceUuid = new Map()
+
   async checkBaseVdis(baseUuidToSrcVdi) {
     const sr = this._sr
+    this.#baseVdisBySourceUuid = new Map()
+
     if (baseUuidToSrcVdi.size === 0) {
       // searching for the vdis is expensive
       // don't do it if there is nothing to find
       return
     }
 
-    // @todo use an index if possible
-    // @todo : this seems similar to decorateVmMetadata
-    const replicatedVdis = sr.$VDIs
-      .filter(vdi => {
-        // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
-        return (
-          vdi?.managed &&
-          !vdi?.is_a_snapshot /* only look for real vdi */ &&
-          baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
-        )
-      })
-      .map(({ other_config }) => other_config?.[COPY_OF])
-      .filter(_ => !!_)
+    // look for the same snapshot
+    // ensure there are no data between the snapshot and the active disk
+
+    const snapshotCandidates = sr.$VDIs.filter(vdi => {
+      return (
+        vdi?.managed &&
+        vdi?.is_a_snapshot &&
+        vdi.other_config[JOB_ID] === this._job.id &&
+        vdi.other_config[VM_UUID] === this._vmUuid &&
+        baseUuidToSrcVdi.has(vdi?.other_config[COPY_OF])
+      )
+    })
+    debug('checkBaseVdis, got snapshot candidates,', snapshotCandidates.length)
+
+    // ensure no data have been written since this snapshot
+    // but there may be have some other snapshot for another job
+    let targetVmRef
+    let canChainToTargetVm = true
+    await asyncEach(
+      snapshotCandidates,
+      async snapshot => {
+        let diffDisk
+        try {
+          const activeVdi = sr.$xapi.getObject(snapshot.$snapshot_of)
+          const userVbds = activeVdi.$VBDs?.filter(vbd => vbd.$VM && !vbd.$VM.is_control_domain) ?? []
+          if (userVbds.length !== 1) {
+            debug('checkBaseVdis, share vbd ', { ref: snapshot.$ref, userVbds })
+            // shared vdi ignore
+            return
+          }
+          const vm = userVbds[0].$VM
+          if (!('start' in vm.blocked_operations)) {
+            debug('checkBaseVdis, vm not blocked', { vmRef: vm.$ref })
+            // vm start unlocked
+            // not really an issue since we have check the delta
+            // but it indicates the users played with the blocked operations
+            return
+          }
+          diffDisk = new XapiDiskSource({ xapi: sr.$xapi, vdiRef: activeVdi.$ref, baseRef: snapshot.$ref })
+          await diffDisk.init()
+          if (diffDisk.getBlockIndexes().length === 0) {
+            const sourceUuid = snapshot.other_config?.[COPY_OF]
+            if (sourceUuid) {
+              this.#baseVdisBySourceUuid.set(sourceUuid, activeVdi)
+            }
+            // Track the target VM (the replicated VM to update on the next transfer).
+            targetVmRef = vm.$ref
+          } else {
+            // not empty, we will create a new VM
+            canChainToTargetVm = false
+            debug('checkBaseVdis, data between snapshot and active disk', {
+              vdiRef: snapshot.$ref,
+              nbBlocks: diffDisk.getBlockIndexes().length,
+            })
+          }
+        } catch (error) {
+          debug('checkBaseVdis, skipping snapshot', { ref: snapshot.$ref, error })
+          return
+        } finally {
+          await diffDisk?.close().catch(error => debug('checkBaseVdis, error closing', error))
+        }
+      },
+      {
+        concurrency: 4,
+      }
+    )
+
+    if (canChainToTargetVm && targetVmRef !== undefined) {
+      debug('checkBaseVdis,got a valid vm target', targetVmRef)
+      this._targetVmRef = targetVmRef
+    }
 
     for (const uuid of baseUuidToSrcVdi.keys()) {
-      if (!replicatedVdis.includes(uuid)) {
+      if (!this.#baseVdisBySourceUuid.has(uuid)) {
         baseUuidToSrcVdi.delete(uuid)
       }
     }
@@ -76,12 +146,21 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
     const settings = this._settings
     const { uuid: srUuid, $xapi: xapi } = this._sr
     const vmUuid = this._vmUuid
-    const scheduleId = this._scheduleId
+    const scheduleId = this._schedule.id
 
     // delete previous interrupted copies
     ignoreErrors.call(asyncMapSettled(listReplicatedVms(xapi, scheduleId, undefined, vmUuid), vm => vm.$destroy))
 
-    this._oldEntries = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vmUuid))
+    const allEntries = listReplicatedVms(xapi, scheduleId, srUuid, vmUuid)
+
+    // In the snapshot-based flow a non-snapshot VM (the live target) coexists with its
+    // snapshots (one per transfer). That VM must not be subject to retention — only its
+    // snapshots are. Build the set of VM refs that already have snapshots in the list so
+    // we can exclude them, while keeping old-style non-snapshot VMs (no snapshots).
+    const vmRefsWithSnapshots = new Set(allEntries.filter(e => e.is_a_snapshot).map(e => e.snapshot_of))
+    const retentionEntries = allEntries.filter(e => e.is_a_snapshot || !vmRefsWithSnapshots.has(e.$ref))
+    retentionEntries.sort(compareReplicatedVmDatetime)
+    this._oldEntries = getOldEntries(settings.copyRetention - 1, retentionEntries)
 
     if (settings.deleteFirst && settings.skipDeleteOldEntries) {
       // we want to keep the baseVM when copying a delta
@@ -110,9 +189,8 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
     const sr = this._sr
     const vm = backup.vm
     const job = this._job
-    const scheduleId = this._scheduleId
+    const scheduleId = this._schedule.id
 
-    vm.name_label = `${vm.name_label} - ${job.name} - (${formatFilenameDate(timestamp)})`
     // update other_config data as soon as possible to ensure the next job
     // will be able to detect any partial transfer and lean them
     vm.other_config[COPY_OF] = vm.uuid
@@ -130,18 +208,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
     if (!_warmMigration) {
       vm.tags.push('Continuous Replication')
     }
-    // extracting the uuid of each delta vdi on the source
-    // get all in one pass, since there is a lot of objects
-    const sourceVdiUuids = Object.values(backup.vdis)
-      .map(({ other_config }) => other_config[BASE_DELTA_VDI])
-      // full vdi don't have a base
-      .filter(_ => !!_)
-    // @todo use index ?
-
-    const replicatedVdis = sr.$VDIs.filter(vdi => {
-      // REPLICATED_TO_SR_UUID is not used here since we are already filtering from sr.$VDIs
-      return vdi?.managed && !vdi?.is_a_snapshot && sourceVdiUuids.includes(vdi?.other_config[COPY_OF])
-    })
 
     Object.values(backup.vdis).forEach(vdi => {
       vdi.other_config[COPY_OF] = vdi.uuid
@@ -150,18 +216,12 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
       vdi.other_config[REPLICATED_TO_SR_UUID] = sr.uuid
       vdi.other_config[VM_UUID] = vm.uuid
 
-      if (sourceVdiUuids.length > 0) {
-        const baseReplicatedTo = replicatedVdis.filter(
-          replicatedVdi => replicatedVdi.other_config[COPY_OF] === vdi.other_config[BASE_DELTA_VDI]
-        )
-        assert.ok(
-          baseReplicatedTo.length <= 1,
-          `Target of a replication must be unique, got ${baseReplicatedTo.length} candidates`
-        )
-        // baseReplicatedTo can be undefined if a new disk is added and other are already replicated
-        vdi.baseVdi = baseReplicatedTo[0]
+      const baseDeltaVdiUuid = vdi.other_config[BASE_DELTA_VDI]
+      if (baseDeltaVdiUuid !== undefined) {
+        // reuse the validated mapping built by checkBaseVdis
+        vdi.baseVdi = this.#baseVdisBySourceUuid.get(baseDeltaVdiUuid)
       } else {
-        // first replication of this disk
+        // first replication of this disk (full, no base)
         vdi.baseVdi = undefined
       }
       // ensure the VDI are created on the target SR
@@ -175,18 +235,43 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
     const { _warmMigration } = this._settings
     const sr = this._sr
     const job = this._job
-    const scheduleId = this._scheduleId
+    const schedule = this._schedule
+    const scheduleId = schedule.id
     const { uuid: srUuid, $xapi: xapi } = sr
 
     let targetVmRef
     await Task.run({ name: 'transfer' }, async () => {
-      targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport, timestamp), sr)
-      // size is mandatory to ensure the task have the right data
+      targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport, timestamp), sr, {
+        targetRef: this._targetVmRef,
+      })
+      // this also ensure the data are up to date on the snapshot
+      await setVmOtherConfig(xapi, targetVmRef, {
+        timestamp, // updated at the end to mark the transfer as complete
+        jobId: job.id,
+        scheduleId,
+        vmUuid: vm.uuid,
+        srUuid,
+      })
+      const size = Object.values(deltaExport.disks).reduce(
+        (sum, disk) => sum + disk.getNbGeneratedBlock() * disk.getBlockSize(),
+        0
+      )
+      await xapi.setField(
+        'VM',
+        targetVmRef,
+        'name_description',
+        deltaExport.vm.name_description +
+          ` -- last replication: ${formatFilenameDate(timestamp)} ${humanFormat.bytes(size)} read`
+      )
+      // take a snapshot to ensure these data are not modified until next snapshot
+      await Task.run({ name: 'target snapshot' }, async () => {
+        await xapi.VM_snapshot(targetVmRef, {
+          name_label: `${vm.name_label} - ${job.name} / ${schedule.name} ${formatFilenameDate(timestamp)}`,
+        })
+      })
+
       return {
-        size: Object.values(deltaExport.disks).reduce(
-          (sum, disk) => sum + disk.getNbGeneratedBlock() * disk.getBlockSize(),
-          0
-        ),
+        size,
       }
     })
     this._targetVmRef = targetVmRef
@@ -197,13 +282,6 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
       !_warmMigration &&
         targetVm.ha_restart_priority !== '' &&
         Promise.all([targetVm.set_ha_restart_priority(''), targetVm.add_tags('HA disabled')]),
-      setVmOtherConfig(xapi, targetVmRef, {
-        timestamp, // updated at the end to mark the transfer as complete
-        jobId: job.id,
-        scheduleId,
-        vmUuid: vm.uuid,
-        srUuid,
-      }),
     ])
   }
 }

package/_runners/_writers/_AbstractWriter.mjs

@@ -1,9 +1,9 @@
 export class AbstractWriter {
-  constructor({ config, healthCheckSr, job, vmUuid, scheduleId, settings }) {
+  constructor({ config, healthCheckSr, job, vmUuid, schedule, settings }) {
     this._config = config
     this._healthCheckSr = healthCheckSr
     this._job = job
-    this._scheduleId = scheduleId
+    this._schedule = schedule
     this._settings = settings
     this._vmUuid = vmUuid
   }

package/_runners/_writers/_MixinRemoteWriter.mjs

@@ -50,7 +50,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
 
     async getLongTermRetentionTags(currentEntry) {
       const settings = this._settings
-      const scheduleId = this._scheduleId
+      const scheduleId = this._schedule.id
       const vmUuid = this._vmUuid
       const adapter = this._adapter
 

package/_runners/_writers/_listReplicatedVms.mjs

@@ -23,12 +23,13 @@ export function listReplicatedVms(xapi, scheduleOrJobId, srUuid, vmUuid) {
     const oc = object.other_config
     if (
       object.$type === 'VM' &&
-      !object.is_a_snapshot &&
       !object.is_a_template &&
-      'start' in object.blocked_operations &&
       (oc[JOB_ID] === scheduleOrJobId || oc[SCHEDULE_ID] === scheduleOrJobId) &&
       oc[REPLICATED_TO_SR_UUID] === srUuid &&
-      oc[VM_UUID] === vmUuid
+      oc[VM_UUID] === vmUuid &&
+      // Old-style replication: one VM per transfer (non-snapshot, start blocked)
+      // New-style replication: snapshots of the target VM represent each transfer
+      (!object.is_a_snapshot ? 'start' in object.blocked_operations : true)
     ) {
       vms[object.$id] = object
     }

package/disks/MergeRemoteDisk.mjs

@@ -22,6 +22,7 @@ const { warn } = createLogger('remote-disk:merge')
  * @property {number} mergedDataSize
  * @property {'mergeBlocks' | 'cleanup'} step
  * @property {number} diskSize
+ * @typedef {(message: string, data?: Record<string, unknown>) => void} Logger
  */
 
 export class MergeRemoteDisk {
@@ -85,11 +86,11 @@ export class MergeRemoteDisk {
   /**
    * @param {FileAccessor} handler
    * @param {Object} params
-   * @param {Function} params.onProgress
-   * @param {Logger | Function} params.logInfo
-   * @param {boolean} params.removeUnused
-   * @param {number} params.mergeBlockConcurrency
-   * @param {number} params.writeStateDelay
+   * @param {Function} [params.onProgress]
+   * @param {Logger | Function} [params.logInfo]
+   * @param {boolean} [params.removeUnused]
+   * @param {number} [params.mergeBlockConcurrency]
+   * @param {number} [params.writeStateDelay]
    */
   constructor(
     handler,
@@ -202,8 +203,8 @@ export class MergeRemoteDisk {
 
       parentDisk.setAllocatedBlocks(alreadyMergedBlocks)
     } else {
-      this.#state.child = { uuid: childDisk.getUuid() ?? 0 }
-      this.#state.parent = { uuid: parentDisk.getUuid() ?? 0 }
+      this.#state.child = { uuid: childDisk.getUuid() ?? undefined }
+      this.#state.parent = { uuid: parentDisk.getUuid() ?? undefined }
 
       // Finds first allocated block for the 2 following loops
       while (this.#state.currentBlock < getMaxBlockCount && !childDisk.hasBlock(this.#state.currentBlock)) {

package/disks/RemoteVhdDisk.mjs

@@ -95,10 +95,14 @@ export class RemoteVhdDisk extends RemoteDisk {
 
   /**
    * Closes the VHD.
+   * We replace the dispose function call so the disk can only be closed once.
+   *
    * @returns {Promise<void>}
    */
   async close() {
-    await this.#dispose()
+    const dispose = this.#dispose
+    this.#dispose = () => Promise.resolve()
+    await dispose()
   }
 
   /**

package/disks/index.mjs

@@ -0,0 +1,49 @@
+import { RemoteVhdDisk } from './RemoteVhdDisk.mjs'
+
+export { RemoteDisk } from './RemoteDisk.mjs'
+export { openDiskChain } from './openDiskChain.mjs'
+
+/**
+ * @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
+ * @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
+ */
+
+/**
+ * @param {Object} params
+ * @param {FileAccessor} params.handler
+ * @param {string} params.path
+ * @returns {Promise<RemoteDisk>}
+ */
+export async function openDisk({ handler, path }) {
+  const disk = new RemoteVhdDisk({ handler, path })
+  await disk.init()
+  return disk
+}
+/**
+ *
+ * @param {Object} params
+ * @param {FileAccessor} params.handler
+ * @param {string} params.path
+ * @returns {Promise<Disposable<RemoteDisk>>}
+ */
+export async function openDisposableDisk({ handler, path }) {
+  const disk = new RemoteVhdDisk({ handler, path })
+  await disk.init()
+  return {
+    value: disk,
+    dispose: () => disk.close(),
+  }
+}
+
+const DISK_EXTENSIONS = ['.vhd']
+
+/**
+ * Returns true if the path points to a supported disk format.
+ *
+ * @param {FileAccessor} _handler - Remote file handler (reserved for future use)
+ * @param {string} path - Path to check
+ * @returns {boolean}
+ */
+export function isDisk(_handler, path) {
+  return DISK_EXTENSIONS.some(ext => path.endsWith(ext))
+}

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.69.4",
+  "version": "0.71.0",
   "engines": {
     "node": ">=14.18"
   },
@@ -19,20 +19,20 @@
   "dependencies": {
     "@iarna/toml": "^2.2.5",
     "@kldzj/stream-throttle": "^1.1.1",
-    "@vates/async-each": "^1.0.1",
+    "@vates/async-each": "^1.0.2",
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.1.0",
     "@vates/disposable": "^0.1.6",
     "@vates/fuse-vhd": "^2.1.2",
-    "@vates/generator-toolbox": "^1.1.0",
-    "@vates/nbd-client": "^3.2.3",
+    "@vates/generator-toolbox": "^1.1.1",
+    "@vates/nbd-client": "^3.3.0",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/disk-transform": "^1.2.1",
-    "@xen-orchestra/fs": "^4.6.7",
+    "@xen-orchestra/disk-transform": "^1.2.2",
+    "@xen-orchestra/fs": "^4.7.0",
     "@xen-orchestra/log": "^0.7.1",
-    "@xen-orchestra/qcow2": "^1.1.2",
+    "@xen-orchestra/qcow2": "^1.2.0",
     "@xen-orchestra/template": "^0.1.0",
     "app-conf": "^3.0.0",
     "compare-versions": "^6.0.0",
@@ -51,7 +51,7 @@
     "tar": "^7.5.3",
     "uuid": "^9.0.0",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^4.14.7",
+    "vhd-lib": "^4.15.0",
     "xen-api": "^4.7.6",
     "yazl": "^2.5.1"
   },
@@ -62,11 +62,15 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^8.6.6"
+    "@xen-orchestra/xapi": "^8.7.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",
     "url": "https://vates.fr"
+  },
+  "exports": {
+    "./disks": "./disks/index.mjs",
+    "./*": "./*"
   }
 }