@xen-orchestra/backups 0.58.4 → 0.60.0

package/ImportVmBackup.mjs

@@ -4,12 +4,13 @@ import { formatFilenameDate } from './_filenameDate.mjs'
 import { importIncrementalVm } from './_incrementalVm.mjs'
 import { Task } from './Task.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
-import { VhdNegative, VhdSynthetic } from 'vhd-lib'
 import { decorateClass } from '@vates/decorate-with'
 import { createLogger } from '@xen-orchestra/log'
 import { dirname, join } from 'node:path'
 import pickBy from 'lodash/pickBy.js'
 import { defer } from 'golike-defer'
+import { NegativeDisk } from '@xen-orchestra/disk-transform'
+import { openDiskChain } from './disks/openDiskChain.mjs'
 
 const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
 async function resolveUuid(xapi, cache, uuid, type) {
@@ -59,7 +60,7 @@ export class ImportVmBackup {
     const metadata = this._metadata
     const { mapVdisSrs } = this._importIncrementalVmSettings
     const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
-    const streams = {}
+    const disks = {}
     const metdataDir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
 
@@ -110,20 +111,21 @@ export class ImportVmBackup {
         }
       }
 
-      let stream
+      let disk
       const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
       if (vhdPath === backupWithSnapshotPath) {
         // all the data are already on the host
         debug('direct reuse of a snapshot')
-        stream = null
+        disk = null
         vdis[vdiRef].baseVdi = snapshotCandidate
         // go next disk , we won't use this stream
         continue
       }
 
-      let disposableDescendants
-
-      const disposableSynthetic = await VhdSynthetic.fromVhdChain(this._adapter._handler, vhdPath)
+      const parent = await openDiskChain({
+        handler: this._adapter._handler,
+        path: vhdPath,
+      })
 
       // this will also clean if another disk of this VM backup fails
       // if user really only need to restore non failing disks he can retry with ignoredVdis
@@ -132,8 +134,7 @@ export class ImportVmBackup {
         if (!disposed) {
           disposed = true
           try {
-            await disposableDescendants?.dispose()
-            await disposableSynthetic?.dispose()
+            await parent?.close()
           } catch (error) {
             warn('openVhd: failed to dispose VHDs', { error })
           }
@@ -141,11 +142,10 @@ export class ImportVmBackup {
       }
       $defer.onFailure(() => disposeOnce())
 
-      const parentVhd = disposableSynthetic.value
-      await parentVhd.readBlockAllocationTable()
-      debug('got vhd synthetic of parents', parentVhd.length)
+      debug('got vhd synthetic of parents', parent)
 
       if (snapshotCandidate !== undefined) {
+        let descendant, negativeDisk
         try {
           debug('will try to use differential restore', {
             backupWithSnapshotPath,
@@ -153,39 +153,37 @@ export class ImportVmBackup {
             vdiRef,
           })
 
-          disposableDescendants = await VhdSynthetic.fromVhdChain(this._adapter._handler, backupWithSnapshotPath, {
+          descendant = await openDiskChain({
+            handler: this._adapter._handler,
+            path: backupWithSnapshotPath,
             until: vhdPath,
           })
-          const descendantsVhd = disposableDescendants.value
-          await descendantsVhd.readBlockAllocationTable()
+
           debug('got vhd synthetic of descendants')
-          const negativeVhd = new VhdNegative(parentVhd, descendantsVhd)
+          negativeDisk = new NegativeDisk(parent, descendant)
           debug('got vhd negative')
 
           // update the stream with the negative vhd stream
-          stream = await negativeVhd.stream()
+          disk = negativeDisk
           vdis[vdiRef].baseVdi = snapshotCandidate
         } catch (error) {
           // can be a broken VHD chain, a vhd chain with a key backup, ....
           // not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
           warn(`can't use differential restore`, { error })
-          disposableDescendants?.dispose()
+          descendant?.close()
+          negativeDisk?.close()
         }
       }
       // didn't make a negative stream : fallback to classic stream
-      if (stream === undefined) {
+      if (disk === undefined) {
         debug('use legacy restore')
-        stream = await parentVhd.stream()
+        disk = parent
       }
-
-      stream.on('end', disposeOnce)
-      stream.on('close', disposeOnce)
-      stream.on('error', disposeOnce)
-      info('everything is ready, will transfer', stream.length)
-      streams[`${vdiRef}.vhd`] = stream
+      info('everything is ready, will transfer', disk)
+      disks[vdiRef] = disk
     }
     return {
-      streams,
+      disks,
       vbds,
       vdis,
       version: '1.0.0',
@@ -241,7 +239,6 @@ export class ImportVmBackup {
       assert.strictEqual(metadata.mode, 'delta')
 
       backup = await this.#decorateIncrementalVmMetadata()
-      Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
     }
 
     return Task.run(

package/RemoteAdapter.mjs

@@ -2,7 +2,7 @@ import { asyncEach } from '@vates/async-each'
 import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
 import { compose } from '@vates/compose'
 import { createLogger } from '@xen-orchestra/log'
-import { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } from 'vhd-lib'
+import { VhdDirectory, VhdSynthetic } from 'vhd-lib'
 import { decorateMethodsWith } from '@vates/decorate-with'
 import { deduped } from '@vates/disposable/deduped.js'
 import { dirname, join, resolve } from 'node:path'
@@ -10,7 +10,6 @@ import { execFile } from 'child_process'
 import { mount } from '@vates/fuse-vhd'
 import { readdir, lstat } from 'node:fs/promises'
 import { synchronized } from 'decorator-synchronized'
-import { v4 as uuidv4 } from 'uuid'
 import { ZipFile } from 'yazl'
 import Disposable from 'promise-toolbox/Disposable'
 import fromCallback from 'promise-toolbox/fromCallback'
@@ -32,6 +31,10 @@ import { listPartitions, LVM_PARTITION_TYPE } from './_listPartitions.mjs'
 import { lvs, pvs } from './_lvm.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
 
+import { RemoteVhd } from './disks/RemoteVhd.mjs'
+import { openDiskChain } from './disks/openDiskChain.mjs'
+import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
+
 export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 
 export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
@@ -423,12 +426,20 @@ export class RemoteAdapter {
   async listAllVms() {
     const handler = this._handler
     const vmsUuids = []
-    await asyncEach(await handler.list(BACKUP_DIR), async entry => {
-      // ignore hidden and lock files
-      if (entry[0] !== '.' && !entry.endsWith('.lock')) {
-        vmsUuids.push(entry)
+    try {
+      await asyncEach(await handler.list(BACKUP_DIR), async entry => {
+        // ignore hidden and lock files
+        if (entry[0] !== '.' && !entry.endsWith('.lock')) {
+          vmsUuids.push(entry)
+        }
+      })
+    } catch (error) {
+      // remote without any VM backup are ok
+      if (error.code !== 'ENOENT') {
+        throw error
       }
-    })
+    }
+
     return vmsUuids
   }
 
@@ -681,22 +692,24 @@ export class RemoteAdapter {
     return path
   }
 
-  async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency } = {}) {
+  async writeVhd(path, disk, { validator = noop, writeBlockConcurrency } = {}) {
     const handler = this._handler
+
     if (this.useVhdDirectory()) {
-      const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
-      const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
-        concurrency: writeBlockConcurrency,
-        compression: this.#getCompressionType(),
-        async validator() {
-          await input.task
-          return validator.apply(this, arguments)
+      await writeToVhdDirectory({
+        disk,
+        target: {
+          handler,
+          path,
+          concurrency: writeBlockConcurrency,
+          validator,
+          compression: 'brotli',
         },
       })
-      await VhdAbstract.createAlias(handler, path, dataPath)
-      return size
     } else {
-      return this.outputStream(path, input, { checksum, validator })
+      const stream = await toVhdStream({ disk })
+      await this.outputStream(path, stream, { validator })
+      await validator(path)
     }
   }
 
@@ -720,30 +733,15 @@ export class RemoteAdapter {
   }
 
   // open the  hierarchy of ancestors until we find a full one
-  async _createVhdStream(handler, path, { useChain }) {
-    const disposableSynthetic = useChain ? await VhdSynthetic.fromVhdChain(handler, path) : await openVhd(handler, path)
-    // I don't want the vhds to be disposed on return
-    // but only when the stream is done ( or failed )
-
-    let disposed = false
-    const disposeOnce = async () => {
-      if (!disposed) {
-        disposed = true
-        try {
-          await disposableSynthetic.dispose()
-        } catch (error) {
-          warn('openVhd: failed to dispose VHDs', { error })
-        }
-      }
+  async _createVhdDisk(handler, path, { useChain }) {
+    let disk
+    if (useChain) {
+      disk = await openDiskChain({ handler, path })
+    } else {
+      disk = new RemoteVhd({ handler, path })
+      await disk.init()
     }
-    const synthetic = disposableSynthetic.value
-    await synthetic.readBlockAllocationTable()
-    const stream = await synthetic.stream()
-
-    stream.on('end', disposeOnce)
-    stream.on('close', disposeOnce)
-    stream.on('error', disposeOnce)
-    return stream
+    return disk
   }
 
   async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
@@ -751,14 +749,14 @@ export class RemoteAdapter {
     const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
     const dir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
-
-    const streams = {}
+    const disks = {}
     await asyncMapSettled(Object.keys(vdis), async ref => {
-      streams[`${ref}.vhd`] = await this._createVhdStream(handler, join(dir, vhds[ref]), { useChain })
+      delete vdis[ref].baseVdi
+      disks[ref] = await this._createVhdDisk(handler, join(dir, vhds[ref]), { useChain })
     })
 
     return {
-      streams,
+      disks,
       vbds,
       vdis,
       version: '1.0.0',

package/_cleanVm.mjs

@@ -595,7 +595,7 @@ export async function cleanVm(
         // all disks are now key disk
         metadata.isVhdDifferencing = {}
         for (const id of Object.keys(metadata.vdis ?? {})) {
-          metadata.isVhdDifferencing[`${id}.vhd`] = false
+          metadata.isVhdDifferencing[id] = false
         }
       }
       mustRegenerateCache = true

package/_incrementalVm.mjs

@@ -10,6 +10,9 @@ import { Task } from './Task.mjs'
 import pick from 'lodash/pick.js'
 import { BASE_DELTA_VDI, COPY_OF, VM_UUID } from './_otherConfig.mjs'
 
+import { XapiDiskSource } from '@xen-orchestra/xapi'
+import { toVhdStream } from 'vhd-lib/disk-consumer/index.mjs'
+
 const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
 
 export async function exportIncrementalVm(
@@ -19,7 +22,7 @@ export async function exportIncrementalVm(
 ) {
   // refs of VM's VDIs → base's VDIs.
 
-  const streams = {}
+  const disks = {}
   const vdis = {}
   const vbds = {}
   await cancelableMap(cancelToken, vm.$VBDs, async (cancelToken, vbd) => {
@@ -52,32 +55,14 @@ export async function exportIncrementalVm(
       $snapshot_of$uuid: vdi.$snapshot_of?.uuid,
       $SR$uuid: vdi.$SR.uuid,
     }
-    try {
-      streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
-        baseRef: baseVdi?.$ref,
-        cancelToken,
-        format: 'vhd',
-        nbdConcurrency,
-        preferNbd,
-      })
-    } catch (err) {
-      if (err.code === 'VDI_CANT_DO_DELTA') {
-        // fall back to a base
-        Task.info(`Can't do delta, will try to get a full stream`, { vdi })
-        streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
-          cancelToken,
-          format: 'vhd',
-          nbdConcurrency,
-          preferNbd,
-        })
-        // only warn if the fall back succeed
-        Task.warning(`Can't do delta with this vdi, transfer will be a full`, {
-          vdi,
-        })
-      } else {
-        throw err
-      }
-    }
+    disks[vdiRef] = new XapiDiskSource({
+      vdiRef,
+      xapi: vm.$xapi,
+      baseRef: baseVdi?.$ref,
+      nbdConcurrency,
+      preferNbd,
+    })
+    await disks[vdiRef].init()
   })
 
   const suspendVdi = vm.$suspend_VDI
@@ -87,10 +72,13 @@ export async function exportIncrementalVm(
       ...suspendVdi,
       $SR$uuid: suspendVdi.$SR.uuid,
     }
-    streams[`${vdiRef}.vhd`] = await suspendVdi.$exportContent({
-      cancelToken,
-      format: 'vhd',
+    disks[vdiRef] = new XapiDiskSource({
+      vdiRef: suspendVdi.$ref,
+      xapi: vm.$xapi,
+      nbdConcurrency,
+      preferNbd,
     })
+    await disks[vdiRef].init()
   }
 
   const vifs = {}
@@ -116,24 +104,17 @@ export async function exportIncrementalVm(
     })
   )
 
-  return Object.defineProperty(
-    {
-      version: '1.1.0',
-      vbds,
-      vdis,
-      vifs,
-      vm: {
-        ...vm,
-      },
-      vtpms,
+  return {
+    version: '1.1.0',
+    vbds,
+    vdis,
+    vifs,
+    vm: {
+      ...vm,
     },
-    'streams',
-    {
-      configurable: true,
-      value: streams,
-      writable: true,
-    }
-  )
+    vtpms,
+    disks,
+  }
 }
 
 export const importIncrementalVm = defer(async function importIncrementalVm(
@@ -199,8 +180,8 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     const vdi = vdiRecords[vdiRef]
     let newVdi
 
-    if (vdi.baseVdi !== undefined) {
-      newVdi = await xapi.getRecord('VDI', await vdi.baseVdi.$clone())
+    if (vdi.baseVdi?.$ref !== undefined) {
+      newVdi = await xapi.getRecord('VDI', await xapi.VDI_clone(vdi.baseVdi.$ref))
       $defer.onFailure(() => newVdi.$destroy())
 
       await newVdi.update_other_config(COPY_OF, vdi.uuid)
@@ -244,20 +225,17 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
     }
   })
 
-  const { streams } = incrementalVm
-
+  const { disks } = incrementalVm
   await Promise.all([
     // Import VDI contents.
     cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
-      for (let stream of ensureArray(streams[`${id}.vhd`])) {
-        if (stream === null) {
+      for (const disk of ensureArray(disks[id])) {
+        if (disk === null) {
           // we restore a backup and reuse completly a local snapshot
           continue
         }
-        if (typeof stream === 'function') {
-          stream = await stream()
-        }
         await xapi.setField('VDI', vdi.$ref, 'name_label', `[Importing] ${vdiRecords[id].name_label}`)
+        const stream = await toVhdStream({ disk })
         await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
         await xapi.setField('VDI', vdi.$ref, 'name_label', vdiRecords[id].name_label)
       }

package/_runners/VmsRemote.mjs

@@ -4,11 +4,11 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
 
 import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
 import { Task } from '../Task.mjs'
-import createStreamThrottle from './_createStreamThrottle.mjs'
 import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
 import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { FullRemote } from './_vmRunners/FullRemote.mjs'
 import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
+import { Throttle } from '@vates/generator-toolbox'
 
 const noop = Function.prototype
 
@@ -41,7 +41,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
     const schedule = this._schedule
     const settings = this._settings
 
-    const throttleStream = createStreamThrottle(settings.maxExportRate)
+    const throttleGenerator = new Throttle()
 
     const config = this._config
 
@@ -89,7 +89,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
             schedule,
             settings: vmSettings,
             sourceRemoteAdapter,
-            throttleStream,
+            throttleGenerator,
             vmUuid,
           }
           let vmBackup

package/_runners/VmsXapi.mjs

@@ -4,12 +4,12 @@ import { limitConcurrency } from 'limit-concurrency-decorator'
 
 import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
 import { Task } from '../Task.mjs'
-import createStreamThrottle from './_createStreamThrottle.mjs'
 import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
 import { runTask } from './_runTask.mjs'
 import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
 import { FullXapi } from './_vmRunners/FullXapi.mjs'
+import { Throttle } from '@vates/generator-toolbox'
 
 const noop = Function.prototype
 
@@ -55,7 +55,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
     const schedule = this._schedule
     const settings = this._settings
 
-    const throttleStream = createStreamThrottle(settings.maxExportRate)
+    const throttleGenerator = new Throttle()
 
     const config = this._config
 
@@ -147,7 +147,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
                       schedule,
                       settings: vmSettings,
                       srs,
-                      throttleStream,
+                      throttleGenerator,
                       vm,
                     }
 

package/_runners/_vmRunners/FullRemote.mjs

@@ -26,6 +26,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
             stream: forkStreamUnpipe(stream),
             // stream will be forked and transformed, it's not safe to attach additionnal properties to it
             streamLength: stream.length,
+            maxStreamLength: stream.maxStreamLength, // on encrypted source
             timestamp: metadata.timestamp,
             vm: metadata.vm,
             vmSnapshot: metadata.vmSnapshot,

package/_runners/_vmRunners/FullXapi.mjs

@@ -29,14 +29,16 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
     const { compression } = this.job
     const vm = this._vm
     const exportedVm = this._exportedVm
-    const stream = this._throttleStream(
+    // @todo put back throttle for full backup/Replication
+    const stream =
+      /* this._throttleStream( */
       (
         await this._xapi.VM_export(exportedVm.$ref, {
           compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
           useSnapshot: false,
         })
       ).body
-    )
+    /* ) */
 
     const vdis = await exportedVm.$getDisks()
     let maxStreamLength = 1024 * 1024 // Ovf file and tar headers are a few KB, let's stay safe

package/_runners/_vmRunners/IncrementalRemote.mjs

@@ -3,11 +3,8 @@ import { createLogger } from '@xen-orchestra/log'
 import { asyncEach } from '@vates/async-each'
 import assert from 'node:assert'
 import * as UUID from 'uuid'
-import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
-import mapValues from 'lodash/mapValues.js'
 
 import { AbstractRemote } from './_AbstractRemote.mjs'
-import { forkDeltaExport } from './_forkDeltaExport.mjs'
 import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
 import { Disposable } from 'promise-toolbox'
 import { openVhd } from 'vhd-lib'
@@ -37,7 +34,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
       return
     }
     await asyncEach(Object.entries(metadata.vdis), async ([id, vdi]) => {
-      const isDifferencing = metadata.isVhdDifferencing[`${id}.vhd`]
+      const isDifferencing = metadata.isVhdDifferencing[id]
       if (isDifferencing) {
         const vmDir = getVmBackupDir(metadata.vm.uuid)
         const path = `${vmDir}/${metadata.vhds[id]}`
@@ -71,8 +68,8 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
       // recompute if disks are differencing or not
       const isVhdDifferencing = {}
 
-      await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
-        isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
+      Object.entries(incrementalExport.disks).forEach(([key, disk]) => {
+        isVhdDifferencing[key] = disk.isDifferencing()
       })
       const hasDifferencingDisk = Object.values(isVhdDifferencing).includes(true)
       if (metadata.isBase === hasDifferencingDisk) {
@@ -87,11 +84,10 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
       await this._selectBaseVm(metadata)
       await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
 
-      incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
       await this._callWriters(
         writer =>
           writer.transfer({
-            deltaExport: forkDeltaExport(incrementalExport),
+            deltaExport: incrementalExport,
             isVhdDifferencing,
             timestamp: metadata.timestamp,
             vm: metadata.vm,

package/_runners/_vmRunners/IncrementalXapi.mjs

@@ -1,18 +1,11 @@
-import { asyncEach } from '@vates/async-each'
 import { createLogger } from '@xen-orchestra/log'
-import { pipeline } from 'node:stream'
-import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
 import keyBy from 'lodash/keyBy.js'
-import mapValues from 'lodash/mapValues.js'
-import vhdStreamValidator from 'vhd-lib/vhdStreamValidator.js'
 
 import { AbstractXapi } from './_AbstractXapi.mjs'
 import { exportIncrementalVm } from '../../_incrementalVm.mjs'
-import { forkDeltaExport } from './_forkDeltaExport.mjs'
 import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
 import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
 import { Task } from '../../Task.mjs'
-import { watchStreamSize } from '../../_watchStreamSize.mjs'
 import {
   DATETIME,
   DELTA_CHAIN_LENGTH,
@@ -20,9 +13,9 @@ import {
   setVmDeltaChainLength,
   markExportSuccessfull,
 } from '../../_otherConfig.mjs'
+import { SynchronizedDisk } from '@xen-orchestra/disk-transform'
 
 const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
-const noop = Function.prototype
 
 export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
   _getWriters() {
@@ -45,42 +38,40 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       nbdConcurrency: this._settings.nbdConcurrency,
       preferNbd: this._settings.preferNbd,
     })
-    // since NBD is network based, if one disk use nbd , all the disk use them
-    // except the suspended VDI
-    if (Object.values(deltaExport.streams).some(({ _nbd }) => _nbd)) {
-      Task.info('Transfer data using NBD')
-    }
 
     const isVhdDifferencing = {}
-    // since isVhdDifferencingDisk is reading and unshifting data in stream
-    // it should be done BEFORE any other stream transform
-    await asyncEach(Object.entries(deltaExport.streams), async ([key, stream]) => {
-      isVhdDifferencing[key] = await isVhdDifferencingDisk(stream)
-    })
-    const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
-
-    if (this._settings.validateVhdStreams) {
-      deltaExport.streams = mapValues(deltaExport.streams, stream => pipeline(stream, vhdStreamValidator, noop))
+    let useNbd = false
+    for (const key in deltaExport.disks) {
+      const disk = deltaExport.disks[key]
+      isVhdDifferencing[key] = disk.isDifferencing()
+      deltaExport.disks[key] = new SynchronizedDisk(disk)
+      useNbd = useNbd || disk.useNbd()
+    }
+    if (useNbd) {
+      Task.info('Transfer data using NBD')
+    }
+    function fork(deltaExport, label) {
+      const { disks, ...forked } = deltaExport
+      forked.disks = {}
+      for (const key in disks) {
+        forked.disks[key] = disks[key].fork(label)
+      }
+      return forked
     }
-    deltaExport.streams = mapValues(deltaExport.streams, this._throttleStream)
 
+    // @todo : reimplement throttle,nbsource: d use
     const timestamp = Date.now()
-
     await this._callWriters(
       writer =>
         writer.transfer({
-          deltaExport: forkDeltaExport(deltaExport),
+          deltaExport: fork(deltaExport, writer.constructor.name + ' ' + Math.random()),
           isVhdDifferencing,
-          sizeContainers,
           timestamp,
           vm,
           vmSnapshot: exportedVm,
         }),
       'writer.transfer()'
     )
-
-    // we want to control the uuid of the vhd in the chain
-    // and ensure they are correctly chained
     await this._callWriters(
       writer =>
         writer.updateUuidAndChain({
@@ -102,15 +93,6 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
       await markExportSuccessfull(this._xapi, exportedVm.$ref)
     }
 
-    const size = Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0)
-    const end = Date.now()
-    const duration = end - timestamp
-    debug('transfer complete', {
-      duration,
-      speed: duration !== 0 ? (size * 1e3) / 1024 / 1024 / duration : 0,
-      size,
-    })
-
     await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
   }
 

package/_runners/_vmRunners/_AbstractRemote.mjs

@@ -21,7 +21,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
     schedule,
     settings,
     sourceRemoteAdapter,
-    throttleStream,
+    throttleGenerator,
     vmUuid,
   }) {
     super()
@@ -34,7 +34,7 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
 
     this._healthCheckSr = healthCheckSr
     this._sourceRemoteAdapter = sourceRemoteAdapter
-    this._throttleStream = throttleStream
+    this._throttleGenerator = throttleGenerator
     this._vmUuid = vmUuid
 
     const allSettings = job.settings

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -28,7 +28,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     schedule,
     settings,
     srs,
-    throttleStream,
+    throttleGenerator,
     vm,
   }) {
     super()
@@ -62,7 +62,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     this._healthCheckSr = healthCheckSr
     this._jobId = job.id
     this._jobSnapshotVdis = undefined
-    this._throttleStream = throttleStream
+    this._throttleGenerator = throttleGenerator
     this._xapi = vm.$xapi
 
     // Base VM for the export

package/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -136,7 +136,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     const { handler } = this._adapter
     const vhds = this.#vhds
     await asyncEach(Object.entries(vdis), async ([id, vdi]) => {
-      const isDifferencing = isVhdDifferencing[`${id}.vhd`]
+      const isDifferencing = isVhdDifferencing[id]
       const path = `${this._vmBackupDir}/${vhds[id]}`
       if (isDifferencing) {
         assert.notStrictEqual(
@@ -203,8 +203,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
     let metadataContent = await this._isAlreadyTransferred(timestamp)
     if (metadataContent !== undefined) {
       // skip backup while being vigilant to not stuck the forked stream
-      Task.info('This backup has already been transfered')
-      Object.values(deltaExport.streams).forEach(stream => stream.destroy())
+      /** @todo destroy fork  */
       return { size: 0 }
     }
 
@@ -223,36 +222,33 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
       vmSnapshot,
       vtpms: deltaExport.vtpms,
     }
-
-    const { size } = await Task.run({ name: 'transfer' }, async () => {
-      let transferSize = 0
+    let size = 0
+    await Task.run({ name: 'transfer' }, async () => {
       await asyncEach(
-        Object.keys(deltaExport.vdis),
-        async id => {
-          const path = `${this._vmBackupDir}/${vhds[id]}`
-          // don't write it as transferSize += await async function
-          // since i += await asyncFun lead to race condition
-          // as explained : https://eslint.org/docs/latest/rules/require-atomic-updates
-          const transferSizeOneDisk = await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
+        Object.entries(deltaExport.disks),
+        async ([diskRef, disk]) => {
+          const path = `${this._vmBackupDir}/${vhds[diskRef]}`
+          await adapter.writeVhd(path, disk, {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
             writeBlockConcurrency: this._config.writeBlockConcurrency,
           })
-          transferSize += transferSizeOneDisk
+          size = size + disk.getNbGeneratedBlock() * disk.getBlockSize()
         },
         {
           concurrency: settings.diskPerVmConcurrency,
         }
       )
 
-      return { size: transferSize }
+      return { size }
     })
-    metadataContent.size = size
+    metadataContent.size = size // @todo return exactly the size written by this writer
     this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
 
     // TODO: run cleanup?
+    return { size }
   }
 }
 decorateClass(IncrementalRemoteWriter, {

package/_runners/_writers/IncrementalXapiWriter.mjs

@@ -130,7 +130,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
     return backup
   }
 
-  async _transfer({ timestamp, deltaExport, sizeContainers, vm }) {
+  async _transfer({ timestamp, deltaExport, vm }) {
     const { _warmMigration } = this._settings
     const sr = this._sr
     const job = this._job
@@ -141,8 +141,12 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
     let targetVmRef
     await Task.run({ name: 'transfer' }, async () => {
       targetVmRef = await importIncrementalVm(this.#decorateVmMetadata(deltaExport), sr)
+      // size is mandatory to ensure the task have the right data
       return {
-        size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0),
+        size: Object.values(deltaExport.disks).reduce(
+          (sum, disk) => sum + disk.getNbGeneratedBlock() * disk.getBlockSize(),
+          0
+        ),
       }
     })
     this._targetVmRef = targetVmRef

package/_runners/_writers/_AbstractIncrementalWriter.mjs

@@ -20,10 +20,14 @@ export class AbstractIncrementalWriter extends AbstractWriter {
   async transfer({ deltaExport, ...other }) {
     try {
       return await this._transfer({ deltaExport, ...other })
+    } catch (err) {
+      console.error({ err })
     } finally {
-      // ensure all streams are properly closed
-      for (const stream of Object.values(deltaExport.streams)) {
-        stream.destroy()
+      // ensure all sources are properly closed
+      for (const disk of Object.values(deltaExport.disks)) {
+        try {
+          await disk.close()
+        } catch (err) {}
       }
     }
   }

package/disks/RemoteVhd.mjs

@@ -0,0 +1,177 @@
+// @ts-check
+
+/**
+ * @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
+ * @typedef {import('@xen-orchestra/disk-transform').DiskBlock} DiskBlock
+ * @typedef {import('@xen-orchestra/disk-transform').Disk} Disk
+ * @typedef {import('vhd-lib/Vhd/VhdDirectory.js').VhdDirectory} VhdDirectory
+ * @typedef {import('vhd-lib/Vhd/VhdFile.js').VhdFile} VhdFile
+ */
+
+import { openVhd } from 'vhd-lib'
+import { DISK_TYPES } from 'vhd-lib/_constants.js'
+import { dirname, join } from 'node:path'
+import { RandomAccessDisk } from '@xen-orchestra/disk-transform'
+/**
+ * Represents a remote VHD (Virtual Hard Disk) that extends RandomAccessDisk.
+ */
+export class RemoteVhd extends RandomAccessDisk {
+  /**
+   * @type {string}
+   */
+  #path
+
+  /**
+   * @type {FileAccessor}
+   */
+  #handler
+
+  /**
+   * @type {VhdFile | VhdDirectory | undefined}
+   */
+  #vhd
+
+  /**
+   * @type {boolean | undefined}
+   */
+  #isDifferencing
+
+  /**
+   * @type {() => any}
+   */
+  #dispose = () => {}
+
+  /**
+   * @returns {string}
+   */
+  get path() {
+    return this.#path
+  }
+
+  /**
+   * @param {Object} params
+   * @param {FileAccessor} params.handler
+   * @param {string} params.path
+   */
+  constructor({ handler, path }) {
+    super()
+    // @todo : ensure this is the full path from the root of the remote
+    this.#path = path
+    this.#handler = handler
+  }
+
+  /**
+   * @returns {number}
+   */
+  getVirtualSize() {
+    if (this.#vhd === undefined) {
+      throw new Error(`can't call getvirtualsize of a RemoteVhd before init`)
+    }
+    return this.#vhd.footer.currentSize
+  }
+
+  /**
+   * @returns {number}
+   */
+  getBlockSize() {
+    return 2 * 1024 * 1024
+  }
+
+  /**
+   * Initializes the VHD.
+   * @returns {Promise<void>}
+   */
+  async init() {
+    const { value, dispose } = await openVhd(this.#handler, this.#path)
+    this.#vhd = value
+    this.#dispose = dispose
+    await this.#vhd.readBlockAllocationTable()
+    this.#isDifferencing = value.footer.diskType === DISK_TYPES.DIFFERENCING
+  }
+
+  /**
+   * Closes the VHD.
+   * @returns {Promise<void>}
+   */
+  async close() {
+    try {
+      await this.#dispose()
+    } catch (err) {
+      if (err.code !== 'EBADF') {
+        throw err // handle double dispose
+      }
+    }
+  }
+
+  /**
+   * Checks if the VHD contains a specific block.
+   * @param {number} index
+   * @returns {boolean}
+   */
+  hasBlock(index) {
+    if (this.#vhd === undefined) {
+      throw new Error(`can't call hasblock of a RemoteVhd before init`)
+    }
+    return this.#vhd.containsBlock(index)
+  }
+
+  /**
+   * Gets the indexes of all blocks in the VHD.
+   * @returns {Array<number>}
+   */
+  getBlockIndexes() {
+    if (this.#vhd === undefined) {
+      throw new Error(`can't call getBlockIndexes of a RemoteVhd before init`)
+    }
+    const index = []
+    for (let blockIndex = 0; blockIndex < this.#vhd.header.maxTableEntries; blockIndex++) {
+      if (this.hasBlock(blockIndex)) {
+        index.push(blockIndex)
+      }
+    }
+    return index
+  }
+
+  /**
+   * Reads a specific block from the VHD.
+   * @param {number} index
+   * @returns {Promise<DiskBlock>}
+   */
+  async readBlock(index) {
+    if (this.#vhd === undefined) {
+      throw new Error(`can't call readBlock of a RemoteVhd before init`)
+    }
+    const { data } = await this.#vhd.readBlock(index)
+    return {
+      index,
+      data,
+    }
+  }
+  /**
+   *
+   * @returns {RandomAccessDisk}
+   */
+  instantiateParent() {
+    if (this.#vhd === undefined) {
+      throw new Error(`can't call openParent of a RemoteVhd before init`)
+    }
+    const parentPath = this.#vhd.header.parentUnicodeName
+    const fullParentPath = join(dirname(this.#path), parentPath)
+    if (!parentPath) {
+      throw new Error(`Disk ${this.#path} doesn't have parents`)
+    }
+    const parent = new RemoteVhd({ handler: this.#handler, path: fullParentPath })
+    return parent
+  }
+
+  /**
+   * Checks if the VHD is a differencing disk.
+   * @returns {boolean}
+   */
+  isDifferencing() {
+    if (this.#isDifferencing === undefined) {
+      throw new Error(`can't call isDifferencing of a RemoteVhd before init`)
+    }
+    return this.#isDifferencing
+  }
+}

package/disks/openDiskChain.mjs

@@ -0,0 +1,35 @@
+// @ts-check
+/**
+ *
+ * @typedef {import('../../disk-transform/src/FileAccessor.mjs').FileAccessor} FileAccessor
+ */
+import { DiskChain } from '@xen-orchestra/disk-transform'
+import { RemoteVhd } from './RemoteVhd.mjs'
+
+import { defer } from 'golike-defer'
+/**
+ * @param {Object} params
+ * @param {FileAccessor} params.handler
+ * @param {string} params.path
+ * @param {string | undefined} params.until
+ */
+async function _openDiskChain($defer, { handler, path, until }) {
+  let disk
+  const disks = []
+  $defer.onFailure(() => Promise.all(disks.map(disk => disk.close())))
+  disk = new RemoteVhd({ handler, path })
+
+  await disk.init()
+  disks.push(disk)
+  while (disk.isDifferencing()) {
+    disk = await disk.openParent()
+    if (disk.path === until) {
+      break
+    }
+    disks.unshift(disk)
+  }
+  // the root disk
+  return new DiskChain({ disks })
+}
+
+export const openDiskChain = defer(_openDiskChain)

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.58.4",
+  "version": "0.60.0",
   "engines": {
     "node": ">=14.18"
   },
@@ -25,10 +25,12 @@
     "@vates/decorate-with": "^2.1.0",
     "@vates/disposable": "^0.1.6",
     "@vates/fuse-vhd": "^2.1.2",
-    "@vates/nbd-client": "^3.1.2",
+    "@vates/generator-toolbox": "^1.0.2",
+    "@vates/nbd-client": "^3.1.3",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^4.4.1",
+    "@xen-orchestra/disk-transform": "^1.0.0",
+    "@xen-orchestra/fs": "^4.5.1",
     "@xen-orchestra/log": "^0.7.1",
     "@xen-orchestra/template": "^0.1.0",
     "app-conf": "^3.0.0",
@@ -59,7 +61,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^8.1.1"
+    "@xen-orchestra/xapi": "^8.2.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/_runners/_vmRunners/_forkDeltaExport.mjs

@@ -1,11 +0,0 @@
-import cloneDeep from 'lodash/cloneDeep.js'
-import mapValues from 'lodash/mapValues.js'
-
-import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
-
-export function forkDeltaExport(deltaExport) {
-  const { streams, ...rest } = deltaExport
-  const newMetadata = cloneDeep(rest)
-  newMetadata.streams = mapValues(streams, forkStreamUnpipe)
-  return newMetadata
-}