@xen-orchestra/backups 0.68.2 → 0.69.0

package/RemoteAdapter.mjs

@@ -31,7 +31,7 @@ import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from '
 import { lvs, pvs } from './_lvm.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
 
-import { RemoteVhd } from './disks/RemoteVhd.mjs'
+import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
 import { openDiskChain } from './disks/openDiskChain.mjs'
 import { toVhdStream, writeToVhdDirectory } from 'vhd-lib/disk-consumer/index.mjs'
 import { ReadAhead } from '@xen-orchestra/disk-transform'
@@ -44,7 +44,7 @@ const IMMUTABILITY_METADATA_FILENAME = '/immutability.json'
 
 const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
 
-const compareTimestamp = (a, b) => a.timestamp - b.timestamp
+export const compareTimestamp = (a, b) => a.timestamp - b.timestamp
 
 const noop = Function.prototype
 
@@ -744,7 +744,7 @@ export class RemoteAdapter {
     if (useChain) {
       disk = await openDiskChain({ handler, path })
     } else {
-      disk = new RemoteVhd({ handler, path })
+      disk = new RemoteVhdDisk({ handler, path })
       await disk.init()
     }
     disk = new ReadAhead(disk)

package/_cleanVm.mjs

@@ -6,7 +6,9 @@ import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
 import { basename, dirname, resolve } from 'node:path'
 import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
 import { limitConcurrency } from 'limit-concurrency-decorator'
-import { mergeVhdChain } from 'vhd-lib/merge.js'
+import { RemoteVhdDisk } from './disks/RemoteVhdDisk.mjs'
+import { RemoteVhdDiskChain } from './disks/RemoteVhdDiskChain.mjs'
+import { MergeRemoteDisk } from './disks/MergeRemoteDisk.mjs'
 
 import { Task } from './Task.mjs'
 import { Disposable } from 'promise-toolbox'
@@ -51,7 +53,15 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
     }
   }, 10e3)
   try {
-    return await mergeVhdChain(handler, chain, {
+    const parentDisk = new RemoteVhdDisk({ handler, path: chain.shift() })
+
+    const childDisks = []
+    for (const path of chain) {
+      childDisks.push(new RemoteVhdDisk({ handler, path }))
+    }
+    const childDiskChain = new RemoteVhdDiskChain({ disks: childDisks })
+
+    const mergeRemoteDisk = new MergeRemoteDisk(handler, {
       logInfo,
       mergeBlockConcurrency,
       onProgress({ done: d, total: t }) {
@@ -60,6 +70,14 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
       },
       removeUnused: remove,
     })
+
+    const isResumingMerge = await mergeRemoteDisk.isResuming(parentDisk)
+    await parentDisk.init({ force: isResumingMerge })
+    await childDiskChain.init({ force: isResumingMerge })
+
+    const result = await mergeRemoteDisk.merge(parentDisk, childDiskChain)
+
+    return result
   } finally {
     clearInterval(handle)
   }
@@ -521,14 +539,14 @@ export async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const { finalVhdSize } = await limitedMergeVhdChain(handler, chain, {
+      const { finalDiskSize } = await limitedMergeVhdChain(handler, chain, {
         logInfo,
         logWarn,
         remove,
         mergeBlockConcurrency,
       })
       const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metadata file
-      metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalVhdSize
+      metadataWithMergedVhd[metadataPath] = (metadataWithMergedVhd[metadataPath] ?? 0) + finalDiskSize
     })
   }
 

package/_runners/_vmRunners/FullRemote.mjs

@@ -2,10 +2,11 @@ import { AbstractRemote } from './_AbstractRemote.mjs'
 import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
 import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
 import { watchStreamSize } from '../../_watchStreamSize.mjs'
+import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
 
 export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
-  _getRemoteWriter() {
-    return FullRemoteWriter
+  _getRemoteWriters() {
+    return [FullRemoteWriter, AggregatedFullRemoteWriter]
   }
 
   _filterTransferList(transferList) {

package/_runners/_vmRunners/FullXapi.mjs

@@ -5,12 +5,14 @@ import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
 import { FullXapiWriter } from '../_writers/FullXapiWriter.mjs'
 import { watchStreamSize } from '../../_watchStreamSize.mjs'
 import { AbstractXapi } from './_AbstractXapi.mjs'
+import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
+import { AggregatedFullXapiWriter } from '../_writers/AggregatedFullXapiWriter.mjs'
 
 const { debug } = createLogger('xo:backups:FullXapiVmBackup')
 
 export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
   _getWriters() {
-    return [FullRemoteWriter, FullXapiWriter]
+    return [FullRemoteWriter, FullXapiWriter, AggregatedFullRemoteWriter, AggregatedFullXapiWriter]
   }
 
   async _mustDoSnapshot() {

package/_runners/_vmRunners/IncrementalRemote.mjs

@@ -11,11 +11,12 @@ import { Disposable } from 'promise-toolbox'
 import { openVhd } from 'vhd-lib'
 import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
 import { SynchronizedDisk, ThrottledDisk } from '@xen-orchestra/disk-transform'
+import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
 
 const { warn } = createLogger('xo:backups:Incrementalremote')
 class IncrementalRemoteVmBackupRunner extends AbstractRemote {
-  _getRemoteWriter() {
-    return IncrementalRemoteWriter
+  _getRemoteWriters() {
+    return [IncrementalRemoteWriter, AggregatedIncrementalRemoteWriter]
   }
 
   // we'll transfer the full list if at least one backup should be transferred

package/_runners/_vmRunners/IncrementalXapi.mjs

@@ -15,12 +15,19 @@ import {
   markExportSuccessfull,
 } from '../../_otherConfig.mjs'
 import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
+import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
+import { AggregatedIncrementalXapiWriter } from '../_writers/AggregatedIncrementalXapiWriter.mjs'
 
 const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
 
 export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
   _getWriters() {
-    return [IncrementalRemoteWriter, IncrementalXapiWriter]
+    return [
+      IncrementalRemoteWriter,
+      IncrementalXapiWriter,
+      AggregatedIncrementalRemoteWriter,
+      AggregatedIncrementalXapiWriter,
+    ]
   }
 
   async _mustDoSnapshot() {

package/_runners/_vmRunners/_AbstractRemote.mjs

@@ -45,26 +45,41 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
     const writers = new Set()
     this._writers = writers
 
-    const RemoteWriter = this._getRemoteWriter()
-    extractIdsFromSimplePattern(job.remotes).forEach(remoteId => {
-      const adapter = remoteAdapters[remoteId]
-      const targetSettings = {
-        ...settings,
-        ...allSettings[remoteId],
-      }
+    const [BackupWriter, AggregratedBackupWriter] = this._getRemoteWriters()
+    if (settings.distributeBackups) {
       writers.add(
-        new RemoteWriter({
-          adapter,
+        new AggregratedBackupWriter({
+          adapters: remoteAdapters,
+          BackupWriter,
           config,
           healthCheckSr,
           job,
           scheduleId: schedule.id,
           vmUuid,
-          remoteId,
-          settings: targetSettings,
+          settings,
         })
       )
-    })
+    } else {
+      extractIdsFromSimplePattern(job.remotes).forEach(remoteId => {
+        const adapter = remoteAdapters[remoteId]
+        const targetSettings = {
+          ...settings,
+          ...allSettings[remoteId],
+        }
+        writers.add(
+          new BackupWriter({
+            adapter,
+            config,
+            healthCheckSr,
+            job,
+            scheduleId: schedule.id,
+            vmUuid,
+            remoteId,
+            settings: targetSettings,
+          })
+        )
+      })
+    }
     const { filter } = job
     if (filter === undefined) {
       this._filterPredicate = () => true

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -82,48 +82,84 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
       const writers = new Set()
       this._writers = writers
 
-      const [BackupWriter, ReplicationWriter] = this._getWriters()
+      const [BackupWriter, ReplicationWriter, AggregratedBackupWriter, AggregratedReplicationWriter] =
+        this._getWriters()
 
       const allSettings = job.settings
-      Object.entries(remoteAdapters).forEach(([remoteId, adapter]) => {
-        const targetSettings = {
-          ...settings,
-          ...allSettings[remoteId],
-        }
-        if (targetSettings.exportRetention !== 0) {
+
+      if (settings.exportRetention > 0) {
+        if (settings.distributeBackups) {
           writers.add(
-            new BackupWriter({
-              adapter,
+            new AggregratedBackupWriter({
+              adapters: remoteAdapters,
+              BackupWriter,
               config,
               healthCheckSr,
               job,
               scheduleId: schedule.id,
               vmUuid: vm.uuid,
-              remoteId,
-              settings: targetSettings,
+              settings,
             })
           )
+        } else {
+          Object.entries(remoteAdapters).forEach(([remoteId, adapter]) => {
+            const targetSettings = {
+              ...settings,
+              ...allSettings[remoteId],
+            }
+            if (targetSettings.exportRetention !== 0) {
+              writers.add(
+                new BackupWriter({
+                  adapter,
+                  config,
+                  healthCheckSr,
+                  job,
+                  scheduleId: schedule.id,
+                  vmUuid: vm.uuid,
+                  remoteId,
+                  settings: targetSettings,
+                })
+              )
+            }
+          })
         }
-      })
-      srs.forEach(sr => {
-        const targetSettings = {
-          ...settings,
-          ...allSettings[sr.uuid],
-        }
-        if (targetSettings.copyRetention !== 0) {
+      }
+      if (settings.copyRetention) {
+        if (settings.distributeReplications) {
           writers.add(
-            new ReplicationWriter({
+            new AggregratedReplicationWriter({
               config,
               healthCheckSr,
               job,
+              ReplicationWriter,
               scheduleId: schedule.id,
               vmUuid: vm.uuid,
-              sr,
-              settings: targetSettings,
+              srs,
+              settings,
             })
           )
+        } else {
+          srs.forEach(sr => {
+            const targetSettings = {
+              ...settings,
+              ...allSettings[sr.uuid],
+            }
+            if (targetSettings.copyRetention !== 0) {
+              writers.add(
+                new ReplicationWriter({
+                  config,
+                  healthCheckSr,
+                  job,
+                  scheduleId: schedule.id,
+                  vmUuid: vm.uuid,
+                  sr,
+                  settings: targetSettings,
+                })
+              )
+            }
+          })
         }
-      })
+      }
     }
   }
 

package/_runners/_writers/AggregatedFullRemoteWriter.mjs

@@ -0,0 +1,38 @@
+import { AbstractAggregatedRemoteWriter } from './_AbstractAggregatedRemoteWriter.mjs'
+
+export class AggregatedFullRemoteWriter extends AbstractAggregatedRemoteWriter {
+  async deleteOldBackupsOnAdapter(adapter, backups) {
+    await adapter.deleteFullVmBackups(backups)
+  }
+
+  // take the lock on every adapters with this VM
+  // cleanup if deleteFirst is selected
+  async beforeBackup() {
+    await this.setupWriters()
+    await Promise.all(this.writers.map(writer => writer.beforeBackup()))
+
+    await this.setOldBackups()
+    if (this.props.settings.deleteFirst) {
+      await this.deleteOldBackups()
+    }
+  }
+
+  async run(props) {
+    const mainAdapter = await this.computeAdapterCandidate()
+    this.mainWriter = this.getWriterByAdapter(mainAdapter)
+    return this.mainWriter.run(props)
+  }
+
+  // cleanup if deleteFirst is NOT selected
+  // release  the lock on every adapters with this VM
+  async afterBackup() {
+    if (!this.props.settings.deleteFirst) {
+      await this.deleteOldBackups()
+    }
+    await Promise.all(this.writers.map(writer => writer.afterBackup()))
+  }
+
+  healthCheck() {
+    return this.mainWriter.healthCheck()
+  }
+}

package/_runners/_writers/AggregatedFullXapiWriter.mjs

@@ -0,0 +1,31 @@
+import { AbstractAggregatedXapiWriter } from './_AbstractAggregatedXapiWriter.mjs'
+
+export class AggregatedFullXapiWriter extends AbstractAggregatedXapiWriter {
+  // delete replica if deleteFirst is selected
+  async beforeBackup() {
+    await this.setupWriters()
+    await Promise.all(this.writers.map(writer => writer.beforeBackup()))
+    this.setOldReplicaList()
+    if (this.props.settings.deleteFirst) {
+      await this.deleteOldReplicas()
+    }
+  }
+
+  async run(props) {
+    const mainSr = await this.computeSRCandidate()
+    this.mainWriter = this.getWriterBySr(mainSr)
+    await this.mainWriter.run(props)
+  }
+
+  // delete replica if not using deleteFirst
+  async afterBackup() {
+    if (!this.props.settings.deleteFirst) {
+      await this.deleteOldReplicas()
+    }
+    await Promise.all(this.writers.map(writer => writer.afterBackup()))
+  }
+
+  healthCheck() {
+    return this.mainWriter.healthCheck()
+  }
+}

package/_runners/_writers/AggregatedIncrementalRemoteWriter.mjs

@@ -0,0 +1,78 @@
+import { AbstractAggregatedRemoteWriter } from './_AbstractAggregatedRemoteWriter.mjs'
+export class AggregatedIncrementalRemoteWriter extends AbstractAggregatedRemoteWriter {
+  async deleteOldBackupsOnAdapter(adapter, backups) {
+    await adapter.deleteDeltaVmBackups(backups)
+  }
+
+  /**
+   *
+   * @param {Map<string,string>} baseUuidToSrcVdi
+   */
+  async checkBaseVdis(baseUuidToSrcVdi) {
+    let selectedCopy = new Map()
+    for (const writer of this.writers) {
+      const copy = new Map(baseUuidToSrcVdi)
+      await writer.checkBaseVdis(copy)
+      if (copy.size > 0) {
+        // there can be multiple candidates
+        // we use the last one in this case
+        this.mainWriter = writer
+        selectedCopy = copy
+      }
+    }
+    // update the parameter with our best candidate
+    // that can be an empty map
+    baseUuidToSrcVdi.forEach(key => {
+      if (!selectedCopy.has(key)) {
+        baseUuidToSrcVdi.delete(key)
+      }
+    })
+  }
+
+  // take the lock on every adapters with this VM
+  // cleanup if deleteFirst is selected
+  async beforeBackup() {
+    await this.setupWriters()
+    await Promise.all(this.writers.map(writer => writer.beforeBackup()))
+
+    await this.setOldBackups()
+    if (this.props.settings.deleteFirst) {
+      await this.deleteOldBackups()
+    }
+  }
+
+  async prepare({ isFull }) {
+    if (this.mainWriter === undefined) {
+      // use the remote with the most empty space for starting a new chain
+      const mainAdapter = await this.computeAdapterCandidate()
+      this.mainWriter = this.getWriterByAdapter(mainAdapter)
+    }
+    await Promise.all(this.writers.map(writer => writer.prepare({ isFull })))
+  }
+  // write only on the main writer
+  transfer({ isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot }) {
+    return this.mainWriter.transfer({ isVhdDifferencing, timestamp, deltaExport, vm, vmSnapshot })
+  }
+
+  // chain only on the main writer
+  updateUuidAndChain({ isVhdDifferencing, vdis }) {
+    return this.mainWriter.updateUuidAndChain({ isVhdDifferencing, vdis })
+  }
+
+  // remove the backups and remove the entries
+  async cleanup() {
+    if (!this.props.settings.deleteFirst) {
+      await this.deleteOldBackups()
+    }
+    await Promise.all(this.writers.map(writer => writer.cleanup()))
+  }
+  // cleanVM on all, since we may want to delete a chani
+  // that is not on the main writer
+  async afterBackup() {
+    await Promise.all(this.writers.map(writer => writer.afterBackup()))
+  }
+
+  healthCheck() {
+    return this.mainWriter.healthCheck()
+  }
+}

package/_runners/_writers/AggregatedIncrementalXapiWriter.mjs

@@ -0,0 +1,85 @@
+import { AbstractAggregatedXapiWriter } from './_AbstractAggregatedXapiWriter.mjs'
+import { createLogger } from '@xen-orchestra/log'
+
+const { debug } = createLogger('xo:backups:AggregatedIncrementalXapiWriter')
+export class AggregatedIncrementalXapiWriter extends AbstractAggregatedXapiWriter {
+  /**
+   *
+   * @param {Map<string,string>} baseUuidToSrcVdi
+   */
+  async checkBaseVdis(baseUuidToSrcVdi) {
+    debug('checkBaseVdis', { baseUuidToSrcVdi })
+    let selectedCopy = new Map()
+    for (const writer of this.writers) {
+      const copy = new Map(baseUuidToSrcVdi)
+      await writer.checkBaseVdis(copy)
+      if (copy.size > 0) {
+        debug('checkBaseVdis found a mainwriter candidate ', writer._sr.name_label)
+        // there can be multiple candidates
+        // we use the last one in this case
+        this.mainWriter = writer
+        selectedCopy = copy
+      } else {
+        debug(writer._sr.name_label, 'is not a delta candidate')
+      }
+    }
+    // update the parameter with our best candidate
+    // that can be an empty map
+    baseUuidToSrcVdi.forEach(key => {
+      if (!selectedCopy.has(key)) {
+        baseUuidToSrcVdi.delete(key)
+      }
+    })
+    debug('checkBaseVdis', { baseUuidToSrcVdi })
+  }
+
+  // take the lock on every adapters with this VM
+  // cleanup if deleteFirst is selected
+  async beforeBackup() {
+    await this.setupWriters()
+    await Promise.all(this.writers.map(writer => writer.beforeBackup()))
+    this.setOldReplicaList()
+    if (this.props.settings.deleteFirst) {
+      await this.deleteOldReplicas()
+    }
+  }
+
+  async prepare(args) {
+    if (this.mainWriter === undefined) {
+      debug('no mainwriter found, fallback to a new SR')
+      // use the SR with the most empty space for starting a new chain
+      const mainSr = await this.computeSRCandidate()
+      this.mainWriter = this.getWriterBySr(mainSr)
+    }
+    await Promise.all(this.writers.map(writer => writer.prepare(args)))
+  }
+
+  // write only on the main writer
+  transfer(args) {
+    return this.mainWriter.transfer(args)
+  }
+
+  // chain only on the main writer
+  updateUuidAndChain(args) {
+    return this.mainWriter.updateUuidAndChain(args)
+  }
+
+  // remove the backups and remove the entries
+  async cleanup() {
+    debug('cleanup')
+    if (!this.props.settings.deleteFirst) {
+      await this.deleteOldReplicas()
+    }
+    await Promise.all(this.writers.map(writer => writer.cleanup()))
+  }
+
+  // cleanVM on all, since we may want to delete a chani
+  // that is not on the main writer
+  async afterBackup() {
+    await Promise.all(this.writers.map(writer => writer.afterBackup()))
+  }
+
+  healthCheck() {
+    return this.mainWriter.healthCheck()
+  }
+}

package/_runners/_writers/FullRemoteWriter.mjs

@@ -59,8 +59,8 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
       xva: './' + dataBasename,
     }
 
-    const { deleteFirst } = settings
-    if (deleteFirst) {
+    const { skipDeleteOldEntries, deleteFirst } = settings
+    if (!skipDeleteOldEntries && deleteFirst) {
       await deleteOldBackups()
     }
 
@@ -76,7 +76,7 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
     metadata.tags = await this.getLongTermRetentionTags(metadata)
     this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadata)
 
-    if (!deleteFirst) {
+    if (!skipDeleteOldEntries && !deleteFirst) {
       await deleteOldBackups()
     }
 

package/_runners/_writers/FullXapiWriter.mjs

@@ -46,8 +46,8 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
     const oldVms = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vm.uuid))
 
     const deleteOldBackups = () => asyncMapSettled(oldVms, vm => xapi.VM_destroy(vm.$ref))
-    const { deleteFirst, _warmMigration } = settings
-    if (deleteFirst) {
+    const { deleteFirst, skipDeleteOldEntries, _warmMigration } = settings
+    if (!skipDeleteOldEntries && deleteFirst) {
       await deleteOldBackups()
     }
 
@@ -85,7 +85,7 @@ export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
       }),
     ])
 
-    if (!deleteFirst) {
+    if (!skipDeleteOldEntries && !deleteFirst) {
       await deleteOldBackups()
     }
   }

package/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -114,13 +114,13 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
       oldEntries.length = maxMergedDeltasPerRun
     }
 
-    if (settings.deleteFirst) {
+    if (settings.deleteFirst && !settings.skipDeleteOldEntries) {
       await this._deleteOldEntries()
     }
   }
 
   async cleanup() {
-    if (!this._settings.deleteFirst) {
+    if (!this._settings.deleteFirst && !this._settings.skipDeleteOldEntries) {
       await this._deleteOldEntries()
     }
   }

package/_runners/_writers/IncrementalXapiWriter.mjs

@@ -83,7 +83,7 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
 
     this._oldEntries = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vmUuid))
 
-    if (settings.deleteFirst) {
+    if (settings.deleteFirst && settings.skipDeleteOldEntries) {
       // we want to keep the baseVM when copying a delta
       // even if we want to keep only one after
       let mostRecentEntry
@@ -96,7 +96,9 @@ export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWr
   }
 
   async cleanup() {
-    await this._deleteOldEntries()
+    if (!this._settings.skipDeleteOldEntries) {
+      await this._deleteOldEntries()
+    }
   }
 
   async _deleteOldEntries() {