@xen-orchestra/backups 0.36.1 → 0.38.0

package/_runners/VmsRemote.js

@@ -0,0 +1,98 @@
+'use strict'
+
+const { asyncMapSettled } = require('@xen-orchestra/async-map')
+const Disposable = require('promise-toolbox/Disposable')
+const { limitConcurrency } = require('limit-concurrency-decorator')
+
+const { extractIdsFromSimplePattern } = require('../extractIdsFromSimplePattern.js')
+const { Task } = require('../Task.js')
+const createStreamThrottle = require('./_createStreamThrottle.js')
+const { DEFAULT_SETTINGS, Abstract } = require('./_Abstract.js')
+const { runTask } = require('./_runTask.js')
+const { getAdaptersByRemote } = require('./_getAdaptersByRemote.js')
+const { FullRemote } = require('./_vmRunners/FullRemote.js')
+const { IncrementalRemote } = require('./_vmRunners/IncrementalRemote.js')
+
+const DEFAULT_REMOTE_VM_SETTINGS = {
+  concurrency: 2,
+  copyRetention: 0,
+  deleteFirst: false,
+  exportRetention: 0,
+  healthCheckSr: undefined,
+  healthCheckVmsWithTags: [],
+  maxExportRate: 0,
+  maxMergedDeltasPerRun: Infinity,
+  timeout: 0,
+  validateVhdStreams: false,
+  vmTimeout: 0,
+}
+
+exports.VmsRemote = class RemoteVmsBackupRunner extends Abstract {
+  _computeBaseSettings(config, job) {
+    const baseSettings = { ...DEFAULT_SETTINGS }
+    Object.assign(baseSettings, DEFAULT_REMOTE_VM_SETTINGS, config.defaultSettings, config.vm?.defaultSettings)
+    Object.assign(baseSettings, job.settings[''])
+    return baseSettings
+  }
+
+  async run() {
+    const job = this._job
+    const schedule = this._schedule
+    const settings = this._settings
+
+    const throttleStream = createStreamThrottle(settings.maxExportRate)
+
+    const config = this._config
+    await Disposable.use(
+      () => this._getAdapter(job.sourceRemote),
+      () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
+      Disposable.all(
+        extractIdsFromSimplePattern(job.remotes).map(id => id !== job.sourceRemote && this._getAdapter(id))
+      ),
+      async ({ adapter: sourceRemoteAdapter }, healthCheckSr, remoteAdapters) => {
+        // remove adapters that failed (already handled)
+        remoteAdapters = remoteAdapters.filter(_ => !!_)
+        if (remoteAdapters.length === 0) {
+          return
+        }
+
+        const vmsUuids = await sourceRemoteAdapter.listAllVms()
+
+        Task.info('vms', { vms: vmsUuids })
+
+        remoteAdapters = getAdaptersByRemote(remoteAdapters)
+        const allSettings = this._job.settings
+        const baseSettings = this._baseSettings
+
+        const handleVm = vmUuid => {
+          const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
+
+          const opts = {
+            baseSettings,
+            config,
+            job,
+            healthCheckSr,
+            remoteAdapters,
+            schedule,
+            settings: { ...settings, ...allSettings[vmUuid] },
+            sourceRemoteAdapter,
+            throttleStream,
+            vmUuid,
+          }
+          let vmBackup
+          if (job.mode === 'delta') {
+            vmBackup = new IncrementalRemote(opts)
+          } else if (job.mode === 'full') {
+            vmBackup = new FullRemote(opts)
+          } else {
+            throw new Error(`Job mode ${job.mode} not implemented for mirror backup`)
+          }
+
+          return runTask(taskStart, () => vmBackup.run())
+        }
+        const { concurrency } = settings
+        await asyncMapSettled(vmsUuids, !concurrency ? handleVm : limitConcurrency(concurrency)(handleVm))
+      }
+    )
+  }
+}

package/_runners/VmsXapi.js

@@ -0,0 +1,138 @@
+'use strict'
+
+const { asyncMapSettled } = require('@xen-orchestra/async-map')
+const Disposable = require('promise-toolbox/Disposable')
+const { limitConcurrency } = require('limit-concurrency-decorator')
+
+const { extractIdsFromSimplePattern } = require('../extractIdsFromSimplePattern.js')
+const { Task } = require('../Task.js')
+const createStreamThrottle = require('./_createStreamThrottle.js')
+const { DEFAULT_SETTINGS, Abstract } = require('./_Abstract.js')
+const { runTask } = require('./_runTask.js')
+const { getAdaptersByRemote } = require('./_getAdaptersByRemote.js')
+const { IncrementalXapi } = require('./_vmRunners/IncrementalXapi.js')
+const { FullXapi } = require('./_vmRunners/FullXapi.js')
+
+const DEFAULT_XAPI_VM_SETTINGS = {
+  bypassVdiChainsCheck: false,
+  checkpointSnapshot: false,
+  concurrency: 2,
+  copyRetention: 0,
+  deleteFirst: false,
+  exportRetention: 0,
+  fullInterval: 0,
+  healthCheckSr: undefined,
+  healthCheckVmsWithTags: [],
+  maxExportRate: 0,
+  maxMergedDeltasPerRun: Infinity,
+  offlineBackup: false,
+  offlineSnapshot: false,
+  snapshotRetention: 0,
+  timeout: 0,
+  useNbd: false,
+  unconditionalSnapshot: false,
+  validateVhdStreams: false,
+  vmTimeout: 0,
+}
+
+exports.VmsXapi = class VmsXapiBackupRunner extends Abstract {
+  _computeBaseSettings(config, job) {
+    const baseSettings = { ...DEFAULT_SETTINGS }
+    Object.assign(baseSettings, DEFAULT_XAPI_VM_SETTINGS, config.defaultSettings, config.vm?.defaultSettings)
+    Object.assign(baseSettings, job.settings[''])
+    return baseSettings
+  }
+
+  async run() {
+    const job = this._job
+
+    // FIXME: proper SimpleIdPattern handling
+    const getSnapshotNameLabel = this._getSnapshotNameLabel
+    const schedule = this._schedule
+    const settings = this._settings
+
+    const throttleStream = createStreamThrottle(settings.maxExportRate)
+
+    const config = this._config
+    await Disposable.use(
+      Disposable.all(
+        extractIdsFromSimplePattern(job.srs).map(id =>
+          this._getRecord('SR', id).catch(error => {
+            runTask(
+              {
+                name: 'get SR record',
+                data: { type: 'SR', id },
+              },
+              () => Promise.reject(error)
+            )
+          })
+        )
+      ),
+      Disposable.all(extractIdsFromSimplePattern(job.remotes).map(id => this._getAdapter(id))),
+      () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
+      async (srs, remoteAdapters, healthCheckSr) => {
+        // remove adapters that failed (already handled)
+        remoteAdapters = remoteAdapters.filter(_ => _ !== undefined)
+
+        // remove srs that failed (already handled)
+        srs = srs.filter(_ => _ !== undefined)
+
+        if (remoteAdapters.length === 0 && srs.length === 0 && settings.snapshotRetention === 0) {
+          return
+        }
+
+        const vmIds = extractIdsFromSimplePattern(job.vms)
+
+        Task.info('vms', { vms: vmIds })
+
+        remoteAdapters = getAdaptersByRemote(remoteAdapters)
+
+        const allSettings = this._job.settings
+        const baseSettings = this._baseSettings
+
+        const handleVm = vmUuid => {
+          const taskStart = { name: 'backup VM', data: { type: 'VM', id: vmUuid } }
+
+          return this._getRecord('VM', vmUuid).then(
+            disposableVm =>
+              Disposable.use(disposableVm, vm => {
+                taskStart.data.name_label = vm.name_label
+                return runTask(taskStart, () => {
+                  const opts = {
+                    baseSettings,
+                    config,
+                    getSnapshotNameLabel,
+                    healthCheckSr,
+                    job,
+                    remoteAdapters,
+                    schedule,
+                    settings: { ...settings, ...allSettings[vm.uuid] },
+                    srs,
+                    throttleStream,
+                    vm,
+                  }
+                  let vmBackup
+                  if (job.mode === 'delta') {
+                    vmBackup = new IncrementalXapi(opts)
+                  } else {
+                    if (job.mode === 'full') {
+                      vmBackup = new FullXapi(opts)
+                    } else {
+                      throw new Error(`Job mode ${job.mode} not implemented`)
+                    }
+                  }
+                  return vmBackup.run()
+                })
+              }),
+            error =>
+              runTask(taskStart, () => {
+                throw error
+              })
+          )
+        }
+        const { concurrency } = settings
+        await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
+      }
+    )
+  }
+}

package/_runners/_Abstract.js

@@ -0,0 +1,51 @@
+'use strict'
+
+const Disposable = require('promise-toolbox/Disposable')
+const pTimeout = require('promise-toolbox/timeout')
+const { compileTemplate } = require('@xen-orchestra/template')
+const { runTask } = require('./_runTask.js')
+const { RemoteTimeoutError } = require('./_RemoteTimeoutError.js')
+
+exports.DEFAULT_SETTINGS = {
+  getRemoteTimeout: 300e3,
+  reportWhen: 'failure',
+}
+
+exports.Abstract = class AbstractRunner {
+  constructor({ config, getAdapter, getConnectedRecord, job, schedule }) {
+    this._config = config
+    this._getRecord = getConnectedRecord
+    this._job = job
+    this._schedule = schedule
+
+    this._getSnapshotNameLabel = compileTemplate(config.snapshotNameLabelTpl, {
+      '{job.name}': job.name,
+      '{vm.name_label}': vm => vm.name_label,
+    })
+
+    const baseSettings = this._computeBaseSettings(config, job)
+    this._baseSettings = baseSettings
+    this._settings = { ...baseSettings, ...job.settings[schedule.id] }
+
+    const { getRemoteTimeout } = this._settings
+    this._getAdapter = async function (remoteId) {
+      try {
+        const disposable = await pTimeout.call(getAdapter(remoteId), getRemoteTimeout, new RemoteTimeoutError(remoteId))
+
+        return new Disposable(() => disposable.dispose(), {
+          adapter: disposable.value,
+          remoteId,
+        })
+      } catch (error) {
+        // See https://github.com/vatesfr/xen-orchestra/commit/6aa6cfba8ec939c0288f0fa740f6dfad98c43cbb
+        runTask(
+          {
+            name: 'get remote adapter',
+            data: { type: 'remote', id: remoteId },
+          },
+          () => Promise.reject(error)
+        )
+      }
+    }
+  }
+}

package/{_PoolMetadataBackup.js → _runners/_PoolMetadataBackup.js}

@@ -2,10 +2,10 @@
 
 const { asyncMap } = require('@xen-orchestra/async-map')
 
-const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
+const { DIR_XO_POOL_METADATA_BACKUPS } = require('../RemoteAdapter.js')
 const { forkStreamUnpipe } = require('./_forkStreamUnpipe.js')
-const { formatFilenameDate } = require('./_filenameDate.js')
-const { Task } = require('./Task.js')
+const { formatFilenameDate } = require('../_filenameDate.js')
+const { Task } = require('../Task.js')
 
 const PATH_DB_DUMP = '/pool/xmldbdump'
 exports.PATH_DB_DUMP = PATH_DB_DUMP

package/_runners/_RemoteTimeoutError.js

@@ -0,0 +1,8 @@
+'use strict'
+class RemoteTimeoutError extends Error {
+  constructor(remoteId) {
+    super('timeout while getting the remote ' + remoteId)
+    this.remoteId = remoteId
+  }
+}
+exports.RemoteTimeoutError = RemoteTimeoutError

package/{_XoMetadataBackup.js → _runners/_XoMetadataBackup.js}

@@ -2,9 +2,9 @@
 
 const { asyncMap } = require('@xen-orchestra/async-map')
 
-const { DIR_XO_CONFIG_BACKUPS } = require('./RemoteAdapter.js')
-const { formatFilenameDate } = require('./_filenameDate.js')
-const { Task } = require('./Task.js')
+const { DIR_XO_CONFIG_BACKUPS } = require('../RemoteAdapter.js')
+const { formatFilenameDate } = require('../_filenameDate.js')
+const { Task } = require('../Task.js')
 
 exports.XoMetadataBackup = class XoMetadataBackup {
   constructor({ config, job, remoteAdapters, schedule, settings }) {

package/_runners/_getAdaptersByRemote.js

@@ -0,0 +1,9 @@
+'use strict'
+const getAdaptersByRemote = adapters => {
+  const adaptersByRemote = {}
+  adapters.forEach(({ adapter, remoteId }) => {
+    adaptersByRemote[remoteId] = adapter
+  })
+  return adaptersByRemote
+}
+exports.getAdaptersByRemote = getAdaptersByRemote

package/_runners/_runTask.js

@@ -0,0 +1,6 @@
+'use strict'
+const { Task } = require('../Task.js')
+const noop = Function.prototype
+const runTask = (...args) => Task.run(...args).catch(noop) // errors are handled by logs
+
+exports.runTask = runTask

package/_runners/_vmRunners/FullRemote.js

@@ -0,0 +1,53 @@
+'use strict'
+
+const { decorateMethodsWith } = require('@vates/decorate-with')
+const { defer } = require('golike-defer')
+const { AbstractRemote } = require('./_AbstractRemote')
+const { FullRemoteWriter } = require('../_writers/FullRemoteWriter')
+const { forkStreamUnpipe } = require('../_forkStreamUnpipe')
+const { watchStreamSize } = require('../../_watchStreamSize')
+const { Task } = require('../../Task')
+
+class FullRemoteVmBackupRunner extends AbstractRemote {
+  _getRemoteWriter() {
+    return FullRemoteWriter
+  }
+  async _run($defer) {
+    const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
+
+    await this._callWriters(async writer => {
+      await writer.beforeBackup()
+      $defer(async () => {
+        await writer.afterBackup()
+      })
+    }, 'writer.beforeBackup()')
+    if (transferList.length > 0) {
+      for (const metadata of transferList) {
+        const stream = await this._sourceRemoteAdapter.readFullVmBackup(metadata)
+        const sizeContainer = watchStreamSize(stream)
+
+        // @todo shouldn't transfer backup if it will be deleted by retention policy (higher retention on source than destination)
+        await this._callWriters(
+          writer =>
+            writer.run({
+              stream: forkStreamUnpipe(stream),
+              timestamp: metadata.timestamp,
+              vm: metadata.vm,
+              vmSnapshot: metadata.vmSnapshot,
+              sizeContainer,
+            }),
+          'writer.run()'
+        )
+        // for healthcheck
+        this._tags = metadata.vm.tags
+      }
+    } else {
+      Task.info('No new data to upload for this VM')
+    }
+  }
+}
+
+exports.FullRemote = FullRemoteVmBackupRunner
+decorateMethodsWith(FullRemoteVmBackupRunner, {
+  _run: defer,
+})

package/_runners/_vmRunners/FullXapi.js

@@ -0,0 +1,65 @@
+'use strict'
+
+const { createLogger } = require('@xen-orchestra/log')
+
+const { forkStreamUnpipe } = require('../_forkStreamUnpipe.js')
+const { FullRemoteWriter } = require('../_writers/FullRemoteWriter.js')
+const { FullXapiWriter } = require('../_writers/FullXapiWriter.js')
+const { watchStreamSize } = require('../../_watchStreamSize.js')
+const { AbstractXapi } = require('./_AbstractXapi.js')
+
+const { debug } = createLogger('xo:backups:FullXapiVmBackup')
+
+exports.FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
+  _getWriters() {
+    return [FullRemoteWriter, FullXapiWriter]
+  }
+
+  _mustDoSnapshot() {
+    const vm = this._vm
+
+    const settings = this._settings
+    return (
+      settings.unconditionalSnapshot ||
+      (!settings.offlineBackup && vm.power_state === 'Running') ||
+      settings.snapshotRetention !== 0
+    )
+  }
+  _selectBaseVm() {}
+
+  async _copy() {
+    const { compression } = this.job
+    const vm = this._vm
+    const exportedVm = this._exportedVm
+    const stream = this._throttleStream(
+      await this._xapi.VM_export(exportedVm.$ref, {
+        compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
+        useSnapshot: false,
+      })
+    )
+    const sizeContainer = watchStreamSize(stream)
+
+    const timestamp = Date.now()
+
+    await this._callWriters(
+      writer =>
+        writer.run({
+          sizeContainer,
+          stream: forkStreamUnpipe(stream),
+          timestamp,
+          vm,
+          vmSnapshot: exportedVm,
+        }),
+      'writer.run()'
+    )
+
+    const { size } = sizeContainer
+    const end = Date.now()
+    const duration = end - timestamp
+    debug('transfer complete', {
+      duration,
+      speed: duration !== 0 ? (size * 1e3) / 1024 / 1024 / duration : 0,
+      size,
+    })
+  }
+}

package/_runners/_vmRunners/IncrementalRemote.js

@@ -0,0 +1,67 @@
+'use strict'
+const assert = require('node:assert')
+
+const { decorateMethodsWith } = require('@vates/decorate-with')
+const { defer } = require('golike-defer')
+const { mapValues } = require('lodash')
+const { Task } = require('../../Task')
+const { AbstractRemote } = require('./_AbstractRemote')
+const { IncrementalRemoteWriter } = require('../_writers/IncrementalRemoteWriter')
+const { forkDeltaExport } = require('./_forkDeltaExport')
+const isVhdDifferencingDisk = require('vhd-lib/isVhdDifferencingDisk')
+const { asyncEach } = require('@vates/async-each')
+
+class IncrementalRemoteVmBackupRunner extends AbstractRemote {
+  _getRemoteWriter() {
+    return IncrementalRemoteWriter
+  }
+  async _run($defer) {
+    const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
+    await this._callWriters(async writer => {
+      await writer.beforeBackup()
+      $defer(async () => {
+        await writer.afterBackup()
+      })
+    }, 'writer.beforeBackup()')
+
+    if (transferList.length > 0) {
+      for (const metadata of transferList) {
+        assert.strictEqual(metadata.mode, 'delta')
+
+        await this._callWriters(writer => writer.prepare({ isBase: metadata.isBase }), 'writer.prepare()')
+        const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
+          useChain: false,
+        })
+
+        const differentialVhds = {}
+
+        await asyncEach(Object.entries(incrementalExport.streams), async ([key, stream]) => {
+          differentialVhds[key] = await isVhdDifferencingDisk(stream)
+        })
+
+        incrementalExport.streams = mapValues(incrementalExport.streams, this._throttleStream)
+        await this._callWriters(
+          writer =>
+            writer.transfer({
+              deltaExport: forkDeltaExport(incrementalExport),
+              differentialVhds,
+              timestamp: metadata.timestamp,
+              vm: metadata.vm,
+              vmSnapshot: metadata.vmSnapshot,
+            }),
+          'writer.transfer()'
+        )
+        await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
+        // for healthcheck
+        this._tags = metadata.vm.tags
+      }
+    } else {
+      Task.info('No new data to upload for this VM')
+    }
+  }
+}
+
+exports.IncrementalRemote = IncrementalRemoteVmBackupRunner
+decorateMethodsWith(IncrementalRemoteVmBackupRunner, {
+  _run: defer,
+})