@xen-orchestra/backups 0.68.2 → 0.69.0

package/_runners/_writers/_AbstractAggregatedRemoteWriter.mjs

@@ -0,0 +1,157 @@
+import { asyncEach } from '@vates/async-each'
+import { getOldEntries } from '../../_getOldEntries.mjs'
+import { compareTimestamp } from '../../RemoteAdapter.mjs'
+import { createLogger } from '@xen-orchestra/log'
+
+const { debug } = createLogger('xo:backups:AbstractAggregatedRemoteWriter')
+
+export const DEFAULT_AGGREGATED_REMOTE_DELETE_CONCURRENCY = 2
+export class AbstractAggregatedRemoteWriter {
+  #BackupWriter
+  #adapters
+  get adapters() {
+    return this.#adapters
+  }
+
+  mainWriter
+  #props
+  get props() {
+    return this.#props
+  }
+  #oldBackups
+
+  #writers = []
+  get writers() {
+    return this.#writers
+  }
+  constructor({ adapters, BackupWriter, ...props }) {
+    debug('instantiate AggregatedFullRemoteWriter')
+    if (Object.keys(adapters).length < 1) {
+      throw new Error('AggregatedRemoteWriter need at least one backup repository to work ')
+    }
+    this.#props = props
+    this.#adapters = adapters
+    this.#BackupWriter = BackupWriter
+  }
+
+  getWriterByAdapter(adapter) {
+    for (const writer of this.#writers) {
+      if (writer._adapter === adapter) {
+        return writer
+      }
+    }
+    throw new Error("Can't find writer for adapter")
+  }
+
+  /**
+   * select the best adapter candidate to write this backup
+   *   if all the adapter infos expose an available info use the one with the more free space
+   *   if none of the adapter exposes this info : choose at random
+   *   if mixed : throw an error
+   * @returns {RemoteAdapter}
+   */
+  async computeAdapterCandidate() {
+    const adapters = Object.values(this.#adapters)
+    const infos = await Promise.all(
+      adapters.map(async adapter => {
+        const infos = await adapter._handler.getInfo()
+        return { adapter, infos }
+      })
+    )
+    const withLimit = infos.filter(({ infos: { available } }) => available > 0).length
+    if (withLimit !== 0 && withLimit !== infos.length) {
+      throw new Error(`an Aggregate remote can't use a mix of limited and unlimited storages`)
+    }
+    let candidates
+    if (withLimit === 0) {
+      debug('no storage limit , choose at random')
+      candidates = Object.values(this.adapters)
+    } else {
+      debug('storage limit , choose at random between those with the max space avilable')
+      let maxSpace = 0
+      infos.forEach(({ infos: { available } }) => {
+        if (available > maxSpace) {
+          maxSpace = available
+        }
+      })
+
+      // candidates are the adapter with the same free space , equal to the max free space
+      candidates = infos.filter(({ infos }) => infos.available === maxSpace).map(({ adapter }) => adapter)
+    }
+    const adapterCandidate = candidates[Math.floor(Math.random() * candidates.length)]
+    debug('adapter candidate is selected ', adapterCandidate._handler._remote.id)
+    return adapterCandidate
+  }
+
+  async setupWriters() {
+    debug('Setup writers ')
+    const settings = this.#props.settings
+    const allSettings = this.#props.job.settings
+
+    Object.entries(this.#adapters).forEach(([remoteId, adapter]) => {
+      const targetSettings = {
+        ...settings,
+        ...allSettings[remoteId],
+        skipDeleteOldEntries: true, // delete is handled globally
+      }
+      if (targetSettings.exportRetention !== 0) {
+        const writer = new this.#BackupWriter({
+          ...this.#props,
+          adapter,
+          remoteId,
+          settings: targetSettings,
+        })
+        this.#writers.push(writer)
+      }
+    })
+  }
+
+  async getEntriesPerAdapter(adapter) {
+    const scheduleId = this.#props.scheduleId
+    const vmUuid = this.#props.vmUuid
+    return (await adapter.listVmBackups(vmUuid, _ => _.scheduleId === scheduleId)).map(entry => ({
+      ...entry,
+      adapter,
+    }))
+  }
+
+  async setOldBackups() {
+    const entriesPerAdapter = await Promise.all(
+      Object.values(this.#adapters).map(adapter => this.getEntriesPerAdapter(adapter))
+    )
+    const entries = entriesPerAdapter
+      .flat(1)
+      .filter(_ => !!_)
+      .sort(compareTimestamp)
+
+    const settings = this.#props.settings
+    this.#oldBackups = getOldEntries(settings.exportRetention - 1, entries, {
+      longTermRetention: settings.longTermRetention,
+      timezone: settings.timezone,
+    })
+    debug(`got ${this.#oldBackups.length} old backup(s) to delete `)
+  }
+
+  async deleteOldBackupsOnAdapter(adapter, backup) {
+    throw new Error('not implemented')
+  }
+
+  async deleteOldBackups() {
+    const byAdapters = new Map()
+    this.#oldBackups.forEach(({ adapter, ...backup }) => {
+      const current = byAdapters.has(adapter) ? byAdapters.get(adapter) : []
+      current.push(backup)
+      byAdapters.set(adapter, current)
+    })
+    await asyncEach(
+      byAdapters.entries(),
+      async ([adapter, backups]) => {
+        debug(`delete ${backups.length} old backup(s) to delete from ${adapter._handler._remote.id}`)
+        await this.deleteOldBackupsOnAdapter(adapter, backups)
+      },
+      {
+        concurrency: this.#props.deleteConcurrency ?? DEFAULT_AGGREGATED_REMOTE_DELETE_CONCURRENCY,
+      }
+    )
+  }
+}

package/_runners/_writers/_AbstractAggregatedXapiWriter.mjs

@@ -0,0 +1,114 @@
+import { getOldEntries } from '../../_getOldEntries.mjs'
+import { createLogger } from '@xen-orchestra/log'
+import { compareReplicatedVmDatetime, listReplicatedVms } from './_listReplicatedVms.mjs'
+import { asyncMapSettled } from '@xen-orchestra/async-map'
+
+const { debug } = createLogger('xo:backups:AbstractAggregatedXapiWriter')
+export class AbstractAggregatedXapiWriter {
+  #ReplicationWriter
+  #storageRepositories
+
+  #writers = []
+  get writers() {
+    return this.#writers
+  }
+  mainWriter
+  #props
+  get props() {
+    return this.#props
+  }
+  #oldVmReplicaList = []
+
+  constructor({ srs, ReplicationWriter, ...props }) {
+    debug('instantiate AbstractAggregatedXapiWriter')
+    if (srs.length < 1) {
+      throw new Error('AggregatedXapiWriter need at least one storage repository to work ')
+    }
+    this.#props = props
+    this.#ReplicationWriter = ReplicationWriter
+    this.#storageRepositories = srs
+  }
+
+  getWriterBySr(storageRepository) {
+    for (const writer of this.#writers) {
+      if (writer._sr === storageRepository) {
+        return writer
+      }
+    }
+    throw new Error("Can't find writer for storage repository")
+  }
+
+  async computeSRCandidate() {
+    let maxSpace = 0
+    this.#storageRepositories.forEach(({ physical_utilisation, physical_size }) => {
+      if (physical_size - physical_utilisation > maxSpace) {
+        maxSpace = physical_size - physical_utilisation
+      }
+    })
+    debug(`computeSRCandidate found the max free space  ${maxSpace} for vm${this.#props.vmUuid}`)
+
+    const srCandidates = this.#storageRepositories.filter(
+      ({ physical_utilisation, physical_size }) => maxSpace === physical_size - physical_utilisation
+    )
+    // one at random from the sr with the most free space
+    const candidate = srCandidates[Math.floor(Math.random() * srCandidates.length)]
+
+    debug(
+      `computeSRCandidate found ${srCandidates.length} sr with ${maxSpace} free size, and chose ${candidate.name_label} for vm${this.#props.vmUuid}`
+    )
+    return candidate
+  }
+
+  setOldReplicaList() {
+    debug(`setOldReplicaList for vm ${this.#props.vmUuid}`)
+    const scheduleId = this.#props.scheduleId
+    const vmUuid = this.#props.vmUuid
+    const settings = this.#props.settings
+
+    const replicatedVms = this.#storageRepositories
+      .map(sr => {
+        return listReplicatedVms(sr.$xapi, scheduleId, sr.uuid, vmUuid)
+      })
+      .flat(1)
+      .filter(_ => !!_)
+      .sort(compareReplicatedVmDatetime)
+
+    this.#oldVmReplicaList = getOldEntries(settings.copyRetention - 1, replicatedVms)
+
+    debug(
+      `setOldReplicaList found ${replicatedVms.length} replica, and will delete ${this.#oldVmReplicaList.length} (retention ${settings.copyRetention})  for vm ${this.#props.vmUuid}`
+    )
+  }
+
+  async deleteOldReplicas() {
+    debug(`deleteOldReplicas will delete ${this.#oldVmReplicaList.length} replicated vms  for vm ${this.#props.vmUuid}`)
+    // destroy the VM on the right xapi
+    await asyncMapSettled(this.#oldVmReplicaList, async vm => {
+      debug('will delete old replica', vm.name_label)
+      await vm.$xapi.VM_destroy(vm.$ref)
+    })
+    debug(`deleteOldReplicas deleted  ${this.#oldVmReplicaList.length} replicated vms  for vm ${this.#props.vmUuid}`)
+  }
+
+  async setupWriters() {
+    debug('Setup writers ')
+    const settings = this.#props.settings
+    const allSettings = this.#props.job.settings
+
+    this.#storageRepositories.forEach(storageRepository => {
+      const targetSettings = {
+        ...settings,
+        ...allSettings[storageRepository.uuid],
+        skipDeleteOldEntries: true, // delete is handled globally
+      }
+      if (targetSettings.copyRetention !== 0) {
+        const writer = new this.#ReplicationWriter({
+          ...this.#props,
+          sr: storageRepository,
+          settings: targetSettings,
+        })
+        this.#writers.push(writer)
+      }
+    })
+  }
+}

package/_runners/_writers/_listReplicatedVms.mjs

@@ -5,10 +5,18 @@ const getReplicatedVmDatetime = vm => {
   return datetime
 }
 
-const compareReplicatedVmDatetime = (a, b) => (getReplicatedVmDatetime(a) < getReplicatedVmDatetime(b) ? -1 : 1)
+export const compareReplicatedVmDatetime = (a, b) => (getReplicatedVmDatetime(a) < getReplicatedVmDatetime(b) ? -1 : 1)
 
+/**
+ *
+ * @param {Xapi} xapi
+ * @param {string} scheduleOrJobId
+ * @param {string} srUuid
+ * @param {string} vmUuid
+ * @returns {Array<XoVm>}
+ */
 export function listReplicatedVms(xapi, scheduleOrJobId, srUuid, vmUuid) {
-  const { all } = xapi.objects
+  const all = xapi.objects.indexes.type.VM
   const vms = {}
   for (const key in all) {
     const object = all[key]

package/disks/MergeRemoteDisk.mjs

@@ -0,0 +1,317 @@
+// @ts-check
+
+/**
+ * @typedef {import('./RemoteDisk.mjs').RemoteDisk} RemoteDisk
+ * @typedef {import('@xen-orchestra/disk-transform').FileAccessor} FileAccessor
+ */
+
+import assert from 'assert'
+import { createLogger } from '@xen-orchestra/log'
+
+import { basename, dirname } from 'path'
+import { asyncEach } from '@vates/async-each'
+
+// @ts-ignore
+const { warn } = createLogger('remote-disk:merge')
+
+/**
+ * @typedef {Object} MergeState
+ * @property {{ uuid: string }} child
+ * @property {{ uuid: string }} parent
+ * @property {number} currentBlock
+ * @property {number} mergedDataSize
+ * @property {'mergeBlocks' | 'cleanup'} step
+ * @property {number} diskSize
+ */
+
+export class MergeRemoteDisk {
+  /**
+   * @type {MergeState}
+   */
+  #state = {
+    child: { uuid: '0' },
+    parent: { uuid: '0' },
+    currentBlock: 0,
+    mergedDataSize: 0,
+    step: 'mergeBlocks',
+    diskSize: 0,
+  }
+
+  /**
+   * @type {string}
+   */
+  #statePath = ''
+
+  /**
+   * @type {boolean}
+   */
+  #isResuming
+
+  /**
+   * @type {Logger | Function}
+   */
+  #logInfo
+
+  /**
+   * @type {number}
+   */
+  #mergeBlockConcurrency
+
+  /**
+   * @type {Function}
+   */
+  #onProgress
+
+  /**
+   * @type {boolean}
+   */
+  #removeUnused
+
+  /**
+   * @type {number}
+   */
+  #writeStateDelay
+
+  /**
+   * @type {number}
+   */
+  #lastStateWrittenAt
+
+  /**
+   * @type {FileAccessor}
+   */
+  #handler
+
+  /**
+   * @param {FileAccessor} handler
+   * @param {Object} params
+   * @param {Function} params.onProgress
+   * @param {Logger | Function} params.logInfo
+   * @param {boolean} params.removeUnused
+   * @param {number} params.mergeBlockConcurrency
+   * @param {number} params.writeStateDelay
+   */
+  constructor(
+    handler,
+    {
+      onProgress = () => {},
+      logInfo = () => {},
+      removeUnused = false,
+      mergeBlockConcurrency = 2,
+      writeStateDelay = 10e3,
+    }
+  ) {
+    this.#handler = handler
+    this.#logInfo = logInfo
+    this.#onProgress = onProgress
+    this.#removeUnused = removeUnused
+    this.#mergeBlockConcurrency = mergeBlockConcurrency
+    this.#writeStateDelay = writeStateDelay
+
+    this.#isResuming = false
+    this.#lastStateWrittenAt = 0
+  }
+
+  /**
+   * @param {RemoteDisk} parentDisk
+   * @returns {Promise<boolean>} isResuming
+   */
+  async isResuming(parentDisk) {
+    try {
+      await this.#handler.readFile(
+        dirname(parentDisk.getPath()) + '/.' + basename(parentDisk.getPath()) + '.merge.json'
+      )
+      return true
+    } catch (error) {
+      return false
+    }
+  }
+
+  /**
+   * @param {RemoteDisk} parentDisk
+   * @param {RemoteDisk} childDisk
+   */
+  async merge(parentDisk, childDisk) {
+    this.#statePath = dirname(parentDisk.getPath()) + '/.' + basename(parentDisk.getPath()) + '.merge.json'
+
+    try {
+      const mergeStateContent = await this.#handler.readFile(this.#statePath)
+      this.#state = JSON.parse(mergeStateContent)
+
+      // work-around a bug introduce in 97d94b795
+      //
+      // currentBlock could be `null` due to the JSON.stringify of a `NaN` value
+      if (this.#state?.currentBlock === null) this.#state.currentBlock = 0
+      this.#isResuming = true
+    } catch (error) {
+      // @ts-ignore
+      if (error.code !== 'ENOENT') {
+        warn('problem while checking the merge state', { error })
+      }
+    }
+
+    try {
+      /* eslint-disable no-fallthrough */
+      switch (this.#state?.step ?? 'mergeBlocks') {
+        case 'mergeBlocks':
+          await this.#step_mergeBlocks(parentDisk, childDisk)
+        case 'cleanup':
+          await this.#step_cleanup(parentDisk, childDisk)
+          return this.#cleanup(parentDisk, childDisk, true)
+        default:
+          warn(`Step ${this.#state?.step} is unknown`, { state: this.#state })
+      }
+      /* eslint-enable no-fallthrough */
+    } catch (error) {
+      await this.#cleanup(parentDisk, childDisk, false)
+      throw error
+    }
+  }
+
+  async #writeState() {
+    try {
+      await this.#handler.writeFile(this.#statePath, JSON.stringify(this.#state), { flags: 'w' })
+    } catch (err) {
+      warn('failed to write merge state', { error: err })
+    }
+  }
+
+  async #writeStateThrottled() {
+    const now = Date.now()
+    if (now - this.#lastStateWrittenAt > this.#writeStateDelay) {
+      this.#lastStateWrittenAt = now
+      await this.#writeState()
+    }
+  }
+
+  /**
+   * @param {RemoteDisk} parentDisk
+   * @param {RemoteDisk} childDisk
+   */
+  async #step_mergeBlocks(parentDisk, childDisk) {
+    const getMaxBlockCount = childDisk.getMaxBlockCount()
+
+    if (this.#isResuming) {
+      const alreadyMergedBlocks = []
+      for (let blockId = 0; blockId < this.#state.currentBlock; blockId++) {
+        if (childDisk.hasBlock(blockId)) {
+          alreadyMergedBlocks.push(blockId)
+        }
+      }
+
+      parentDisk.setAllocatedBlocks(alreadyMergedBlocks)
+    } else {
+      this.#state.child = { uuid: childDisk.getUuid() ?? 0 }
+      this.#state.parent = { uuid: parentDisk.getUuid() ?? 0 }
+
+      // Finds first allocated block for the 2 following loops
+      while (this.#state.currentBlock < getMaxBlockCount && !childDisk.hasBlock(this.#state.currentBlock)) {
+        ++this.#state.currentBlock
+      }
+      await this.#writeState()
+    }
+
+    await this.#mergeBlocks(parentDisk, childDisk)
+    await parentDisk.flushMetadata(childDisk)
+    parentDisk.mergeMetadata(childDisk)
+  }
+
+  /**
+   * @param {RemoteDisk} parentDisk
+   * @param {RemoteDisk} childDisk
+   */
+  async #mergeBlocks(parentDisk, childDisk) {
+    this.#mergeBlockConcurrency =
+      (await parentDisk.canMergeConcurently()) && (await childDisk.canMergeConcurently())
+        ? this.#mergeBlockConcurrency
+        : 1
+
+    const toMerge = []
+    for (let block = this.#state.currentBlock; block < childDisk.getMaxBlockCount(); block++) {
+      if (childDisk.hasBlock(block)) {
+        toMerge.push(block)
+      }
+    }
+
+    const nBlocks = toMerge.length
+    this.#onProgress({ total: nBlocks, done: 0 })
+
+    const merging = new Set()
+    let counter = 0
+    await asyncEach(
+      toMerge,
+      async blockId => {
+        merging.add(blockId)
+
+        const blockSize = await parentDisk.mergeBlock(childDisk, blockId, this.#isResuming)
+        this.#state.mergedDataSize += blockSize
+
+        this.#state.currentBlock = Math.min(...merging) - 1
+        merging.delete(blockId)
+
+        this.#onProgress({ total: nBlocks, done: counter + 1 })
+        counter++
+        await this.#writeStateThrottled()
+      },
+      { concurrency: this.#mergeBlockConcurrency }
+    )
+
+    await this.#writeState()
+
+    this.#state.diskSize = childDisk.getSizeOnDisk()
+
+    this.#onProgress({ total: nBlocks, done: nBlocks })
+  }
+
+  /**
+   * @param {RemoteDisk} parentDisk
+   * @param {RemoteDisk} childDisk
+   */
+  async #step_cleanup(parentDisk, childDisk) {
+    assert.notEqual(this.#state, undefined)
+    this.#state.step = 'cleanup'
+    await this.#writeState()
+
+    const mergeTargetPath = childDisk.getPath()
+
+    // delete intermediate children if needed
+    if (this.#removeUnused) {
+      await childDisk.unlink()
+    }
+
+    try {
+      await parentDisk.rename(mergeTargetPath)
+    } catch (error) {
+      // @ts-ignore
+      if (error.code === 'ENOENT' && this.#isResuming) {
+        // @ts-ignore
+        this.#logInfo(`the parent disk was already renamed`, {
+          parent: parentDisk.getPath(),
+          mergeTarget: mergeTargetPath,
+        })
+      } else {
+        throw error
+      }
+    }
+  }
+
+  /**
+   * @param {RemoteDisk} parentDisk
+   * @param {RemoteDisk} childDisk
+   * @param {boolean} cleanStateFile
+   *
+   * @returns {Promise<{mergedDataSize: number, finalDiskSize: number}>} result
+   */
+  async #cleanup(parentDisk, childDisk, cleanStateFile) {
+    const finalDiskSize = this.#state?.diskSize ?? 0
+    const mergedDataSize = this.#state?.mergedDataSize ?? 0
+    await parentDisk.close().catch(warn)
+    await childDisk.close().catch(warn)
+
+    if (cleanStateFile) {
+      await this.#handler.unlink(this.#statePath).catch(warn)
+    }
+
+    return { mergedDataSize, finalDiskSize }
+  }
+}