@xen-orchestra/backups 0.14.0

package/merge-worker/cli.js

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+const { catchGlobalErrors } = require('@xen-orchestra/log/configure.js')
+const { createLogger } = require('@xen-orchestra/log')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { join } = require('path')
+const Disposable = require('promise-toolbox/Disposable')
+const min = require('lodash/min')
+
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { RemoteAdapter } = require('../RemoteAdapter.js')
+
+const { CLEAN_VM_QUEUE } = require('./index.js')
+
+// -------------------------------------------------------------------
+
+catchGlobalErrors(createLogger('xo:backups:mergeWorker'))
+
+const { fatal, info, warn } = createLogger('xo:backups:mergeWorker')
+
+// -------------------------------------------------------------------
+
+const main = Disposable.wrap(async function* main(args) {
+  const handler = yield getSyncedHandler({ url: 'file://' + process.cwd() })
+
+  yield handler.lock(CLEAN_VM_QUEUE)
+
+  const adapter = new RemoteAdapter(handler)
+
+  const listRetry = async () => {
+    const timeoutResolver = resolve => setTimeout(resolve, 10e3)
+    for (let i = 0; i < 10; ++i) {
+      const entries = await handler.list(CLEAN_VM_QUEUE)
+      if (entries.length !== 0) {
+        return entries
+      }
+      await new Promise(timeoutResolver)
+    }
+  }
+
+  let taskFiles
+  while ((taskFiles = await listRetry()) !== undefined) {
+    const taskFileBasename = min(taskFiles)
+    const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
+
+    // move this task to the end
+    await handler.rename(join(CLEAN_VM_QUEUE, taskFileBasename), taskFile)
+    try {
+      const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
+      await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+
+      handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
+    } catch (error) {
+      warn('failure handling task', { error })
+    }
+  }
+})
+
+info('starting')
+main(process.argv.slice(2)).then(
+  () => {
+    info('bye :-)')
+  },
+  error => {
+    fatal(error)
+
+    process.exit(1)
+  }
+)

package/merge-worker/index.js

@@ -0,0 +1,25 @@
+const { join, resolve } = require('path')
+const { spawn } = require('child_process')
+const { check } = require('proper-lockfile')
+
+const CLEAN_VM_QUEUE = (exports.CLEAN_VM_QUEUE = '/xo-vm-backups/.queue/clean-vm/')
+
+const CLI_PATH = resolve(__dirname, 'cli.js')
+exports.run = async function runMergeWorker(remotePath) {
+  try {
+    // TODO: find a way to pass the acquire the lock and then pass it down the worker
+    if (await check(join(remotePath, CLEAN_VM_QUEUE))) {
+      // already locked, don't start another worker
+      return
+    }
+
+    spawn(CLI_PATH, {
+      cwd: remotePath,
+      detached: true,
+      stdio: 'inherit',
+    }).unref()
+  } catch (error) {
+    // we usually don't want to throw if the merge worker failed to start
+    return error
+  }
+}

package/package.json

@@ -0,0 +1,49 @@
+{
+  "private": false,
+  "name": "@xen-orchestra/backups",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backups",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/backups",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "version": "0.14.0",
+  "engines": {
+    "node": ">=14.6"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "dependencies": {
+    "@vates/compose": "^2.0.0",
+    "@vates/disposable": "^0.1.1",
+    "@vates/parse-duration": "^0.1.1",
+    "@xen-orchestra/async-map": "^0.1.2",
+    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/log": "^0.3.0",
+    "@xen-orchestra/template": "^0.1.0",
+    "compare-versions": "^3.6.0",
+    "d3-time-format": "^3.0.0",
+    "end-of-stream": "^1.4.4",
+    "fs-extra": "^10.0.0",
+    "golike-defer": "^0.5.1",
+    "limit-concurrency-decorator": "^0.5.0",
+    "lodash": "^4.17.20",
+    "node-zone": "^0.4.0",
+    "parse-pairs": "^1.1.0",
+    "promise-toolbox": "^0.20.0",
+    "proper-lockfile": "^4.1.2",
+    "pump": "^3.0.0",
+    "vhd-lib": "^1.2.0",
+    "yazl": "^2.5.1"
+  },
+  "peerDependencies": {
+    "@xen-orchestra/xapi": "^0.7.0"
+  },
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  }
+}

package/parseMetadataBackupId.js

@@ -0,0 +1,23 @@
+const { DIR_XO_CONFIG_BACKUPS, DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
+
+exports.parseMetadataBackupId = function parseMetadataBackupId(backupId) {
+  const [dir, ...rest] = backupId.split('/')
+  if (dir === DIR_XO_CONFIG_BACKUPS) {
+    const [scheduleId, timestamp] = rest
+    return {
+      type: 'xoConfig',
+      scheduleId,
+      timestamp,
+    }
+  } else if (dir === DIR_XO_POOL_METADATA_BACKUPS) {
+    const [scheduleId, poolUuid, timestamp] = rest
+    return {
+      type: 'pool',
+      poolUuid,
+      scheduleId,
+      timestamp,
+    }
+  }
+
+  throw new Error(`not supported backup dir (${dir})`)
+}

package/runBackupWorker.js

@@ -0,0 +1,38 @@
+const path = require('path')
+const { createLogger } = require('@xen-orchestra/log')
+const { fork } = require('child_process')
+
+const { warn } = createLogger('xo:backups:backupWorker')
+
+const PATH = path.resolve(__dirname, '_backupWorker.js')
+
+exports.runBackupWorker = function runBackupWorker(params, onLog) {
+  return new Promise((resolve, reject) => {
+    const worker = fork(PATH)
+
+    worker.on('exit', code => reject(new Error(`worker exited with code ${code}`)))
+    worker.on('error', reject)
+
+    worker.on('message', message => {
+      try {
+        if (message.type === 'result') {
+          if (message.status === 'success') {
+            resolve(message.result)
+          } else {
+            reject(message.result)
+          }
+        } else if (message.type === 'log') {
+          onLog(message.data)
+        }
+      } catch (error) {
+        warn(error)
+      }
+    })
+
+    worker.send({
+      action: 'run',
+      data: params,
+      runWithLogs: onLog !== undefined,
+    })
+  })
+}

package/writers/DeltaBackupWriter.js

@@ -0,0 +1,221 @@
+const assert = require('assert')
+const map = require('lodash/map.js')
+const mapValues = require('lodash/mapValues.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const { asyncMap } = require('@xen-orchestra/async-map')
+const { chainVhd, checkVhdChain, default: Vhd } = require('vhd-lib')
+const { createLogger } = require('@xen-orchestra/log')
+const { dirname } = require('path')
+
+const { formatFilenameDate } = require('../_filenameDate.js')
+const { getOldEntries } = require('../_getOldEntries.js')
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { Task } = require('../Task.js')
+
+const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
+const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
+const { checkVhd } = require('./_checkVhd.js')
+const { packUuid } = require('./_packUuid.js')
+
+const { warn } = createLogger('xo:backups:DeltaBackupWriter')
+
+exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
+  async checkBaseVdis(baseUuidToSrcVdi) {
+    const { handler } = this._adapter
+    const backup = this._backup
+
+    const backupDir = getVmBackupDir(backup.vm.uuid)
+    const vdisDir = `${backupDir}/vdis/${backup.job.id}`
+
+    await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdi]) => {
+      let found = false
+      try {
+        const vhds = await handler.list(`${vdisDir}/${srcVdi.uuid}`, {
+          filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
+          prependDir: true,
+        })
+        await asyncMap(vhds, async path => {
+          try {
+            await checkVhdChain(handler, path)
+
+            const vhd = new Vhd(handler, path)
+            await vhd.readHeaderAndFooter()
+            found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
+          } catch (error) {
+            warn('checkBaseVdis', { error })
+            await ignoreErrors.call(handler.unlink(path))
+          }
+        })
+      } catch (error) {
+        warn('checkBaseVdis', { error })
+      }
+      if (!found) {
+        baseUuidToSrcVdi.delete(baseUuid)
+      }
+    })
+  }
+
+  async beforeBackup() {
+    await super.beforeBackup()
+    return this._cleanVm({ merge: true })
+  }
+
+  prepare({ isFull }) {
+    // create the task related to this export and ensure all methods are called in this context
+    const task = new Task({
+      name: 'export',
+      data: {
+        id: this._remoteId,
+        isFull,
+        type: 'remote',
+      },
+    })
+    this.transfer = task.wrapFn(this.transfer)
+    this.cleanup = task.wrapFn(this.cleanup, true)
+
+    return task.run(() => this._prepare())
+  }
+
+  async _prepare() {
+    const adapter = this._adapter
+    const settings = this._settings
+    const { scheduleId, vm } = this._backup
+
+    const oldEntries = getOldEntries(
+      settings.exportRetention - 1,
+      await adapter.listVmBackups(vm.uuid, _ => _.mode === 'delta' && _.scheduleId === scheduleId)
+    )
+    this._oldEntries = oldEntries
+
+    // FIXME: implement optimized multiple VHDs merging with synthetic
+    // delta
+    //
+    // For the time being, limit the number of deleted backups by run
+    // because it can take a very long time and can lead to
+    // interrupted backup with broken VHD chain.
+    //
+    // The old backups will be eventually merged in future runs of the
+    // job.
+    const { maxMergedDeltasPerRun } = this._settings
+    if (oldEntries.length > maxMergedDeltasPerRun) {
+      oldEntries.length = maxMergedDeltasPerRun
+    }
+
+    if (settings.deleteFirst) {
+      await this._deleteOldEntries()
+    }
+  }
+
+  async cleanup() {
+    if (!this._settings.deleteFirst) {
+      await this._deleteOldEntries()
+    }
+  }
+
+  async _deleteOldEntries() {
+    const adapter = this._adapter
+    const oldEntries = this._oldEntries
+
+    // delete sequentially from newest to oldest to avoid unnecessary merges
+    for (let i = oldEntries.length; i-- > 0; ) {
+      await adapter.deleteDeltaVmBackups([oldEntries[i]])
+    }
+  }
+
+  async _transfer({ timestamp, deltaExport, sizeContainers }) {
+    const adapter = this._adapter
+    const backup = this._backup
+
+    const { job, scheduleId, vm } = backup
+
+    const jobId = job.id
+    const handler = adapter.handler
+    const backupDir = getVmBackupDir(vm.uuid)
+
+    // TODO: clean VM backup directory
+
+    const basename = formatFilenameDate(timestamp)
+    const vhds = mapValues(
+      deltaExport.vdis,
+      vdi =>
+        `vdis/${jobId}/${
+          vdi.type === 'suspend'
+            ? // doesn't make sense to group by parent for memory because we
+              // don't do delta for it
+              vdi.uuid
+            : vdi.$snapshot_of$uuid
+        }/${basename}.vhd`
+    )
+
+    const metadataFilename = `${backupDir}/${basename}.json`
+    const metadataContent = {
+      jobId,
+      mode: job.mode,
+      scheduleId,
+      timestamp,
+      vbds: deltaExport.vbds,
+      vdis: deltaExport.vdis,
+      version: '2.0.0',
+      vifs: deltaExport.vifs,
+      vhds,
+      vm,
+      vmSnapshot: this._backup.exportedVm,
+    }
+
+    const { size } = await Task.run({ name: 'transfer' }, async () => {
+      await Promise.all(
+        map(deltaExport.vdis, async (vdi, id) => {
+          const path = `${backupDir}/${vhds[id]}`
+
+          const isDelta = vdi.other_config['xo:base_delta'] !== undefined
+          let parentPath
+          if (isDelta) {
+            const vdiDir = dirname(path)
+            parentPath = (
+              await handler.list(vdiDir, {
+                filter: filename => filename[0] !== '.' && filename.endsWith('.vhd'),
+                prependDir: true,
+              })
+            )
+              .sort()
+              .pop()
+
+            assert.notStrictEqual(parentPath, undefined, `missing parent of ${id}`)
+
+            parentPath = parentPath.slice(1) // remove leading slash
+
+            // TODO remove when this has been done before the export
+            await checkVhd(handler, parentPath)
+          }
+
+          await adapter.outputStream(path, deltaExport.streams[`${id}.vhd`], {
+            // no checksum for VHDs, because they will be invalidated by
+            // merges and chainings
+            checksum: false,
+            validator: tmpPath => checkVhd(handler, tmpPath),
+          })
+
+          if (isDelta) {
+            await chainVhd(handler, parentPath, handler, path)
+          }
+
+          // set the correct UUID in the VHD
+          const vhd = new Vhd(handler, path)
+          await vhd.readHeaderAndFooter()
+          vhd.footer.uuid = packUuid(vdi.uuid)
+          await vhd.readBlockAllocationTable() // required by writeFooter()
+          await vhd.writeFooter()
+        })
+      )
+      return {
+        size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0),
+      }
+    })
+    metadataContent.size = size
+    await handler.outputFile(metadataFilename, JSON.stringify(metadataContent), {
+      dirMode: backup.config.dirMode,
+    })
+
+    // TODO: run cleanup?
+  }
+}

package/writers/DeltaReplicationWriter.js

@@ -0,0 +1,126 @@
+const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const { formatDateTime } = require('@xen-orchestra/xapi')
+
+const { formatFilenameDate } = require('../_filenameDate.js')
+const { getOldEntries } = require('../_getOldEntries.js')
+const { importDeltaVm, TAG_COPY_SRC } = require('../_deltaVm.js')
+const { Task } = require('../Task.js')
+
+const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
+const { MixinReplicationWriter } = require('./_MixinReplicationWriter.js')
+const { listReplicatedVms } = require('./_listReplicatedVms.js')
+
+exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinReplicationWriter(AbstractDeltaWriter) {
+  async checkBaseVdis(baseUuidToSrcVdi, baseVm) {
+    const sr = this._sr
+    const replicatedVm = listReplicatedVms(sr.$xapi, this._backup.job.id, sr.uuid, this._backup.vm.uuid).find(
+      vm => vm.other_config[TAG_COPY_SRC] === baseVm.uuid
+    )
+    if (replicatedVm === undefined) {
+      return baseUuidToSrcVdi.clear()
+    }
+
+    const xapi = replicatedVm.$xapi
+    const replicatedVdis = new Set(
+      await asyncMap(await replicatedVm.$getDisks(), async vdiRef => {
+        const otherConfig = await xapi.getField('VDI', vdiRef, 'other_config')
+        return otherConfig[TAG_COPY_SRC]
+      })
+    )
+
+    for (const uuid of baseUuidToSrcVdi.keys()) {
+      if (!replicatedVdis.has(uuid)) {
+        baseUuidToSrcVdi.delete(uuid)
+      }
+    }
+  }
+
+  prepare({ isFull }) {
+    // create the task related to this export and ensure all methods are called in this context
+    const task = new Task({
+      name: 'export',
+      data: {
+        id: this._sr.uuid,
+        isFull,
+        type: 'SR',
+      },
+    })
+    this.transfer = task.wrapFn(this.transfer)
+    this.cleanup = task.wrapFn(this.cleanup, true)
+
+    return task.run(() => this._prepare())
+  }
+
+  async _prepare() {
+    const settings = this._settings
+    const { uuid: srUuid, $xapi: xapi } = this._sr
+    const { scheduleId, vm } = this._backup
+
+    // delete previous interrupted copies
+    ignoreErrors.call(asyncMapSettled(listReplicatedVms(xapi, scheduleId, undefined, vm.uuid), vm => vm.$destroy))
+
+    this._oldEntries = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vm.uuid))
+
+    if (settings.deleteFirst) {
+      await this._deleteOldEntries()
+    }
+  }
+
+  async cleanup() {
+    if (!this._settings.deleteFirst) {
+      await this._deleteOldEntries()
+    }
+  }
+
+  async _deleteOldEntries() {
+    return asyncMapSettled(this._oldEntries, vm => vm.$destroy())
+  }
+
+  async _transfer({ timestamp, deltaExport, sizeContainers }) {
+    const sr = this._sr
+    const { job, scheduleId, vm } = this._backup
+
+    const { uuid: srUuid, $xapi: xapi } = sr
+
+    let targetVmRef
+    await Task.run({ name: 'transfer' }, async () => {
+      targetVmRef = await importDeltaVm(
+        {
+          __proto__: deltaExport,
+          vm: {
+            ...deltaExport.vm,
+            tags: [...deltaExport.vm.tags, 'Continuous Replication'],
+          },
+        },
+        sr
+      )
+      return {
+        size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0),
+      }
+    })
+
+    const targetVm = await xapi.getRecord('VM', targetVmRef)
+
+    await Promise.all([
+      targetVm.ha_restart_priority !== '' &&
+        Promise.all([targetVm.set_ha_restart_priority(''), targetVm.add_tags('HA disabled')]),
+      targetVm.set_name_label(`${vm.name_label} - ${job.name} - (${formatFilenameDate(timestamp)})`),
+      asyncMap(['start', 'start_on'], op =>
+        targetVm.update_blocked_operations(
+          op,
+          'Start operation for this vm is blocked, clone it if you want to use it.'
+        )
+      ),
+      targetVm.update_other_config({
+        'xo:backup:sr': srUuid,
+
+        // these entries need to be added in case of offline backup
+        'xo:backup:datetime': formatDateTime(timestamp),
+        'xo:backup:job': job.id,
+        'xo:backup:schedule': scheduleId,
+        'xo:backup:vm': vm.uuid,
+      }),
+    ])
+  }
+}

package/writers/FullBackupWriter.js

@@ -0,0 +1,85 @@
+const { formatFilenameDate } = require('../_filenameDate.js')
+const { getOldEntries } = require('../_getOldEntries.js')
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { Task } = require('../Task.js')
+
+const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
+const { AbstractFullWriter } = require('./_AbstractFullWriter.js')
+
+exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(AbstractFullWriter) {
+  constructor(props) {
+    super(props)
+
+    this.run = Task.wrapFn(
+      {
+        name: 'export',
+        data: {
+          id: props.remoteId,
+          type: 'remote',
+
+          // necessary?
+          isFull: true,
+        },
+      },
+      this.run
+    )
+  }
+
+  async _run({ timestamp, sizeContainer, stream }) {
+    const backup = this._backup
+    const settings = this._settings
+
+    const { job, scheduleId, vm } = backup
+
+    const adapter = this._adapter
+    const handler = adapter.handler
+    const backupDir = getVmBackupDir(vm.uuid)
+
+    // TODO: clean VM backup directory
+
+    const oldBackups = getOldEntries(
+      settings.exportRetention - 1,
+      await adapter.listVmBackups(vm.uuid, _ => _.mode === 'full' && _.scheduleId === scheduleId)
+    )
+    const deleteOldBackups = () => adapter.deleteFullVmBackups(oldBackups)
+
+    const basename = formatFilenameDate(timestamp)
+
+    const dataBasename = basename + '.xva'
+    const dataFilename = backupDir + '/' + dataBasename
+
+    const metadataFilename = `${backupDir}/${basename}.json`
+    const metadata = {
+      jobId: job.id,
+      mode: job.mode,
+      scheduleId,
+      timestamp,
+      version: '2.0.0',
+      vm,
+      vmSnapshot: this._backup.exportedVm,
+      xva: './' + dataBasename,
+    }
+
+    const { deleteFirst } = settings
+    if (deleteFirst) {
+      await deleteOldBackups()
+    }
+
+    await Task.run({ name: 'transfer' }, async () => {
+      await adapter.outputStream(dataFilename, stream, {
+        validator: tmpPath => adapter.isValidXva(tmpPath),
+      })
+      return { size: sizeContainer.size }
+    })
+    metadata.size = sizeContainer.size
+    await handler.outputFile(metadataFilename, JSON.stringify(metadata), {
+      dirMode: backup.config.dirMode,
+    })
+
+    if (!deleteFirst) {
+      await deleteOldBackups()
+    }
+
+    // TODO: run cleanup?
+  }
+}