@xen-orchestra/backups 0.14.0

package/_deltaVm.js

@@ -0,0 +1,347 @@
+const compareVersions = require('compare-versions')
+const find = require('lodash/find.js')
+const groupBy = require('lodash/groupBy.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const omit = require('lodash/omit.js')
+const { asyncMap } = require('@xen-orchestra/async-map')
+const { CancelToken } = require('promise-toolbox')
+const { createVhdStreamWithLength } = require('vhd-lib')
+const { defer } = require('golike-defer')
+
+const { cancelableMap } = require('./_cancelableMap.js')
+
+const TAG_BASE_DELTA = 'xo:base_delta'
+exports.TAG_BASE_DELTA = TAG_BASE_DELTA
+
+const TAG_COPY_SRC = 'xo:copy_of'
+exports.TAG_COPY_SRC = TAG_COPY_SRC
+
+const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
+
+exports.exportDeltaVm = async function exportDeltaVm(
+  vm,
+  baseVm,
+  {
+    cancelToken = CancelToken.none,
+
+    // Sets of UUIDs of VDIs that must be exported as full.
+    fullVdisRequired = new Set(),
+
+    disableBaseTags = false,
+  } = {}
+) {
+  // refs of VM's VDIs → base's VDIs.
+  const baseVdis = {}
+  baseVm &&
+    baseVm.$VBDs.forEach(vbd => {
+      let vdi, snapshotOf
+      if ((vdi = vbd.$VDI) && (snapshotOf = vdi.$snapshot_of) && !fullVdisRequired.has(snapshotOf.uuid)) {
+        baseVdis[vdi.snapshot_of] = vdi
+      }
+    })
+
+  const streams = {}
+  const vdis = {}
+  const vbds = {}
+  await cancelableMap(cancelToken, vm.$VBDs, async (cancelToken, vbd) => {
+    let vdi
+    if (vbd.type !== 'Disk' || !(vdi = vbd.$VDI)) {
+      // Ignore this VBD.
+      return
+    }
+
+    // If the VDI name start with `[NOBAK]`, do not export it.
+    if (vdi.name_label.startsWith('[NOBAK]')) {
+      // FIXME: find a way to not create the VDI snapshot in the
+      // first time.
+      //
+      // The snapshot must not exist otherwise it could break the
+      // next export.
+      ignoreErrors.call(vdi.$destroy())
+      return
+    }
+
+    vbds[vbd.$ref] = vbd
+
+    const vdiRef = vdi.$ref
+    if (vdiRef in vdis) {
+      // This VDI has already been managed.
+      return
+    }
+
+    // Look for a snapshot of this vdi in the base VM.
+    const baseVdi = baseVdis[vdi.snapshot_of]
+
+    vdis[vdiRef] = {
+      ...vdi,
+      other_config: {
+        ...vdi.other_config,
+        [TAG_BASE_DELTA]: baseVdi && !disableBaseTags ? baseVdi.uuid : undefined,
+      },
+      $snapshot_of$uuid: vdi.$snapshot_of?.uuid,
+      $SR$uuid: vdi.$SR.uuid,
+    }
+
+    streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
+      baseRef: baseVdi?.$ref,
+      cancelToken,
+      format: 'vhd',
+    })
+  })
+
+  const suspendVdi = vm.$suspend_VDI
+  if (suspendVdi !== undefined) {
+    const vdiRef = suspendVdi.$ref
+    vdis[vdiRef] = {
+      ...suspendVdi,
+      $SR$uuid: suspendVdi.$SR.uuid,
+    }
+    streams[`${vdiRef}.vhd`] = await suspendVdi.$exportContent({
+      cancelToken,
+      format: 'vhd',
+    })
+  }
+
+  const vifs = {}
+  vm.$VIFs.forEach(vif => {
+    const network = vif.$network
+    vifs[vif.$ref] = {
+      ...vif,
+      $network$uuid: network.uuid,
+      $network$name_label: network.name_label,
+      $network$VLAN: network.$PIFs[0]?.VLAN,
+    }
+  })
+
+  return Object.defineProperty(
+    {
+      version: '1.1.0',
+      vbds,
+      vdis,
+      vifs,
+      vm: {
+        ...vm,
+        other_config:
+          baseVm && !disableBaseTags
+            ? {
+                ...vm.other_config,
+                [TAG_BASE_DELTA]: baseVm.uuid,
+              }
+            : omit(vm.other_config, TAG_BASE_DELTA),
+      },
+    },
+    'streams',
+    {
+      configurable: true,
+      value: streams,
+      writable: true,
+    }
+  )
+}
+
+exports.importDeltaVm = defer(async function importDeltaVm(
+  $defer,
+  deltaVm,
+  sr,
+  { cancelToken = CancelToken.none, detectBase = true, mapVdisSrs = {}, newMacAddresses = false } = {}
+) {
+  const { version } = deltaVm
+  if (compareVersions(version, '1.0.0') < 0) {
+    throw new Error(`Unsupported delta backup version: ${version}`)
+  }
+
+  const vmRecord = deltaVm.vm
+  const xapi = sr.$xapi
+
+  let baseVm
+  if (detectBase) {
+    const remoteBaseVmUuid = vmRecord.other_config[TAG_BASE_DELTA]
+    if (remoteBaseVmUuid) {
+      baseVm = find(xapi.objects.all, obj => (obj = obj.other_config) && obj[TAG_COPY_SRC] === remoteBaseVmUuid)
+
+      if (!baseVm) {
+        throw new Error(`could not find the base VM (copy of ${remoteBaseVmUuid})`)
+      }
+    }
+  }
+
+  const baseVdis = {}
+  baseVm &&
+    baseVm.$VBDs.forEach(vbd => {
+      const vdi = vbd.$VDI
+      if (vdi !== undefined) {
+        baseVdis[vbd.VDI] = vbd.$VDI
+      }
+    })
+  const vdiRecords = deltaVm.vdis
+
+  // 0. Create suspend_VDI
+  let suspendVdi
+  if (vmRecord.power_state === 'Suspended') {
+    const vdi = vdiRecords[vmRecord.suspend_VDI]
+    suspendVdi = await xapi.getRecord(
+      'VDI',
+      await xapi.VDI_create({
+        ...vdi,
+        other_config: {
+          ...vdi.other_config,
+          [TAG_BASE_DELTA]: undefined,
+          [TAG_COPY_SRC]: vdi.uuid,
+        },
+        sr: mapVdisSrs[vdi.uuid] ?? sr.$ref,
+      })
+    )
+    $defer.onFailure(() => suspendVdi.$destroy())
+  }
+
+  // 1. Create the VM.
+  const vmRef = await xapi.VM_create(
+    {
+      ...vmRecord,
+      affinity: undefined,
+      blocked_operations: {
+        ...vmRecord.blocked_operations,
+        start: 'Importing…',
+        start_on: 'Importing…',
+      },
+      ha_always_run: false,
+      is_a_template: false,
+      name_label: '[Importing…] ' + vmRecord.name_label,
+      other_config: {
+        ...vmRecord.other_config,
+        [TAG_COPY_SRC]: vmRecord.uuid,
+      },
+    },
+    {
+      bios_strings: vmRecord.bios_strings,
+      generateMacSeed: newMacAddresses,
+      suspend_VDI: suspendVdi?.$ref,
+    }
+  )
+  $defer.onFailure.call(xapi, 'VM_destroy', vmRef)
+
+  // 2. Delete all VBDs which may have been created by the import.
+  await asyncMap(await xapi.getField('VM', vmRef, 'VBDs'), ref => ignoreErrors.call(xapi.call('VBD.destroy', ref)))
+
+  // 3. Create VDIs & VBDs.
+  const vbdRecords = deltaVm.vbds
+  const vbds = groupBy(vbdRecords, 'VDI')
+  const newVdis = {}
+  await asyncMap(Object.keys(vdiRecords), async vdiRef => {
+    const vdi = vdiRecords[vdiRef]
+    let newVdi
+
+    const remoteBaseVdiUuid = detectBase && vdi.other_config[TAG_BASE_DELTA]
+    if (remoteBaseVdiUuid) {
+      const baseVdi = find(baseVdis, vdi => vdi.other_config[TAG_COPY_SRC] === remoteBaseVdiUuid)
+      if (!baseVdi) {
+        throw new Error(`missing base VDI (copy of ${remoteBaseVdiUuid})`)
+      }
+
+      newVdi = await xapi.getRecord('VDI', await baseVdi.$clone())
+      $defer.onFailure(() => newVdi.$destroy())
+
+      await newVdi.update_other_config(TAG_COPY_SRC, vdi.uuid)
+    } else if (vdiRef === vmRecord.suspend_VDI) {
+      // suspendVDI has already created
+      newVdi = suspendVdi
+    } else {
+      newVdi = await xapi.getRecord(
+        'VDI',
+        await xapi.VDI_create({
+          ...vdi,
+          other_config: {
+            ...vdi.other_config,
+            [TAG_BASE_DELTA]: undefined,
+            [TAG_COPY_SRC]: vdi.uuid,
+          },
+          SR: mapVdisSrs[vdi.uuid] ?? sr.$ref,
+        })
+      )
+      $defer.onFailure(() => newVdi.$destroy())
+    }
+
+    const vdiVbds = vbds[vdiRef]
+    if (vdiVbds !== undefined) {
+      await asyncMap(Object.values(vdiVbds), vbd =>
+        xapi.VBD_create({
+          ...vbd,
+          VDI: newVdi.$ref,
+          VM: vmRef,
+        })
+      )
+    }
+
+    newVdis[vdiRef] = newVdi
+  })
+
+  const networksByNameLabelByVlan = {}
+  let defaultNetwork
+  Object.values(xapi.objects.all).forEach(object => {
+    if (object.$type === 'network') {
+      const pif = object.$PIFs[0]
+      if (pif === undefined) {
+        // ignore network
+        return
+      }
+      const vlan = pif.VLAN
+      const networksByNameLabel = networksByNameLabelByVlan[vlan] || (networksByNameLabelByVlan[vlan] = {})
+      defaultNetwork = networksByNameLabel[object.name_label] = object
+    }
+  })
+
+  const { streams } = deltaVm
+
+  await Promise.all([
+    // Import VDI contents.
+    cancelableMap(cancelToken, Object.entries(newVdis), async (cancelToken, [id, vdi]) => {
+      for (let stream of ensureArray(streams[`${id}.vhd`])) {
+        if (typeof stream === 'function') {
+          stream = await stream()
+        }
+        if (stream.length === undefined) {
+          stream = await createVhdStreamWithLength(stream)
+        }
+        await vdi.$importContent(stream, { cancelToken, format: 'vhd' })
+      }
+    }),
+
+    // Create VIFs.
+    asyncMap(Object.values(deltaVm.vifs), vif => {
+      let network = vif.$network$uuid && xapi.getObjectByUuid(vif.$network$uuid, undefined)
+
+      if (network === undefined) {
+        const { $network$VLAN: vlan = -1 } = vif
+        const networksByNameLabel = networksByNameLabelByVlan[vlan]
+        if (networksByNameLabel !== undefined) {
+          network = networksByNameLabel[vif.$network$name_label]
+          if (network === undefined) {
+            network = networksByNameLabel[Object.keys(networksByNameLabel)[0]]
+          }
+        } else {
+          network = defaultNetwork
+        }
+      }
+
+      if (network) {
+        return xapi.VIF_create(
+          {
+            ...vif,
+            network: network.$ref,
+            VM: vmRef,
+          },
+          {
+            MAC: newMacAddresses ? undefined : vif.MAC,
+          }
+        )
+      }
+    }),
+  ])
+
+  await Promise.all([
+    deltaVm.vm.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),
+    xapi.setField('VM', vmRef, 'name_label', deltaVm.vm.name_label),
+  ])
+
+  return vmRef
+})

package/_extractIdsFromSimplePattern.js

@@ -0,0 +1,29 @@
+exports.extractIdsFromSimplePattern = function extractIdsFromSimplePattern(pattern) {
+  if (pattern === undefined) {
+    return []
+  }
+
+  if (pattern !== null && typeof pattern === 'object') {
+    let keys = Object.keys(pattern)
+
+    if (keys.length === 1 && keys[0] === 'id') {
+      pattern = pattern.id
+      if (typeof pattern === 'string') {
+        return [pattern]
+      }
+      if (pattern !== null && typeof pattern === 'object') {
+        keys = Object.keys(pattern)
+        if (
+          keys.length === 1 &&
+          keys[0] === '__or' &&
+          Array.isArray((pattern = pattern.__or)) &&
+          pattern.every(_ => typeof _ === 'string')
+        ) {
+          return pattern
+        }
+      }
+    }
+  }
+
+  throw new Error('invalid pattern')
+}

package/_filenameDate.js

@@ -0,0 +1,6 @@
+const { utcFormat, utcParse } = require('d3-time-format')
+
+// Format a date in ISO 8601 in a safe way to be used in filenames
+// (even on Windows).
+exports.formatFilenameDate = utcFormat('%Y%m%dT%H%M%SZ')
+exports.parseFilenameDate = utcParse('%Y%m%dT%H%M%SZ')

package/_forkStreamUnpipe.js

@@ -0,0 +1,28 @@
+const eos = require('end-of-stream')
+const { PassThrough } = require('stream')
+
+// create a new readable stream from an existing one which may be piped later
+//
+// in case of error in the new readable stream, it will simply be unpiped
+// from the original one
+exports.forkStreamUnpipe = function forkStreamUnpipe(stream) {
+  const { forks = 0 } = stream
+  stream.forks = forks + 1
+
+  const proxy = new PassThrough()
+  stream.pipe(proxy)
+  eos(stream, error => {
+    if (error !== undefined) {
+      proxy.destroy(error)
+    }
+  })
+  eos(proxy, _ => {
+    stream.forks--
+    stream.unpipe(proxy)
+
+    if (stream.forks === 0) {
+      stream.destroy(new Error('no more consumers for this stream'))
+    }
+  })
+  return proxy
+}

package/_getOldEntries.js

@@ -0,0 +1,4 @@
+// returns all entries but the last retention-th
+exports.getOldEntries = function getOldEntries(retention, entries) {
+  return entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries
+}

package/_getTmpDir.js

@@ -0,0 +1,20 @@
+const Disposable = require('promise-toolbox/Disposable.js')
+const { join } = require('path')
+const { mkdir, rmdir } = require('fs-extra')
+const { tmpdir } = require('os')
+
+const MAX_ATTEMPTS = 3
+
+exports.getTmpDir = async function getTmpDir() {
+  for (let i = 0; true; ++i) {
+    const path = join(tmpdir(), Math.random().toString(36).slice(2))
+    try {
+      await mkdir(path)
+      return new Disposable(() => rmdir(path), path)
+    } catch (error) {
+      if (i === MAX_ATTEMPTS) {
+        throw error
+      }
+    }
+  }
+}

package/_getVmBackupDir.js

@@ -0,0 +1,6 @@
+const BACKUP_DIR = 'xo-vm-backups'
+exports.BACKUP_DIR = BACKUP_DIR
+
+exports.getVmBackupDir = function getVmBackupDir(uuid) {
+  return `${BACKUP_DIR}/${uuid}`
+}

package/_isValidXva.js

@@ -0,0 +1,60 @@
+const assert = require('assert')
+
+const isGzipFile = async (handler, fd) => {
+  // https://tools.ietf.org/html/rfc1952.html#page-5
+  const magicNumber = Buffer.allocUnsafe(2)
+
+  assert.strictEqual((await handler.read(fd, magicNumber, 0)).bytesRead, magicNumber.length)
+  return magicNumber[0] === 31 && magicNumber[1] === 139
+}
+
+// TODO: better check?
+//
+// our heuristic is not good enough, there has been some false positives
+// (detected as invalid by us but valid by `tar` and imported with success),
+// either THOUGH THEY MAY HAVE BEEN COMPRESSED FILES:
+// - these files were normal but the check is incorrect
+// - these files were invalid but without data loss
+// - these files were invalid but with silent data loss
+//
+// maybe reading the end of the file looking for a file named
+// /^Ref:\d+/\d+\.checksum$/ and then validating the tar structure from it
+//
+// https://github.com/npm/node-tar/issues/234#issuecomment-538190295
+const isValidTar = async (handler, size, fd) => {
+  if (size <= 1024 || size % 512 !== 0) {
+    return false
+  }
+
+  const buf = Buffer.allocUnsafe(1024)
+  assert.strictEqual((await handler.read(fd, buf, size - buf.length)).bytesRead, buf.length)
+  return buf.every(_ => _ === 0)
+}
+
+// TODO: find an heuristic for compressed files
+async function isValidXva(path) {
+  const handler = this._handler
+  try {
+    const fd = await handler.openFile(path, 'r')
+    try {
+      const size = await handler.getSize(fd)
+      if (size < 20) {
+        // neither a valid gzip not tar
+        return false
+      }
+
+      return (await isGzipFile(handler, fd))
+        ? true // gzip files cannot be validated at this time
+        : await isValidTar(handler, size, fd)
+    } finally {
+      handler.closeFile(fd).catch(noop)
+    }
+  } catch (error) {
+    // never throw, log and report as valid to avoid side effects
+    console.error('isValidXva', path, error)
+    return true
+  }
+}
+exports.isValidXva = isValidXva
+
+const noop = Function.prototype

package/_listPartitions.js

@@ -0,0 +1,52 @@
+const fromCallback = require('promise-toolbox/fromCallback.js')
+const { createLogger } = require('@xen-orchestra/log')
+const { createParser } = require('parse-pairs')
+const { execFile } = require('child_process')
+
+const { debug } = createLogger('xo:backups:listPartitions')
+
+const IGNORED_PARTITION_TYPES = {
+  // https://github.com/jhermsmeier/node-mbr/blob/master/lib/partition.js#L38
+  0x05: true,
+  0x0f: true,
+  0x15: true,
+  0x5e: true,
+  0x5f: true,
+  0x85: true,
+  0x91: true,
+  0x9b: true,
+  0xc5: true,
+  0xcf: true,
+  0xd5: true,
+
+  0x82: true, // swap
+}
+
+const LVM_PARTITION_TYPE = 0x8e
+exports.LVM_PARTITION_TYPE = LVM_PARTITION_TYPE
+
+const parsePartxLine = createParser({
+  keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
+  valueTransform: (value, key) => (key === 'start' || key === 'size' || key === 'type' ? +value : value),
+})
+
+// returns an empty array in case of a non-partitioned disk
+exports.listPartitions = async function listPartitions(devicePath) {
+  const parts = await fromCallback(execFile, 'partx', [
+    '--bytes',
+    '--output=NR,START,SIZE,NAME,UUID,TYPE',
+    '--pairs',
+    devicePath,
+  ]).catch(error => {
+    // partx returns 1 since v2.33 when failing to read partitions.
+    //
+    // Prior versions are correctly handled by the nominal case.
+    debug('listPartitions', { error })
+    return ''
+  })
+
+  return parts
+    .split(/\r?\n/)
+    .map(parsePartxLine)
+    .filter(({ type }) => type != null && !(type in IGNORED_PARTITION_TYPES))
+}

package/_lvm.js

@@ -0,0 +1,31 @@
+const fromCallback = require('promise-toolbox/fromCallback.js')
+const { createParser } = require('parse-pairs')
+const { execFile } = require('child_process')
+
+// ===================================================================
+
+const parse = createParser({
+  keyTransform: key => key.slice(5).toLowerCase(),
+})
+const makeFunction =
+  command =>
+  async (fields, ...args) => {
+    const info = await fromCallback(execFile, command, [
+      '--noheading',
+      '--nosuffix',
+      '--nameprefixes',
+      '--unbuffered',
+      '--units',
+      'b',
+      '-o',
+      String(fields),
+      ...args,
+    ])
+    return info
+      .trim()
+      .split(/\r?\n/)
+      .map(Array.isArray(fields) ? parse : line => parse(line)[fields])
+  }
+
+exports.lvs = makeFunction('lvs')
+exports.pvs = makeFunction('pvs')

package/_watchStreamSize.js

@@ -0,0 +1,7 @@
+exports.watchStreamSize = function watchStreamSize(stream, container = { size: 0 }) {
+  stream.on('data', data => {
+    container.size += data.length
+  })
+  stream.pause()
+  return container
+}

package/formatVmBackups.js

@@ -0,0 +1,34 @@
+const mapValues = require('lodash/mapValues.js')
+const { dirname } = require('path')
+
+function formatVmBackup(backup) {
+  return {
+    disks:
+      backup.vhds === undefined
+        ? []
+        : Object.keys(backup.vhds).map(vdiId => {
+            const vdi = backup.vdis[vdiId]
+            return {
+              id: `${dirname(backup._filename)}/${backup.vhds[vdiId]}`,
+              name: vdi.name_label,
+              uuid: vdi.uuid,
+            }
+          }),
+
+    id: backup.id,
+    jobId: backup.jobId,
+    mode: backup.mode,
+    scheduleId: backup.scheduleId,
+    size: backup.size,
+    timestamp: backup.timestamp,
+    vm: {
+      name_description: backup.vm.name_description,
+      name_label: backup.vm.name_label,
+    },
+  }
+}
+
+// format all backups as returned by RemoteAdapter#listAllVmBackups()
+exports.formatVmBackups = function formatVmBackups(backupsByVM) {
+  return mapValues(backupsByVM, backups => backups.map(formatVmBackup))
+}