@xen-orchestra/backups 0.14.0

package/_backupWorker.js

@@ -0,0 +1,155 @@
+require('@xen-orchestra/log/configure.js').catchGlobalErrors(
+  require('@xen-orchestra/log').createLogger('xo:backups:worker')
+)
+
+const Disposable = require('promise-toolbox/Disposable.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const { compose } = require('@vates/compose')
+const { createDebounceResource } = require('@vates/disposable/debounceResource.js')
+const { deduped } = require('@vates/disposable/deduped.js')
+const { getHandler } = require('@xen-orchestra/fs')
+const { parseDuration } = require('@vates/parse-duration')
+const { Xapi } = require('@xen-orchestra/xapi')
+
+const { Backup } = require('./Backup.js')
+const { RemoteAdapter } = require('./RemoteAdapter.js')
+const { Task } = require('./Task.js')
+
+class BackupWorker {
+  #config
+  #job
+  #recordToXapi
+  #remoteOptions
+  #remotes
+  #schedule
+  #xapiOptions
+  #xapis
+
+  constructor({ config, job, recordToXapi, remoteOptions, remotes, resourceCacheDelay, schedule, xapiOptions, xapis }) {
+    this.#config = config
+    this.#job = job
+    this.#recordToXapi = recordToXapi
+    this.#remoteOptions = remoteOptions
+    this.#remotes = remotes
+    this.#schedule = schedule
+    this.#xapiOptions = xapiOptions
+    this.#xapis = xapis
+
+    const debounceResource = createDebounceResource()
+    debounceResource.defaultDelay = parseDuration(resourceCacheDelay)
+    this.debounceResource = debounceResource
+  }
+
+  run() {
+    return new Backup({
+      config: this.#config,
+      getAdapter: remoteId => this.getAdapter(this.#remotes[remoteId]),
+      getConnectedRecord: Disposable.factory(async function* getConnectedRecord(type, uuid) {
+        const xapiId = this.#recordToXapi[uuid]
+        if (xapiId === undefined) {
+          throw new Error('no XAPI associated to ' + uuid)
+        }
+
+        const xapi = yield this.getXapi(this.#xapis[xapiId])
+        return xapi.getRecordByUuid(type, uuid)
+      }).bind(this),
+      job: this.#job,
+      schedule: this.#schedule,
+    }).run()
+  }
+
+  getAdapter = Disposable.factory(this.getAdapter)
+  getAdapter = deduped(this.getAdapter, remote => [remote.url])
+  getAdapter = compose(this.getAdapter, function (resource) {
+    return this.debounceResource(resource)
+  })
+  async *getAdapter(remote) {
+    const handler = getHandler(remote, this.#remoteOptions)
+    await handler.sync()
+    try {
+      yield new RemoteAdapter(handler, {
+        debounceResource: this.debounceResource,
+        dirMode: this.#config.dirMode,
+      })
+    } finally {
+      await handler.forget()
+    }
+  }
+
+  getXapi = Disposable.factory(this.getXapi)
+  getXapi = deduped(this.getXapi, ({ url }) => [url])
+  getXapi = compose(this.getXapi, function (resource) {
+    return this.debounceResource(resource)
+  })
+  async *getXapi({ credentials: { username: user, password }, ...opts }) {
+    const xapi = new Xapi({
+      ...this.#xapiOptions,
+      ...opts,
+      auth: {
+        user,
+        password,
+      },
+    })
+
+    await xapi.connect()
+    try {
+      await xapi.objectsFetched
+
+      yield xapi
+    } finally {
+      await xapi.disconnect()
+    }
+  }
+}
+
+// Received message:
+//
+// Message {
+//   action: 'run'
+//   data: object
+//   runWithLogs: boolean
+// }
+//
+// Sent message:
+//
+// Message {
+//   type: 'log' | 'result'
+//   data?: object
+//   status?: 'success' | 'failure'
+//   result?: any
+// }
+process.on('message', async message => {
+  if (message.action === 'run') {
+    const backupWorker = new BackupWorker(message.data)
+    try {
+      const result = message.runWithLogs
+        ? await Task.run(
+            {
+              name: 'backup run',
+              onLog: data =>
+                process.send({
+                  data,
+                  type: 'log',
+                }),
+            },
+            () => backupWorker.run()
+          )
+        : await backupWorker.run()
+
+      process.send({
+        type: 'result',
+        result,
+        status: 'success',
+      })
+    } catch (error) {
+      process.send({
+        type: 'result',
+        result: error,
+        status: 'failure',
+      })
+    } finally {
+      await ignoreErrors.call(backupWorker.debounceResource.flushAll())
+      process.disconnect()
+    }
+  }
+})

package/_cancelableMap.js

@@ -0,0 +1,20 @@
+const cancelable = require('promise-toolbox/cancelable.js')
+const CancelToken = require('promise-toolbox/CancelToken.js')
+
+// Similar to `Promise.all` + `map` but pass a cancel token to the callback
+//
+// If any of the executions fails, the cancel token will be triggered and the
+// first reason will be rejected.
+exports.cancelableMap = cancelable(async function cancelableMap($cancelToken, iterable, callback) {
+  const { cancel, token } = CancelToken.source([$cancelToken])
+  try {
+    return await Promise.all(
+      Array.from(iterable, function (item) {
+        return callback.call(this, token, item)
+      })
+    )
+  } catch (error) {
+    await cancel()
+    throw error
+  }
+})

package/_cleanVm.js

@@ -0,0 +1,378 @@
+const assert = require('assert')
+const sum = require('lodash/sum')
+const { asyncMap } = require('@xen-orchestra/async-map')
+const { default: Vhd, mergeVhd } = require('vhd-lib')
+const { dirname, resolve } = require('path')
+const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
+const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
+const { limitConcurrency } = require('limit-concurrency-decorator')
+
+const { Task } = require('./Task.js')
+
+// chain is an array of VHDs from child to parent
+//
+// the whole chain will be merged into parent, parent will be renamed to child
+// and all the others will deleted
+async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
+  assert(chain.length >= 2)
+
+  let child = chain[0]
+  const parent = chain[chain.length - 1]
+  const children = chain.slice(0, -1).reverse()
+
+  chain
+    .slice(1)
+    .reverse()
+    .forEach(parent => {
+      onLog(`the parent ${parent} of the child ${child} is unused`)
+    })
+
+  if (merge) {
+    // `mergeVhd` does not work with a stream, either
+    // - make it accept a stream
+    // - or create synthetic VHD which is not a stream
+    if (children.length !== 1) {
+      // TODO: implement merging multiple children
+      children.length = 1
+      child = children[0]
+    }
+
+    onLog(`merging ${child} into ${parent}`)
+
+    let done, total
+    const handle = setInterval(() => {
+      if (done !== undefined) {
+        onLog(`merging ${child}: ${done}/${total}`)
+      }
+    }, 10e3)
+
+    const mergedSize = await mergeVhd(
+      handler,
+      parent,
+      handler,
+      child,
+      // children.length === 1
+      //   ? child
+      //   : await createSyntheticStream(handler, children),
+      {
+        onProgress({ done: d, total: t }) {
+          done = d
+          total = t
+        },
+      }
+    )
+
+    clearInterval(handle)
+
+    await Promise.all([
+      handler.rename(parent, child),
+      asyncMap(children.slice(0, -1), child => {
+        onLog(`the VHD ${child} is unused`)
+        if (remove) {
+          onLog(`deleting unused VHD ${child}`)
+          return handler.unlink(child)
+        }
+      }),
+    ])
+
+    return mergedSize
+  }
+}
+
+const noop = Function.prototype
+
+const INTERRUPTED_VHDS_REG = /^(?:(.+)\/)?\.(.+)\.merge.json$/
+const listVhds = async (handler, vmDir) => {
+  const vhds = []
+  const interruptedVhds = new Set()
+
+  await asyncMap(
+    await handler.list(`${vmDir}/vdis`, {
+      ignoreMissing: true,
+      prependDir: true,
+    }),
+    async jobDir =>
+      asyncMap(
+        await handler.list(jobDir, {
+          prependDir: true,
+        }),
+        async vdiDir => {
+          const list = await handler.list(vdiDir, {
+            filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
+            prependDir: true,
+          })
+
+          list.forEach(file => {
+            const res = INTERRUPTED_VHDS_REG.exec(file)
+            if (res === null) {
+              vhds.push(file)
+            } else {
+              const [, dir, file] = res
+              interruptedVhds.add(`${dir}/${file}`)
+            }
+          })
+        }
+      )
+  )
+
+  return { vhds, interruptedVhds }
+}
+
+const defaultMergeLimiter = limitConcurrency(1)
+
+exports.cleanVm = async function cleanVm(
+  vmDir,
+  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, onLog = noop }
+) {
+  const handler = this._handler
+
+  const vhds = new Set()
+  const vhdParents = { __proto__: null }
+  const vhdChildren = { __proto__: null }
+
+  const vhdsList = await listVhds(handler, vmDir)
+
+  // remove broken VHDs
+  await asyncMap(vhdsList.vhds, async path => {
+    try {
+      const vhd = new Vhd(handler, path)
+      await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
+      vhds.add(path)
+      if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
+        const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
+        vhdParents[path] = parent
+        if (parent in vhdChildren) {
+          const error = new Error('this script does not support multiple VHD children')
+          error.parent = parent
+          error.child1 = vhdChildren[parent]
+          error.child2 = path
+          throw error // should we throw?
+        }
+        vhdChildren[parent] = path
+      }
+    } catch (error) {
+      onLog(`error while checking the VHD with path ${path}`, { error })
+      if (error?.code === 'ERR_ASSERTION' && remove) {
+        onLog(`deleting broken ${path}`)
+        await handler.unlink(path)
+      }
+    }
+  })
+
+  // remove VHDs with missing ancestors
+  {
+    const deletions = []
+
+    // return true if the VHD has been deleted or is missing
+    const deleteIfOrphan = vhd => {
+      const parent = vhdParents[vhd]
+      if (parent === undefined) {
+        return
+      }
+
+      // no longer needs to be checked
+      delete vhdParents[vhd]
+
+      deleteIfOrphan(parent)
+
+      if (!vhds.has(parent)) {
+        vhds.delete(vhd)
+
+        onLog(`the parent ${parent} of the VHD ${vhd} is missing`)
+        if (remove) {
+          onLog(`deleting orphan VHD ${vhd}`)
+          deletions.push(handler.unlink(vhd))
+        }
+      }
+    }
+
+    // > A property that is deleted before it has been visited will not be
+    // > visited later.
+    // >
+    // > -- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for...in#Deleted_added_or_modified_properties
+    for (const child in vhdParents) {
+      deleteIfOrphan(child)
+    }
+
+    await Promise.all(deletions)
+  }
+
+  const jsons = []
+  const xvas = new Set()
+  const xvaSums = []
+  const entries = await handler.list(vmDir, {
+    prependDir: true,
+  })
+  entries.forEach(path => {
+    if (isMetadataFile(path)) {
+      jsons.push(path)
+    } else if (isXvaFile(path)) {
+      xvas.add(path)
+    } else if (isXvaSumFile(path)) {
+      xvaSums.push(path)
+    }
+  })
+
+  await asyncMap(xvas, async path => {
+    // check is not good enough to delete the file, the best we can do is report
+    // it
+    if (!(await this.isValidXva(path))) {
+      onLog(`the XVA with path ${path} is potentially broken`)
+    }
+  })
+
+  const unusedVhds = new Set(vhds)
+  const unusedXvas = new Set(xvas)
+
+  // compile the list of unused XVAs and VHDs, and remove backup metadata which
+  // reference a missing XVA/VHD
+  await asyncMap(jsons, async json => {
+    const metadata = JSON.parse(await handler.readFile(json))
+    const { mode } = metadata
+    let size
+    if (mode === 'full') {
+      const linkedXva = resolve('/', vmDir, metadata.xva)
+
+      if (xvas.has(linkedXva)) {
+        unusedXvas.delete(linkedXva)
+
+        size = await handler.getSize(linkedXva).catch(error => {
+          onLog(`failed to get size of ${json}`, { error })
+        })
+      } else {
+        onLog(`the XVA linked to the metadata ${json} is missing`)
+        if (remove) {
+          onLog(`deleting incomplete backup ${json}`)
+          await handler.unlink(json)
+        }
+      }
+    } else if (mode === 'delta') {
+      const linkedVhds = (() => {
+        const { vhds } = metadata
+        return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
+      })()
+
+      // FIXME: find better approach by keeping as much of the backup as
+      // possible (existing disks) even if one disk is missing
+      if (linkedVhds.every(_ => vhds.has(_))) {
+        linkedVhds.forEach(_ => unusedVhds.delete(_))
+
+        size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
+          onLog(`failed to get size of ${json}`, { error })
+        })
+      } else {
+        onLog(`Some VHDs linked to the metadata ${json} are missing`)
+        if (remove) {
+          onLog(`deleting incomplete backup ${json}`)
+          await handler.unlink(json)
+        }
+      }
+    }
+
+    const metadataSize = metadata.size
+    if (size !== undefined && metadataSize !== size) {
+      onLog(`incorrect size in metadata: ${metadataSize ?? 'none'} instead of ${size}`)
+
+      // don't update if the the stored size is greater than found files,
+      // it can indicates a problem
+      if (fixMetadata && (metadataSize === undefined || metadataSize < size)) {
+        try {
+          metadata.size = size
+          await handler.writeFile(json, JSON.stringify(metadata), { flags: 'w' })
+        } catch (error) {
+          onLog(`failed to update size in backup metadata ${json}`, { error })
+        }
+      }
+    }
+  })
+
+  // TODO: parallelize by vm/job/vdi
+  const unusedVhdsDeletion = []
+  const toMerge = []
+  {
+    // VHD chains (as list from child to ancestor) to merge indexed by last
+    // ancestor
+    const vhdChainsToMerge = { __proto__: null }
+
+    const toCheck = new Set(unusedVhds)
+
+    const getUsedChildChainOrDelete = vhd => {
+      if (vhd in vhdChainsToMerge) {
+        const chain = vhdChainsToMerge[vhd]
+        delete vhdChainsToMerge[vhd]
+        return chain
+      }
+
+      if (!unusedVhds.has(vhd)) {
+        return [vhd]
+      }
+
+      // no longer needs to be checked
+      toCheck.delete(vhd)
+
+      const child = vhdChildren[vhd]
+      if (child !== undefined) {
+        const chain = getUsedChildChainOrDelete(child)
+        if (chain !== undefined) {
+          chain.push(vhd)
+          return chain
+        }
+      }
+
+      onLog(`the VHD ${vhd} is unused`)
+      if (remove) {
+        onLog(`deleting unused VHD ${vhd}`)
+        unusedVhdsDeletion.push(handler.unlink(vhd))
+      }
+    }
+
+    toCheck.forEach(vhd => {
+      vhdChainsToMerge[vhd] = getUsedChildChainOrDelete(vhd)
+    })
+
+    // merge interrupted VHDs
+    vhdsList.interruptedVhds.forEach(parent => {
+      vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
+    })
+
+    Object.values(vhdChainsToMerge).forEach(chain => {
+      if (chain !== undefined) {
+        toMerge.push(chain)
+      }
+    })
+  }
+
+  const doMerge = () => {
+    const promise = asyncMap(toMerge, async chain => {
+      mergeVhdChain(chain, { handler, onLog, remove, merge })
+    })
+    return merge ? promise.then(sizes => ({ size: sum(sizes) })) : promise
+  }
+
+  await Promise.all([
+    ...unusedVhdsDeletion,
+    toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
+    asyncMap(unusedXvas, path => {
+      onLog(`the XVA ${path} is unused`)
+      if (remove) {
+        onLog(`deleting unused XVA ${path}`)
+        return handler.unlink(path)
+      }
+    }),
+    asyncMap(xvaSums, path => {
+      // no need to handle checksums for XVAs deleted by the script, they will be handled by `unlink()`
+      if (!xvas.has(path.slice(0, -'.checksum'.length))) {
+        onLog(`the XVA checksum ${path} is unused`)
+        if (remove) {
+          onLog(`deleting unused XVA checksum ${path}`)
+          return handler.unlink(path)
+        }
+      }
+    }),
+  ])
+
+  return {
+    // boolean whether some VHDs were merged (or should be merged)
+    merge: toMerge.length !== 0,
+  }
+}