@xen-orchestra/backups 0.33.0 → 0.35.0

package/Backup.js

@@ -49,6 +49,7 @@ const DEFAULT_VM_SETTINGS = {
   timeout: 0,
   useNbd: false,
   unconditionalSnapshot: false,
+  validateVhdStreams: false,
   vmTimeout: 0,
 }
 

package/RemoteAdapter.js

@@ -10,14 +10,7 @@ const groupBy = require('lodash/groupBy.js')
 const pickBy = require('lodash/pickBy.js')
 const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const {
-  createVhdDirectoryFromStream,
-  createVhdStreamWithLength,
-  openVhd,
-  VhdAbstract,
-  VhdDirectory,
-  VhdSynthetic,
-} = require('vhd-lib')
+const { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { decorateMethodsWith } = require('@vates/decorate-with')
 const { compose } = require('@vates/compose')
@@ -39,7 +32,6 @@ const { watchStreamSize } = require('./_watchStreamSize')
 // @todo : this import is marked extraneous , sould be fixed when lib is published
 const { mount } = require('@vates/fuse-vhd')
 const { asyncEach } = require('@vates/async-each')
-const { strictEqual } = require('assert')
 
 const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
@@ -666,7 +658,7 @@ class RemoteAdapter {
     return path
   }
 
-  async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency, nbdClient } = {}) {
+  async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency } = {}) {
     const handler = this._handler
     if (this.useVhdDirectory()) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
@@ -677,42 +669,21 @@ class RemoteAdapter {
           await input.task
           return validator.apply(this, arguments)
         },
-        nbdClient,
       })
       await VhdAbstract.createAlias(handler, path, dataPath)
       return size
     } else {
-      const inputWithSize = await createVhdStreamWithLength(input)
-      return this.outputStream(path, inputWithSize, { checksum, validator, expectedSize: inputWithSize.length })
+      return this.outputStream(path, input, { checksum, validator })
     }
   }
 
-  async outputStream(path, input, { checksum = true, validator = noop, expectedSize } = {}) {
+  async outputStream(path, input, { checksum = true, validator = noop } = {}) {
     const container = watchStreamSize(input)
-
     await this._handler.outputStream(path, input, {
       checksum,
       dirMode: this._dirMode,
       async validator() {
         await input.task
-        if (expectedSize !== undefined) {
-          // check that we read all the stream
-          strictEqual(
-            container.size,
-            expectedSize,
-            `transferred size ${container.size}, expected file size : ${expectedSize}`
-          )
-        }
-        let size
-        try {
-          size = await this._handler.getSize(path)
-        } catch (err) {
-          // can fail is the remote is encrypted
-        }
-        if (size !== undefined) {
-          // check that everything is written to disk
-          strictEqual(size, container.size, `written size ${size}, transfered size : ${container.size}`)
-        }
         return validator.apply(this, arguments)
       },
     })
@@ -748,7 +719,7 @@ class RemoteAdapter {
 
   async readDeltaVmBackup(metadata, ignoredVdis) {
     const handler = this._handler
-    const { vbds, vhds, vifs, vm } = metadata
+    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
     const dir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
 
@@ -763,7 +734,7 @@ class RemoteAdapter {
       vdis,
       version: '1.0.0',
       vifs,
-      vm,
+      vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
     }
   }
 
@@ -775,7 +746,49 @@ class RemoteAdapter {
     // _filename is a private field used to compute the backup id
     //
     // it's enumerable to make it cacheable
-    return { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
+    const metadata = { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
+
+    // backups created on XenServer < 7.1 via JSON in XML-RPC transports have boolean values encoded as integers, which make them unusable with more recent XAPIs
+    if (typeof metadata.vm.is_a_template === 'number') {
+      const properties = {
+        vbds: ['bootable', 'unpluggable', 'storage_lock', 'empty', 'currently_attached'],
+        vdis: [
+          'sharable',
+          'read_only',
+          'storage_lock',
+          'managed',
+          'missing',
+          'is_a_snapshot',
+          'allow_caching',
+          'metadata_latest',
+        ],
+        vifs: ['currently_attached', 'MAC_autogenerated'],
+        vm: ['is_a_template', 'is_control_domain', 'ha_always_run', 'is_a_snapshot', 'is_snapshot_from_vmpp'],
+        vmSnapshot: ['is_a_template', 'is_control_domain', 'ha_always_run', 'is_snapshot_from_vmpp'],
+      }
+
+      function fixBooleans(obj, properties) {
+        properties.forEach(property => {
+          if (typeof obj[property] === 'number') {
+            obj[property] = obj[property] === 1
+          }
+        })
+      }
+
+      for (const [key, propertiesInKey] of Object.entries(properties)) {
+        const value = metadata[key]
+        if (value !== undefined) {
+          // some properties of the metadata are collections indexed by the opaqueRef
+          const isCollection = Object.keys(value).some(subKey => subKey.startsWith('OpaqueRef:'))
+          if (isCollection) {
+            Object.values(value).forEach(subValue => fixBooleans(subValue, propertiesInKey))
+          } else {
+            fixBooleans(value, propertiesInKey)
+          }
+        }
+      }
+    }
+    return metadata
   }
 }
 

package/_VmBackup.js

@@ -6,11 +6,13 @@ const groupBy = require('lodash/groupBy.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const keyBy = require('lodash/keyBy.js')
 const mapValues = require('lodash/mapValues.js')
+const vhdStreamValidator = require('vhd-lib/vhdStreamValidator.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
 const { decorateMethodsWith } = require('@vates/decorate-with')
 const { defer } = require('golike-defer')
 const { formatDateTime } = require('@xen-orchestra/xapi')
+const { pipeline } = require('node:stream')
 
 const { DeltaBackupWriter } = require('./writers/DeltaBackupWriter.js')
 const { DeltaReplicationWriter } = require('./writers/DeltaReplicationWriter.js')
@@ -44,6 +46,8 @@ const forkDeltaExport = deltaExport =>
     },
   })
 
+const noop = Function.prototype
+
 class VmBackup {
   constructor({
     config,
@@ -245,7 +249,17 @@ class VmBackup {
     const deltaExport = await exportDeltaVm(exportedVm, baseVm, {
       fullVdisRequired,
     })
+    // since NBD is network based, if one disk use nbd , all the disk use them
+    // except the suspended VDI
+    if (Object.values(deltaExport.streams).some(({ _nbd }) => _nbd)) {
+      Task.info('Transfer data using NBD')
+    }
     const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
+
+    if (this._settings.validateVhdStreams) {
+      deltaExport.streams = mapValues(deltaExport.streams, stream => pipeline(stream, vhdStreamValidator, noop))
+    }
+
     deltaExport.streams = mapValues(deltaExport.streams, this._throttleStream)
 
     const timestamp = Date.now()

package/_deltaVm.js

@@ -187,11 +187,11 @@ exports.importDeltaVm = defer(async function importDeltaVm(
 
   // 0. Create suspend_VDI
   let suspendVdi
-  if (vmRecord.power_state === 'Suspended') {
+  if (vmRecord.suspend_VDI !== undefined && vmRecord.suspend_VDI !== 'OpaqueRef:NULL') {
     const vdi = vdiRecords[vmRecord.suspend_VDI]
     if (vdi === undefined) {
       Task.warning('Suspend VDI not available for this suspended VM', {
-        vm: pick(vmRecord, 'uuid', 'name_label'),
+        vm: pick(vmRecord, 'uuid', 'name_label', 'suspend_VDI'),
       })
     } else {
       suspendVdi = await xapi.getRecord(

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.33.0",
+  "version": "0.35.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -24,7 +24,7 @@
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.4",
     "@vates/fuse-vhd": "^1.0.0",
-    "@vates/nbd-client": "^1.0.1",
+    "@vates/nbd-client": "^1.2.0",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/fs": "^3.3.4",
@@ -42,7 +42,7 @@
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
     "uuid": "^9.0.0",
-    "vhd-lib": "^4.2.1",
+    "vhd-lib": "^4.4.0",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -52,7 +52,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^2.0.0"
+    "@xen-orchestra/xapi": "^2.2.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/writers/DeltaBackupWriter.js

@@ -20,9 +20,8 @@ const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
 const { Disposable } = require('promise-toolbox')
-const NbdClient = require('@vates/nbd-client')
 
-const { debug, warn, info } = createLogger('xo:backups:DeltaBackupWriter')
+const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
 class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
   async checkBaseVdis(baseUuidToSrcVdi) {
@@ -200,41 +199,12 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
             await checkVhd(handler, parentPath)
           }
 
-          const vdiRef = vm.$xapi.getObject(vdi.uuid).$ref
-
-          let nbdClient
-          if (this._backup.config.useNbd && adapter.useVhdDirectory()) {
-            debug('useNbd is enabled', { vdi: id, path })
-            // get nbd if possible
-            try {
-              // this will always take the first host in the list
-              const [nbdInfo] = await vm.$xapi.call('VDI.get_nbd_info', vdiRef)
-              debug('got NBD info', { nbdInfo, vdi: id, path })
-              nbdClient = new NbdClient(nbdInfo)
-              await nbdClient.connect()
-
-              // this will inform the xapi that we don't need this anymore
-              // and will detach the vdi from dom0
-              $defer(() => nbdClient.disconnect())
-
-              info('NBD client ready', { vdi: id, path })
-              Task.info('NBD used')
-            } catch (error) {
-              Task.warning('NBD configured but unusable', { error })
-              nbdClient = undefined
-              warn('error connecting to NBD server', { error, vdi: id, path })
-            }
-          } else {
-            debug('useNbd is disabled', { vdi: id, path })
-          }
-
           transferSize += await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
             writeBlockConcurrency: this._backup.config.writeBlockConcurrency,
-            nbdClient,
           })
 
           if (isDelta) {