@xen-orchestra/backups 0.29.4 → 0.29.6

package/Backup.js

@@ -38,7 +38,7 @@ const DEFAULT_VM_SETTINGS = {
   fullInterval: 0,
   healthCheckSr: undefined,
   healthCheckVmsWithTags: [],
-  maxMergedDeltasPerRun: 2,
+  maxMergedDeltasPerRun: Infinity,
   offlineBackup: false,
   offlineSnapshot: false,
   snapshotRetention: 0,

package/README.md

@@ -8,8 +8,8 @@
 
 Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/backups):
 
-```
-> npm install --save @xen-orchestra/backups
+```sh
+npm install --save @xen-orchestra/backups
 ```
 
 ## Contributions

package/RemoteAdapter.js

@@ -28,6 +28,7 @@ const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
+const { watchStreamSize } = require('./_watchStreamSize')
 // @todo : this import is marked extraneous , sould be fixed when lib is published
 const { mount } = require('@vates/fuse-vhd')
 const { asyncEach } = require('@vates/async-each')
@@ -661,7 +662,7 @@ class RemoteAdapter {
     const handler = this._handler
     if (this.#useVhdDirectory()) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
-      await createVhdDirectoryFromStream(handler, dataPath, input, {
+      const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
         concurrency: writeBlockConcurrency,
         compression: this.#getCompressionType(),
         async validator() {
@@ -671,12 +672,14 @@ class RemoteAdapter {
         nbdClient,
       })
       await VhdAbstract.createAlias(handler, path, dataPath)
+      return size
     } else {
-      await this.outputStream(path, input, { checksum, validator })
+      return this.outputStream(path, input, { checksum, validator })
     }
   }
 
   async outputStream(path, input, { checksum = true, validator = noop } = {}) {
+    const container = watchStreamSize(input)
     await this._handler.outputStream(path, input, {
       checksum,
       dirMode: this._dirMode,
@@ -685,6 +688,7 @@ class RemoteAdapter {
         return validator.apply(this, arguments)
       },
     })
+    return container.size
   }
 
   // open the  hierarchy of ancestors until we find a full one

package/Task.js

@@ -100,7 +100,7 @@ class Task {
    * In case of error, the task will be failed.
    *
    * @typedef Result
-   * @param {() => Result)} fn
+   * @param {() => Result} fn
    * @param {boolean} last - Whether the task should succeed if there is no error
    * @returns Result
    */

package/_backupWorker.js

@@ -1,6 +1,6 @@
 'use strict'
 
-require('@xen-orchestra/log/configure.js').catchGlobalErrors(
+require('@xen-orchestra/log/configure').catchGlobalErrors(
   require('@xen-orchestra/log').createLogger('xo:backups:worker')
 )
 

package/_deltaVm.js

@@ -258,6 +258,9 @@ exports.importDeltaVm = defer(async function importDeltaVm(
       $defer.onFailure(() => newVdi.$destroy())
 
       await newVdi.update_other_config(TAG_COPY_SRC, vdi.uuid)
+      if (vdi.virtual_size > newVdi.virtual_size) {
+        await newVdi.$callAsync('resize', vdi.virtual_size)
+      }
     } else if (vdiRef === vmRecord.suspend_VDI) {
       // suspendVDI has already created
       newVdi = suspendVdi

package/_forkStreamUnpipe.js

@@ -1,7 +1,6 @@
 'use strict'
 
-const eos = require('end-of-stream')
-const { PassThrough } = require('stream')
+const { finished, PassThrough } = require('node:stream')
 
 const { debug } = require('@xen-orchestra/log').createLogger('xo:backups:forkStreamUnpipe')
 
@@ -9,29 +8,29 @@ const { debug } = require('@xen-orchestra/log').createLogger('xo:backups:forkStr
 //
 // in case of error in the new readable stream, it will simply be unpiped
 // from the original one
-exports.forkStreamUnpipe = function forkStreamUnpipe(stream) {
-  const { forks = 0 } = stream
-  stream.forks = forks + 1
+exports.forkStreamUnpipe = function forkStreamUnpipe(source) {
+  const { forks = 0 } = source
+  source.forks = forks + 1
 
-  debug('forking', { forks: stream.forks })
+  debug('forking', { forks: source.forks })
 
-  const proxy = new PassThrough()
-  stream.pipe(proxy)
-  eos(stream, error => {
+  const fork = new PassThrough()
+  source.pipe(fork)
+  finished(source, { writable: false }, error => {
     if (error !== undefined) {
       debug('error on original stream, destroying fork', { error })
-      proxy.destroy(error)
+      fork.destroy(error)
     }
   })
-  eos(proxy, error => {
-    debug('end of stream, unpiping', { error, forks: --stream.forks })
+  finished(fork, { readable: false }, error => {
+    debug('end of stream, unpiping', { error, forks: --source.forks })
 
-    stream.unpipe(proxy)
+    source.unpipe(fork)
 
-    if (stream.forks === 0) {
+    if (source.forks === 0) {
       debug('no more forks, destroying original stream')
-      stream.destroy(new Error('no more consumers for this stream'))
+      source.destroy(new Error('no more consumers for this stream'))
     }
   })
-  return proxy
+  return fork
 }

package/merge-worker/cli.js

@@ -4,7 +4,7 @@
 
 'use strict'
 
-const { catchGlobalErrors } = require('@xen-orchestra/log/configure.js')
+const { catchGlobalErrors } = require('@xen-orchestra/log/configure')
 const { createLogger } = require('@xen-orchestra/log')
 const { getSyncedHandler } = require('@xen-orchestra/fs')
 const { join } = require('path')

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.29.4",
+  "version": "0.29.6",
   "engines": {
     "node": ">=14.6"
   },
@@ -21,38 +21,37 @@
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
-    "@vates/disposable": "^0.1.3",
+    "@vates/disposable": "^0.1.4",
     "@vates/fuse-vhd": "^1.0.0",
-    "@vates/nbd-client": "*",
+    "@vates/nbd-client": "^1.0.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^3.3.0",
-    "@xen-orchestra/log": "^0.5.0",
+    "@xen-orchestra/fs": "^3.3.1",
+    "@xen-orchestra/log": "^0.6.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^5.0.1",
     "d3-time-format": "^3.0.0",
     "decorator-synchronized": "^0.6.0",
-    "end-of-stream": "^1.4.4",
-    "fs-extra": "^10.0.0",
+    "fs-extra": "^11.1.0",
     "golike-defer": "^0.5.1",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.20",
     "node-zone": "^0.4.0",
-    "parse-pairs": "^1.1.0",
+    "parse-pairs": "^2.0.0",
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
     "uuid": "^9.0.0",
-    "vhd-lib": "^4.2.0",
+    "vhd-lib": "^4.2.1",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
-    "rimraf": "^3.0.2",
-    "sinon": "^14.0.1",
+    "rimraf": "^4.1.1",
+    "sinon": "^15.0.1",
     "test": "^3.2.1",
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^1.6.0"
+    "@xen-orchestra/xapi": "^1.6.1"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/writers/DeltaBackupWriter.js

@@ -7,6 +7,8 @@ const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
+const { decorateClass } = require('@vates/decorate-with')
+const { defer } = require('golike-defer')
 const { dirname } = require('path')
 
 const { formatFilenameDate } = require('../_filenameDate.js')
@@ -20,9 +22,9 @@ const { packUuid } = require('./_packUuid.js')
 const { Disposable } = require('promise-toolbox')
 const NbdClient = require('@vates/nbd-client')
 
-const { debug, warn } = createLogger('xo:backups:DeltaBackupWriter')
+const { debug, warn, info } = createLogger('xo:backups:DeltaBackupWriter')
 
-exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
+class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
   async checkBaseVdis(baseUuidToSrcVdi) {
     const { handler } = this._adapter
     const backup = this._backup
@@ -133,7 +135,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     }
   }
 
-  async _transfer({ timestamp, deltaExport, sizeContainers }) {
+  async _transfer($defer, { timestamp, deltaExport }) {
     const adapter = this._adapter
     const backup = this._backup
 
@@ -172,6 +174,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     }
 
     const { size } = await Task.run({ name: 'transfer' }, async () => {
+      let transferSize = 0
       await Promise.all(
         map(deltaExport.vdis, async (vdi, id) => {
           const path = `${this._vmBackupDir}/${vhds[id]}`
@@ -200,21 +203,30 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           const vdiRef = vm.$xapi.getObject(vdi.uuid).$ref
 
           let nbdClient
-          if (!this._backup.config.useNbd) {
+          if (this._backup.config.useNbd) {
+            debug('useNbd is enabled', { vdi: id, path })
             // get nbd if possible
             try {
               // this will always take the first host in the list
               const [nbdInfo] = await vm.$xapi.call('VDI.get_nbd_info', vdiRef)
+              debug('got NBD info', { nbdInfo, vdi: id, path })
               nbdClient = new NbdClient(nbdInfo)
               await nbdClient.connect()
-              debug(`got nbd connection `, { vdi: vdi.uuid })
+
+              // this will inform the xapi that we don't need this anymore
+              // and will detach the vdi from dom0
+              $defer(() => nbdClient.disconnect())
+
+              info('NBD client ready', { vdi: id, path })
             } catch (error) {
               nbdClient = undefined
-              debug(`can't connect to nbd server or no server available`, { error })
+              warn('error connecting to NBD server', { error, vdi: id, path })
             }
+          } else {
+            debug('useNbd is disabled', { vdi: id, path })
           }
 
-          await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
+          transferSize += await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
@@ -235,9 +247,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           })
         })
       )
-      return {
-        size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0),
-      }
+      return { size: transferSize }
     })
     metadataContent.size = size
     this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
@@ -245,3 +255,6 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     // TODO: run cleanup?
   }
 }
+exports.DeltaBackupWriter = decorateClass(DeltaBackupWriter, {
+  _transfer: defer,
+})