@xen-orchestra/backups 0.29.3 → 0.29.5

package/Backup.js

@@ -38,7 +38,7 @@ const DEFAULT_VM_SETTINGS = {
   fullInterval: 0,
   healthCheckSr: undefined,
   healthCheckVmsWithTags: [],
-  maxMergedDeltasPerRun: 2,
+  maxMergedDeltasPerRun: Infinity,
   offlineBackup: false,
   offlineSnapshot: false,
   snapshotRetention: 0,

package/RemoteAdapter.js

@@ -28,6 +28,7 @@ const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
+const { watchStreamSize } = require('./_watchStreamSize')
 // @todo : this import is marked extraneous , sould be fixed when lib is published
 const { mount } = require('@vates/fuse-vhd')
 const { asyncEach } = require('@vates/async-each')
@@ -661,7 +662,7 @@ class RemoteAdapter {
     const handler = this._handler
     if (this.#useVhdDirectory()) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
-      await createVhdDirectoryFromStream(handler, dataPath, input, {
+      const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
         concurrency: writeBlockConcurrency,
         compression: this.#getCompressionType(),
         async validator() {
@@ -671,12 +672,14 @@ class RemoteAdapter {
         nbdClient,
       })
       await VhdAbstract.createAlias(handler, path, dataPath)
+      return size
     } else {
-      await this.outputStream(path, input, { checksum, validator })
+      return this.outputStream(path, input, { checksum, validator })
     }
   }
 
   async outputStream(path, input, { checksum = true, validator = noop } = {}) {
+    const container = watchStreamSize(input)
     await this._handler.outputStream(path, input, {
       checksum,
       dirMode: this._dirMode,
@@ -685,6 +688,7 @@ class RemoteAdapter {
         return validator.apply(this, arguments)
       },
     })
+    return container.size
   }
 
   // open the  hierarchy of ancestors until we find a full one

package/Task.js

@@ -100,7 +100,7 @@ class Task {
    * In case of error, the task will be failed.
    *
    * @typedef Result
-   * @param {() => Result)} fn
+   * @param {() => Result} fn
    * @param {boolean} last - Whether the task should succeed if there is no error
    * @returns Result
    */

package/_backupWorker.js

@@ -1,6 +1,6 @@
 'use strict'
 
-require('@xen-orchestra/log/configure.js').catchGlobalErrors(
+require('@xen-orchestra/log/configure').catchGlobalErrors(
   require('@xen-orchestra/log').createLogger('xo:backups:worker')
 )
 

package/merge-worker/cli.js

@@ -4,7 +4,7 @@
 
 'use strict'
 
-const { catchGlobalErrors } = require('@xen-orchestra/log/configure.js')
+const { catchGlobalErrors } = require('@xen-orchestra/log/configure')
 const { createLogger } = require('@xen-orchestra/log')
 const { getSyncedHandler } = require('@xen-orchestra/fs')
 const { join } = require('path')

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.29.3",
+  "version": "0.29.5",
   "engines": {
     "node": ">=14.6"
   },
@@ -21,38 +21,38 @@
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
-    "@vates/disposable": "^0.1.3",
+    "@vates/disposable": "^0.1.4",
     "@vates/fuse-vhd": "^1.0.0",
     "@vates/nbd-client": "*",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^3.3.0",
-    "@xen-orchestra/log": "^0.5.0",
+    "@xen-orchestra/fs": "^3.3.1",
+    "@xen-orchestra/log": "^0.6.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^5.0.1",
     "d3-time-format": "^3.0.0",
     "decorator-synchronized": "^0.6.0",
     "end-of-stream": "^1.4.4",
-    "fs-extra": "^10.0.0",
+    "fs-extra": "^11.1.0",
     "golike-defer": "^0.5.1",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.20",
     "node-zone": "^0.4.0",
-    "parse-pairs": "^1.1.0",
+    "parse-pairs": "^2.0.0",
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
     "uuid": "^9.0.0",
-    "vhd-lib": "^4.2.0",
+    "vhd-lib": "^4.2.1",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
-    "rimraf": "^3.0.2",
-    "sinon": "^14.0.1",
+    "rimraf": "^4.1.1",
+    "sinon": "^15.0.1",
     "test": "^3.2.1",
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^1.5.3"
+    "@xen-orchestra/xapi": "^1.6.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

package/writers/DeltaBackupWriter.js

@@ -11,7 +11,6 @@ const { dirname } = require('path')
 
 const { formatFilenameDate } = require('../_filenameDate.js')
 const { getOldEntries } = require('../_getOldEntries.js')
-const { getVmBackupDir } = require('../_getVmBackupDir.js')
 const { Task } = require('../Task.js')
 
 const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
@@ -21,7 +20,7 @@ const { packUuid } = require('./_packUuid.js')
 const { Disposable } = require('promise-toolbox')
 const NbdClient = require('@vates/nbd-client')
 
-const { debug, warn } = createLogger('xo:backups:DeltaBackupWriter')
+const { debug, warn, info } = createLogger('xo:backups:DeltaBackupWriter')
 
 exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
   async checkBaseVdis(baseUuidToSrcVdi) {
@@ -29,8 +28,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     const backup = this._backup
     const adapter = this._adapter
 
-    const backupDir = getVmBackupDir(backup.vm.uuid)
-    const vdisDir = `${backupDir}/vdis/${backup.job.id}`
+    const vdisDir = `${this._vmBackupDir}/vdis/${backup.job.id}`
 
     await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdi]) => {
       let found = false
@@ -135,7 +133,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     }
   }
 
-  async _transfer({ timestamp, deltaExport, sizeContainers }) {
+  async _transfer({ timestamp, deltaExport }) {
     const adapter = this._adapter
     const backup = this._backup
 
@@ -143,7 +141,6 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
 
     const jobId = job.id
     const handler = adapter.handler
-    const backupDir = getVmBackupDir(vm.uuid)
 
     // TODO: clean VM backup directory
 
@@ -175,9 +172,10 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     }
 
     const { size } = await Task.run({ name: 'transfer' }, async () => {
+      let transferSize = 0
       await Promise.all(
         map(deltaExport.vdis, async (vdi, id) => {
-          const path = `${backupDir}/${vhds[id]}`
+          const path = `${this._vmBackupDir}/${vhds[id]}`
 
           const isDelta = vdi.other_config['xo:base_delta'] !== undefined
           let parentPath
@@ -203,21 +201,25 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           const vdiRef = vm.$xapi.getObject(vdi.uuid).$ref
 
           let nbdClient
-          if (!this._backup.config.useNbd) {
+          if (this._backup.config.useNbd) {
+            debug('useNbd is enabled', { vdi: id, path })
             // get nbd if possible
             try {
               // this will always take the first host in the list
               const [nbdInfo] = await vm.$xapi.call('VDI.get_nbd_info', vdiRef)
+              debug('got NBD info', { nbdInfo, vdi: id, path })
               nbdClient = new NbdClient(nbdInfo)
               await nbdClient.connect()
-              debug(`got nbd connection `, { vdi: vdi.uuid })
+              info('NBD client ready', { vdi: id, path })
             } catch (error) {
               nbdClient = undefined
-              debug(`can't connect to nbd server or no server available`, { error })
+              warn('error connecting to NBD server', { error, vdi: id, path })
             }
+          } else {
+            debug('useNbd is disabled', { vdi: id, path })
           }
 
-          await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
+          transferSize += await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
@@ -238,9 +240,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           })
         })
       )
-      return {
-        size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0),
-      }
+      return { size: transferSize }
     })
     metadataContent.size = size
     this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)

package/writers/FullBackupWriter.js

@@ -2,7 +2,6 @@
 
 const { formatFilenameDate } = require('../_filenameDate.js')
 const { getOldEntries } = require('../_getOldEntries.js')
-const { getVmBackupDir } = require('../_getVmBackupDir.js')
 const { Task } = require('../Task.js')
 
 const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
@@ -34,7 +33,6 @@ exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(Abst
     const { job, scheduleId, vm } = backup
 
     const adapter = this._adapter
-    const backupDir = getVmBackupDir(vm.uuid)
 
     // TODO: clean VM backup directory
 
@@ -47,7 +45,7 @@ exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(Abst
     const basename = formatFilenameDate(timestamp)
 
     const dataBasename = basename + '.xva'
-    const dataFilename = backupDir + '/' + dataBasename
+    const dataFilename = this._vmBackupDir + '/' + dataBasename
 
     const metadata = {
       jobId: job.id,

package/writers/_MixinBackupWriter.js

@@ -16,7 +16,6 @@ const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
 exports.MixinBackupWriter = (BaseClass = Object) =>
   class MixinBackupWriter extends BaseClass {
     #lock
-    #vmBackupDir
 
     constructor({ remoteId, ...rest }) {
       super(rest)
@@ -24,13 +23,13 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
       this._adapter = rest.backup.remoteAdapters[remoteId]
       this._remoteId = remoteId
 
-      this.#vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
+      this._vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
     }
 
     async _cleanVm(options) {
       try {
         return await Task.run({ name: 'clean-vm' }, () => {
-          return this._adapter.cleanVm(this.#vmBackupDir, {
+          return this._adapter.cleanVm(this._vmBackupDir, {
             ...options,
             fixMetadata: true,
             logInfo: info,
@@ -50,7 +49,7 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
 
     async beforeBackup() {
       const { handler } = this._adapter
-      const vmBackupDir = this.#vmBackupDir
+      const vmBackupDir = this._vmBackupDir
       await handler.mktree(vmBackupDir)
       this.#lock = await handler.lock(vmBackupDir)
     }