@xen-orchestra/backups 0.40.0 → 0.42.0

package/RestoreMetadataBackup.mjs

@@ -21,7 +21,12 @@ export class RestoreMetadataBackup {
       })
     } else {
       const metadata = JSON.parse(await handler.readFile(join(backupId, 'metadata.json')))
-      return String(await handler.readFile(resolve(backupId, metadata.data ?? 'data.json')))
+      const dataFileName = resolve(backupId, metadata.data ?? 'data.json')
+      const data = await handler.readFile(dataFileName)
+
+      // if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it
+      const isJson = dataFileName.endsWith('.json')
+      return isJson ? data.toString() : { encoding: 'base64', data: data.toString('base64') }
     }
   }
 }

package/_incrementalVm.mjs

@@ -41,6 +41,7 @@ export async function exportIncrementalVm(
     fullVdisRequired = new Set(),
 
     disableBaseTags = false,
+    preferNbd,
   } = {}
 ) {
   // refs of VM's VDIs → base's VDIs.
@@ -88,6 +89,7 @@ export async function exportIncrementalVm(
       baseRef: baseVdi?.$ref,
       cancelToken,
       format: 'vhd',
+      preferNbd,
     })
   })
 

package/_runners/_XoMetadataBackup.mjs

@@ -22,7 +22,13 @@ export class XoMetadataBackup {
     const dir = `${scheduleDir}/${formatFilenameDate(timestamp)}`
 
     const data = job.xoMetadata
-    const dataBaseName = './data.json'
+    let dataBaseName = './data'
+
+    // JSON data is sent as plain string, binary data is sent as an object with `data` and `encoding properties
+    const isJson = typeof data === 'string'
+    if (isJson) {
+      dataBaseName += '.json'
+    }
 
     const metadata = JSON.stringify(
       {
@@ -54,7 +60,7 @@ export class XoMetadataBackup {
           async () => {
             const handler = adapter.handler
             const dirMode = this._config.dirMode
-            await handler.outputFile(dataFileName, data, { dirMode })
+            await handler.outputFile(dataFileName, isJson ? data : Buffer.from(data.data, data.encoding), { dirMode })
             await handler.outputFile(metaDataFileName, metadata, {
               dirMode,
             })

package/_runners/_vmRunners/IncrementalXapi.mjs

@@ -41,6 +41,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
 
     const deltaExport = await exportIncrementalVm(exportedVm, baseVm, {
       fullVdisRequired,
+      preferNbd: this._settings.preferNbd,
     })
     // since NBD is network based, if one disk use nbd , all the disk use them
     // except the suspended VDI

package/_runners/_writers/_MixinXapiWriter.mjs

@@ -18,7 +18,7 @@ export const MixinXapiWriter = (BaseClass = Object) =>
       const vdiRefs = await xapi.VM_getDisks(baseVm.$ref)
       for (const vdiRef of vdiRefs) {
         const vdi = xapi.getObject(vdiRef)
-        if (vdi.$SR.uuid !== this._heathCheckSr.uuid) {
+        if (vdi.$SR.uuid !== this._healthCheckSr.uuid) {
           return false
         }
       }

package/merge-worker/cli.mjs

@@ -2,18 +2,21 @@
 // eslint-disable-next-line eslint-comments/disable-enable-pair
 /* eslint-disable n/shebang */
 
+import { asyncEach } from '@vates/async-each'
 import { catchGlobalErrors } from '@xen-orchestra/log/configure'
 import { createLogger } from '@xen-orchestra/log'
 import { getSyncedHandler } from '@xen-orchestra/fs'
 import { join } from 'node:path'
+import { load as loadConfig } from 'app-conf'
 import Disposable from 'promise-toolbox/Disposable'
-import min from 'lodash/min.js'
 
 import { getVmBackupDir } from '../_getVmBackupDir.mjs'
 import { RemoteAdapter } from '../RemoteAdapter.mjs'
 
 import { CLEAN_VM_QUEUE } from './index.mjs'
 
+const APP_NAME = 'xo-merge-worker'
+const APP_DIR = new URL('.', import.meta.url).pathname
 // -------------------------------------------------------------------
 
 catchGlobalErrors(createLogger('xo:backups:mergeWorker'))
@@ -34,6 +37,7 @@ const main = Disposable.wrap(async function* main(args) {
     for (let i = 0; i < 10; ++i) {
       const entries = await handler.list(CLEAN_VM_QUEUE)
       if (entries.length !== 0) {
+        entries.sort()
         return entries
       }
       await new Promise(timeoutResolver)
@@ -42,38 +46,47 @@ const main = Disposable.wrap(async function* main(args) {
 
   let taskFiles
   while ((taskFiles = await listRetry()) !== undefined) {
-    const taskFileBasename = min(taskFiles)
-    const previousTaskFile = join(CLEAN_VM_QUEUE, taskFileBasename)
-    const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
-
-    // move this task to the end
-    try {
-      await handler.rename(previousTaskFile, taskFile)
-    } catch (error) {
-      // this error occurs if the task failed too many times (i.e. too many `_` prefixes)
-      // there is nothing more that can be done
-      if (error.code === 'ENAMETOOLONG') {
-        await handler.unlink(previousTaskFile)
-      }
-
-      throw error
-    }
+    const { concurrency } = await loadConfig(APP_NAME, {
+      appDir: APP_DIR,
+      ignoreUnknownFormats: true,
+    })
+    await asyncEach(
+      taskFiles,
+      async taskFileBasename => {
+        const previousTaskFile = join(CLEAN_VM_QUEUE, taskFileBasename)
+        const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
+
+        // move this task to the end
+        try {
+          await handler.rename(previousTaskFile, taskFile)
+        } catch (error) {
+          // this error occurs if the task failed too many times (i.e. too many `_` prefixes)
+          // there is nothing more that can be done
+          if (error.code === 'ENAMETOOLONG') {
+            await handler.unlink(previousTaskFile)
+          }
 
-    try {
-      const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
-      try {
-        await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
-      } catch (error) {
-        // consider the clean successful if the VM dir is missing
-        if (error.code !== 'ENOENT') {
           throw error
         }
-      }
 
-      handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
-    } catch (error) {
-      warn('failure handling task', { error })
-    }
+        try {
+          const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
+          try {
+            await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
+          } catch (error) {
+            // consider the clean successful if the VM dir is missing
+            if (error.code !== 'ENOENT') {
+              throw error
+            }
+          }
+
+          handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
+        } catch (error) {
+          warn('failure handling task', { error })
+        }
+      },
+      { concurrency }
+    )
   }
 })
 

package/merge-worker/config.toml

@@ -0,0 +1 @@
+concurrency = 1

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.40.0",
+  "version": "0.42.0",
   "engines": {
     "node": ">=14.18"
   },
@@ -17,6 +17,7 @@
     "test-integration": "node--test *.integ.mjs"
   },
   "dependencies": {
+    "@iarna/toml": "^2.2.5",
     "@kldzj/stream-throttle": "^1.1.1",
     "@vates/async-each": "^1.0.0",
     "@vates/cached-dns.lookup": "^1.0.0",
@@ -27,11 +28,12 @@
     "@vates/nbd-client": "^2.0.0",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^4.0.1",
+    "@xen-orchestra/fs": "^4.1.0",
     "@xen-orchestra/log": "^0.6.0",
     "@xen-orchestra/template": "^0.1.0",
-    "compare-versions": "^5.0.1",
-    "d3-time-format": "^3.0.0",
+    "app-conf": "^2.3.0",
+    "compare-versions": "^6.0.0",
+    "d3-time-format": "^4.1.0",
     "decorator-synchronized": "^0.6.0",
     "golike-defer": "^0.5.1",
     "limit-concurrency-decorator": "^0.5.0",
@@ -43,7 +45,7 @@
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
     "vhd-lib": "^4.5.0",
-    "xen-api": "^1.3.4",
+    "xen-api": "^1.3.5",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -54,7 +56,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^3.0.0"
+    "@xen-orchestra/xapi": "^3.1.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {