@xen-orchestra/backups 0.29.6 → 0.31.0

package/Backup.js

@@ -3,6 +3,7 @@
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const Disposable = require('promise-toolbox/Disposable')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
+const pTimeout = require('promise-toolbox/timeout')
 const { compileTemplate } = require('@xen-orchestra/template')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
@@ -25,6 +26,7 @@ const getAdaptersByRemote = adapters => {
 const runTask = (...args) => Task.run(...args).catch(noop) // errors are handled by logs
 
 const DEFAULT_SETTINGS = {
+  getRemoteTimeout: 300e3,
   reportWhen: 'failure',
 }
 
@@ -53,6 +55,13 @@ const DEFAULT_METADATA_SETTINGS = {
   retentionXoMetadata: 0,
 }
 
+class RemoteTimeoutError extends Error {
+  constructor(remoteId) {
+    super('timeout while getting the remote ' + remoteId)
+    this.remoteId = remoteId
+  }
+}
+
 exports.Backup = class Backup {
   constructor({ config, getAdapter, getConnectedRecord, job, schedule }) {
     this._config = config
@@ -60,13 +69,6 @@ exports.Backup = class Backup {
     this._job = job
     this._schedule = schedule
 
-    this._getAdapter = Disposable.factory(function* (remoteId) {
-      return {
-        adapter: yield getAdapter(remoteId),
-        remoteId,
-      }
-    })
-
     this._getSnapshotNameLabel = compileTemplate(config.snapshotNameLabelTpl, {
       '{job.name}': job.name,
       '{vm.name_label}': vm => vm.name_label,
@@ -87,6 +89,27 @@ exports.Backup = class Backup {
 
     this._baseSettings = baseSettings
     this._settings = { ...baseSettings, ...job.settings[schedule.id] }
+
+    const { getRemoteTimeout } = this._settings
+    this._getAdapter = async function (remoteId) {
+      try {
+        const disposable = await pTimeout.call(getAdapter(remoteId), getRemoteTimeout, new RemoteTimeoutError(remoteId))
+
+        return new Disposable(() => disposable.dispose(), {
+          adapter: disposable.value,
+          remoteId,
+        })
+      } catch (error) {
+        // See https://github.com/vatesfr/xen-orchestra/commit/6aa6cfba8ec939c0288f0fa740f6dfad98c43cbb
+        runTask(
+          {
+            name: 'get remote adapter',
+            data: { type: 'remote', id: remoteId },
+          },
+          () => Promise.reject(error)
+        )
+      }
+    }
   }
 
   async _runMetadataBackup() {
@@ -132,20 +155,7 @@ exports.Backup = class Backup {
           })
         )
       ),
-      Disposable.all(
-        remoteIds.map(id =>
-          this._getAdapter(id).catch(error => {
-            // See https://github.com/vatesfr/xen-orchestra/commit/6aa6cfba8ec939c0288f0fa740f6dfad98c43cbb
-            runTask(
-              {
-                name: 'get remote adapter',
-                data: { type: 'remote', id },
-              },
-              () => Promise.reject(error)
-            )
-          })
-        )
-      ),
+      Disposable.all(remoteIds.map(id => this._getAdapter(id))),
       async (pools, remoteAdapters) => {
         // remove adapters that failed (already handled)
         remoteAdapters = remoteAdapters.filter(_ => _ !== undefined)
@@ -233,19 +243,7 @@ exports.Backup = class Backup {
           })
         )
       ),
-      Disposable.all(
-        extractIdsFromSimplePattern(job.remotes).map(id =>
-          this._getAdapter(id).catch(error => {
-            runTask(
-              {
-                name: 'get remote adapter',
-                data: { type: 'remote', id },
-              },
-              () => Promise.reject(error)
-            )
-          })
-        )
-      ),
+      Disposable.all(extractIdsFromSimplePattern(job.remotes).map(id => this._getAdapter(id))),
       () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
       async (srs, remoteAdapters, healthCheckSr) => {
         // remove adapters that failed (already handled)

package/_backupWorker.js

@@ -1,8 +1,8 @@
 'use strict'
 
-require('@xen-orchestra/log/configure').catchGlobalErrors(
-  require('@xen-orchestra/log').createLogger('xo:backups:worker')
-)
+const logger = require('@xen-orchestra/log').createLogger('xo:backups:worker')
+
+require('@xen-orchestra/log/configure').catchGlobalErrors(logger)
 
 require('@vates/cached-dns.lookup').createCachedLookup().patchGlobal()
 
@@ -20,6 +20,8 @@ const { Backup } = require('./Backup.js')
 const { RemoteAdapter } = require('./RemoteAdapter.js')
 const { Task } = require('./Task.js')
 
+const { debug } = logger
+
 class BackupWorker {
   #config
   #job
@@ -122,6 +124,11 @@ decorateMethodsWith(BackupWorker, {
   ]),
 })
 
+const emitMessage = message => {
+  debug('message emitted', { message })
+  process.send(message)
+}
+
 // Received message:
 //
 // Message {
@@ -139,6 +146,8 @@ decorateMethodsWith(BackupWorker, {
 //   result?: any
 // }
 process.on('message', async message => {
+  debug('message received', { message })
+
   if (message.action === 'run') {
     const backupWorker = new BackupWorker(message.data)
     try {
@@ -147,7 +156,7 @@ process.on('message', async message => {
             {
               name: 'backup run',
               onLog: data =>
-                process.send({
+                emitMessage({
                   data,
                   type: 'log',
                 }),
@@ -156,13 +165,13 @@ process.on('message', async message => {
           )
         : await backupWorker.run()
 
-      process.send({
+      emitMessage({
         type: 'result',
         result,
         status: 'success',
       })
     } catch (error) {
-      process.send({
+      emitMessage({
         type: 'result',
         result: error,
         status: 'failure',

package/_deltaVm.js

@@ -12,7 +12,7 @@ const { defer } = require('golike-defer')
 
 const { cancelableMap } = require('./_cancelableMap.js')
 const { Task } = require('./Task.js')
-const { pick } = require('lodash')
+const pick = require('lodash/pick.js')
 
 const TAG_BASE_DELTA = 'xo:base_delta'
 exports.TAG_BASE_DELTA = TAG_BASE_DELTA

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.29.6",
+  "version": "0.31.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -26,7 +26,7 @@
     "@vates/nbd-client": "^1.0.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^3.3.1",
+    "@xen-orchestra/fs": "^3.3.2",
     "@xen-orchestra/log": "^0.6.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^5.0.1",

package/runBackupWorker.js

@@ -12,7 +12,7 @@ exports.runBackupWorker = function runBackupWorker(params, onLog) {
   return new Promise((resolve, reject) => {
     const worker = fork(PATH)
 
-    worker.on('exit', code => reject(new Error(`worker exited with code ${code}`)))
+    worker.on('exit', (code, signal) => reject(new Error(`worker exited with code ${code} and signal ${signal}`)))
     worker.on('error', reject)
 
     worker.on('message', message => {

package/writers/DeltaBackupWriter.js

@@ -218,7 +218,9 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
               $defer(() => nbdClient.disconnect())
 
               info('NBD client ready', { vdi: id, path })
+              Task.info('NBD used')
             } catch (error) {
+              Task.warning('NBD configured but unusable', { error })
               nbdClient = undefined
               warn('error connecting to NBD server', { error, vdi: id, path })
             }

package/writers/DeltaReplicationWriter.js

@@ -80,6 +80,7 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
   }
 
   async _transfer({ timestamp, deltaExport, sizeContainers }) {
+    const { _warmMigration } = this._settings
     const sr = this._sr
     const { job, scheduleId, vm } = this._backup
 
@@ -92,7 +93,7 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
           __proto__: deltaExport,
           vm: {
             ...deltaExport.vm,
-            tags: [...deltaExport.vm.tags, 'Continuous Replication'],
+            tags: _warmMigration ? deltaExport.vm.tags : [...deltaExport.vm.tags, 'Continuous Replication'],
           },
         },
         sr
@@ -101,11 +102,13 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
         size: Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0),
       }
     })
-
+    this._targetVmRef = targetVmRef
     const targetVm = await xapi.getRecord('VM', targetVmRef)
 
     await Promise.all([
-      targetVm.ha_restart_priority !== '' &&
+      // warm migration does not disable HA , since the goal is to start the new VM in production
+      !_warmMigration &&
+        targetVm.ha_restart_priority !== '' &&
         Promise.all([targetVm.set_ha_restart_priority(''), targetVm.add_tags('HA disabled')]),
       targetVm.set_name_label(`${vm.name_label} - ${job.name} - (${formatFilenameDate(timestamp)})`),
       asyncMap(['start', 'start_on'], op =>

package/writers/FullReplicationWriter.js

@@ -46,7 +46,7 @@ exports.FullReplicationWriter = class FullReplicationWriter extends MixinReplica
     const oldVms = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vm.uuid))
 
     const deleteOldBackups = () => asyncMapSettled(oldVms, vm => xapi.VM_destroy(vm.$ref))
-    const { deleteFirst } = settings
+    const { deleteFirst, _warmMigration } = settings
     if (deleteFirst) {
       await deleteOldBackups()
     }
@@ -55,14 +55,18 @@ exports.FullReplicationWriter = class FullReplicationWriter extends MixinReplica
     await Task.run({ name: 'transfer' }, async () => {
       targetVmRef = await xapi.VM_import(stream, sr.$ref, vm =>
         Promise.all([
-          vm.add_tags('Disaster Recovery'),
-          vm.ha_restart_priority !== '' && Promise.all([vm.set_ha_restart_priority(''), vm.add_tags('HA disabled')]),
+          !_warmMigration && vm.add_tags('Disaster Recovery'),
+          // warm migration does not disable HA , since the goal is to start the new VM in production
+          !_warmMigration &&
+            vm.ha_restart_priority !== '' &&
+            Promise.all([vm.set_ha_restart_priority(''), vm.add_tags('HA disabled')]),
           vm.set_name_label(`${vm.name_label} - ${job.name} - (${formatFilenameDate(timestamp)})`),
         ])
       )
       return { size: sizeContainer.size }
     })
 
+    this._targetVmRef = targetVmRef
     const targetVm = await xapi.getRecord('VM', targetVmRef)
 
     await Promise.all([

package/writers/_MixinReplicationWriter.js

@@ -1,5 +1,17 @@
 'use strict'
 
+const { Task } = require('../Task')
+const assert = require('node:assert/strict')
+const { HealthCheckVmBackup } = require('../HealthCheckVmBackup')
+
+function extractOpaqueRef(str) {
+  const OPAQUE_REF_RE = /OpaqueRef:[0-9a-z-]+/
+  const matches = OPAQUE_REF_RE.exec(str)
+  if (!matches) {
+    throw new Error('no opaque ref found')
+  }
+  return matches[0]
+}
 exports.MixinReplicationWriter = (BaseClass = Object) =>
   class MixinReplicationWriter extends BaseClass {
     constructor({ sr, ...rest }) {
@@ -7,4 +19,32 @@ exports.MixinReplicationWriter = (BaseClass = Object) =>
 
       this._sr = sr
     }
+
+    healthCheck(sr) {
+      assert.notEqual(this._targetVmRef, undefined, 'A vm should have been transfered to be health checked')
+      // copy VM
+      return Task.run(
+        {
+          name: 'health check',
+        },
+        async () => {
+          const { $xapi: xapi } = sr
+          let clonedVm
+          try {
+            const baseVm = xapi.getObject(this._targetVmRef) ?? (await xapi.waitObject(this._targetVmRef))
+            const clonedRef = await xapi
+              .callAsync('VM.clone', this._targetVmRef, `Health Check - ${baseVm.name_label}`)
+              .then(extractOpaqueRef)
+            clonedVm = xapi.getObject(clonedRef) ?? (await xapi.waitObject(clonedRef))
+
+            await new HealthCheckVmBackup({
+              restoredVm: clonedVm,
+              xapi,
+            }).run()
+          } finally {
+            clonedVm && (await xapi.VM_destroy(clonedVm.$ref))
+          }
+        }
+      )
+    }
   }