@xen-orchestra/backups 0.36.1 → 0.38.0

package/{writers/FullReplicationWriter.js → _runners/_writers/FullXapiWriter.js}

@@ -4,15 +4,15 @@ const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
-const { formatFilenameDate } = require('../_filenameDate.js')
-const { getOldEntries } = require('../_getOldEntries.js')
-const { Task } = require('../Task.js')
+const { formatFilenameDate } = require('../../_filenameDate.js')
+const { getOldEntries } = require('../../_getOldEntries.js')
+const { Task } = require('../../Task.js')
 
 const { AbstractFullWriter } = require('./_AbstractFullWriter.js')
-const { MixinReplicationWriter } = require('./_MixinReplicationWriter.js')
+const { MixinXapiWriter } = require('./_MixinXapiWriter.js')
 const { listReplicatedVms } = require('./_listReplicatedVms.js')
 
-exports.FullReplicationWriter = class FullReplicationWriter extends MixinReplicationWriter(AbstractFullWriter) {
+exports.FullXapiWriter = class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
   constructor(props) {
     super(props)
 
@@ -32,10 +32,11 @@ exports.FullReplicationWriter = class FullReplicationWriter extends MixinReplica
     )
   }
 
-  async _run({ timestamp, sizeContainer, stream }) {
+  async _run({ timestamp, sizeContainer, stream, vm }) {
     const sr = this._sr
     const settings = this._settings
-    const { job, scheduleId, vm } = this._backup
+    const job = this._job
+    const scheduleId = this.scheduleId
 
     const { uuid: srUuid, $xapi: xapi } = sr
 

package/{writers/DeltaBackupWriter.js → _runners/_writers/IncrementalRemoteWriter.js}

@@ -11,25 +11,24 @@ const { decorateClass } = require('@vates/decorate-with')
 const { defer } = require('golike-defer')
 const { dirname } = require('path')
 
-const { formatFilenameDate } = require('../_filenameDate.js')
-const { getOldEntries } = require('../_getOldEntries.js')
-const { Task } = require('../Task.js')
+const { formatFilenameDate } = require('../../_filenameDate.js')
+const { getOldEntries } = require('../../_getOldEntries.js')
+const { Task } = require('../../Task.js')
 
-const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
-const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
+const { MixinRemoteWriter } = require('./_MixinRemoteWriter.js')
+const { AbstractIncrementalWriter } = require('./_AbstractIncrementalWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
 const { Disposable } = require('promise-toolbox')
 
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
-class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
+class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWriter) {
   async checkBaseVdis(baseUuidToSrcVdi) {
     const { handler } = this._adapter
-    const backup = this._backup
     const adapter = this._adapter
 
-    const vdisDir = `${this._vmBackupDir}/vdis/${backup.job.id}`
+    const vdisDir = `${this._vmBackupDir}/vdis/${this._job.id}`
 
     await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdi]) => {
       let found = false
@@ -91,11 +90,12 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
   async _prepare() {
     const adapter = this._adapter
     const settings = this._settings
-    const { scheduleId, vm } = this._backup
+    const scheduleId = this._scheduleId
+    const vmUuid = this._vmUuid
 
     const oldEntries = getOldEntries(
       settings.exportRetention - 1,
-      await adapter.listVmBackups(vm.uuid, _ => _.mode === 'delta' && _.scheduleId === scheduleId)
+      await adapter.listVmBackups(vmUuid, _ => _.mode === 'delta' && _.scheduleId === scheduleId)
     )
     this._oldEntries = oldEntries
 
@@ -134,16 +134,19 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
     }
   }
 
-  async _transfer($defer, { timestamp, deltaExport }) {
+  async _transfer($defer, { differentialVhds, timestamp, deltaExport, vm, vmSnapshot }) {
     const adapter = this._adapter
-    const backup = this._backup
-
-    const { job, scheduleId, vm } = backup
+    const job = this._job
+    const scheduleId = this._scheduleId
 
     const jobId = job.id
     const handler = adapter.handler
 
-    // TODO: clean VM backup directory
+    let metadataContent = await this._isAlreadyTransferred(timestamp)
+    if (metadataContent !== undefined) {
+      // @todo : should skip backup while being vigilant to not stuck the forked stream
+      Task.info('This backup has already been transfered')
+    }
 
     const basename = formatFilenameDate(timestamp)
     const vhds = mapValues(
@@ -158,7 +161,7 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
         }/${adapter.getVhdFileName(basename)}`
     )
 
-    const metadataContent = {
+    metadataContent = {
       jobId,
       mode: job.mode,
       scheduleId,
@@ -169,16 +172,15 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
       vifs: deltaExport.vifs,
       vhds,
       vm,
-      vmSnapshot: this._backup.exportedVm,
+      vmSnapshot,
     }
-
     const { size } = await Task.run({ name: 'transfer' }, async () => {
       let transferSize = 0
       await Promise.all(
         map(deltaExport.vdis, async (vdi, id) => {
           const path = `${this._vmBackupDir}/${vhds[id]}`
 
-          const isDelta = vdi.other_config['xo:base_delta'] !== undefined
+          const isDelta = differentialVhds[`${id}.vhd`]
           let parentPath
           if (isDelta) {
             const vdiDir = dirname(path)
@@ -191,7 +193,11 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
               .sort()
               .pop()
 
-            assert.notStrictEqual(parentPath, undefined, `missing parent of ${id}`)
+            assert.notStrictEqual(
+              parentPath,
+              undefined,
+              `missing parent of ${id} in ${dirname(path)}, looking for ${vdi.other_config['xo:base_delta']}`
+            )
 
             parentPath = parentPath.slice(1) // remove leading slash
 
@@ -204,7 +210,7 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
             // merges and chainings
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
-            writeBlockConcurrency: this._backup.config.writeBlockConcurrency,
+            writeBlockConcurrency: this._config.writeBlockConcurrency,
           })
 
           if (isDelta) {
@@ -227,6 +233,6 @@ class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
     // TODO: run cleanup?
   }
 }
-exports.DeltaBackupWriter = decorateClass(DeltaBackupWriter, {
+exports.IncrementalRemoteWriter = decorateClass(IncrementalRemoteWriter, {
   _transfer: defer,
 })

package/{writers/DeltaReplicationWriter.js → _runners/_writers/IncrementalXapiWriter.js}

@@ -4,19 +4,19 @@ const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
-const { formatFilenameDate } = require('../_filenameDate.js')
-const { getOldEntries } = require('../_getOldEntries.js')
-const { importDeltaVm, TAG_COPY_SRC } = require('../_deltaVm.js')
-const { Task } = require('../Task.js')
+const { formatFilenameDate } = require('../../_filenameDate.js')
+const { getOldEntries } = require('../../_getOldEntries.js')
+const { importIncrementalVm, TAG_COPY_SRC } = require('../../_incrementalVm.js')
+const { Task } = require('../../Task.js')
 
-const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
-const { MixinReplicationWriter } = require('./_MixinReplicationWriter.js')
+const { AbstractIncrementalWriter } = require('./_AbstractIncrementalWriter.js')
+const { MixinXapiWriter } = require('./_MixinXapiWriter.js')
 const { listReplicatedVms } = require('./_listReplicatedVms.js')
 
-exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinReplicationWriter(AbstractDeltaWriter) {
+exports.IncrementalXapiWriter = class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
   async checkBaseVdis(baseUuidToSrcVdi, baseVm) {
     const sr = this._sr
-    const replicatedVm = listReplicatedVms(sr.$xapi, this._backup.job.id, sr.uuid, this._backup.vm.uuid).find(
+    const replicatedVm = listReplicatedVms(sr.$xapi, this._job.id, sr.uuid, this._vmUuid).find(
       vm => vm.other_config[TAG_COPY_SRC] === baseVm.uuid
     )
     if (replicatedVm === undefined) {
@@ -49,9 +49,10 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
         type: 'SR',
       },
     })
+    const hasHealthCheckSr = this._healthCheckSr !== undefined
     this.transfer = task.wrapFn(this.transfer)
-    this.cleanup = task.wrapFn(this.cleanup)
-    this.healthCheck = task.wrapFn(this.healthCheck, true)
+    this.cleanup = task.wrapFn(this.cleanup, !hasHealthCheckSr)
+    this.healthCheck = task.wrapFn(this.healthCheck, hasHealthCheckSr)
 
     return task.run(() => this._prepare())
   }
@@ -59,12 +60,13 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
   async _prepare() {
     const settings = this._settings
     const { uuid: srUuid, $xapi: xapi } = this._sr
-    const { scheduleId, vm } = this._backup
+    const vmUuid = this._vmUuid
+    const scheduleId = this._scheduleId
 
     // delete previous interrupted copies
-    ignoreErrors.call(asyncMapSettled(listReplicatedVms(xapi, scheduleId, undefined, vm.uuid), vm => vm.$destroy))
+    ignoreErrors.call(asyncMapSettled(listReplicatedVms(xapi, scheduleId, undefined, vmUuid), vm => vm.$destroy))
 
-    this._oldEntries = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vm.uuid))
+    this._oldEntries = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vmUuid))
 
     if (settings.deleteFirst) {
       await this._deleteOldEntries()
@@ -81,16 +83,17 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
     return asyncMapSettled(this._oldEntries, vm => vm.$destroy())
   }
 
-  async _transfer({ timestamp, deltaExport, sizeContainers }) {
+  async _transfer({ timestamp, deltaExport, sizeContainers, vm }) {
     const { _warmMigration } = this._settings
     const sr = this._sr
-    const { job, scheduleId, vm } = this._backup
+    const job = this._job
+    const scheduleId = this._scheduleId
 
     const { uuid: srUuid, $xapi: xapi } = sr
 
     let targetVmRef
     await Task.run({ name: 'transfer' }, async () => {
-      targetVmRef = await importDeltaVm(
+      targetVmRef = await importIncrementalVm(
         {
           __proto__: deltaExport,
           vm: {

package/{writers → _runners/_writers}/_AbstractFullWriter.js

@@ -3,9 +3,9 @@
 const { AbstractWriter } = require('./_AbstractWriter.js')
 
 exports.AbstractFullWriter = class AbstractFullWriter extends AbstractWriter {
-  async run({ timestamp, sizeContainer, stream }) {
+  async run({ timestamp, sizeContainer, stream, vm, vmSnapshot }) {
     try {
-      return await this._run({ timestamp, sizeContainer, stream })
+      return await this._run({ timestamp, sizeContainer, stream, vm, vmSnapshot })
     } finally {
       // ensure stream is properly closed
       stream.destroy()

package/{writers/_AbstractDeltaWriter.js → _runners/_writers/_AbstractIncrementalWriter.js}

@@ -2,7 +2,7 @@
 
 const { AbstractWriter } = require('./_AbstractWriter.js')
 
-exports.AbstractDeltaWriter = class AbstractDeltaWriter extends AbstractWriter {
+exports.AbstractIncrementalWriter = class AbstractIncrementalWriter extends AbstractWriter {
   checkBaseVdis(baseUuidToSrcVdi, baseVm) {
     throw new Error('Not implemented')
   }
@@ -15,9 +15,9 @@ exports.AbstractDeltaWriter = class AbstractDeltaWriter extends AbstractWriter {
     throw new Error('Not implemented')
   }
 
-  async transfer({ timestamp, deltaExport, sizeContainers }) {
+  async transfer({ deltaExport, ...other }) {
     try {
-      return await this._transfer({ timestamp, deltaExport, sizeContainers })
+      return await this._transfer({ deltaExport, ...other })
     } finally {
       // ensure all streams are properly closed
       for (const stream of Object.values(deltaExport.streams)) {

package/_runners/_writers/_AbstractWriter.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const { formatFilenameDate } = require('../../_filenameDate')
+const { getVmBackupDir } = require('../../_getVmBackupDir')
+
+exports.AbstractWriter = class AbstractWriter {
+  constructor({ config, healthCheckSr, job, vmUuid, scheduleId, settings }) {
+    this._config = config
+    this._healthCheckSr = healthCheckSr
+    this._job = job
+    this._scheduleId = scheduleId
+    this._settings = settings
+    this._vmUuid = vmUuid
+  }
+
+  beforeBackup() {}
+
+  afterBackup() {}
+
+  healthCheck(sr) {}
+
+  _isAlreadyTransferred(timestamp) {
+    const vmUuid = this._vmUuid
+    const adapter = this._adapter
+    const backupDir = getVmBackupDir(vmUuid)
+    try {
+      const actualMetadata = JSON.parse(adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`))
+      return actualMetadata
+    } catch (error) {}
+  }
+}

package/{writers/_MixinBackupWriter.js → _runners/_writers/_MixinRemoteWriter.js}

@@ -4,26 +4,26 @@ const { createLogger } = require('@xen-orchestra/log')
 const { join } = require('path')
 
 const assert = require('assert')
-const { formatFilenameDate } = require('../_filenameDate.js')
-const { getVmBackupDir } = require('../_getVmBackupDir.js')
-const { HealthCheckVmBackup } = require('../HealthCheckVmBackup.js')
-const { ImportVmBackup } = require('../ImportVmBackup.js')
-const { Task } = require('../Task.js')
-const MergeWorker = require('../merge-worker/index.js')
+const { formatFilenameDate } = require('../../_filenameDate.js')
+const { getVmBackupDir } = require('../../_getVmBackupDir.js')
+const { HealthCheckVmBackup } = require('../../HealthCheckVmBackup.js')
+const { ImportVmBackup } = require('../../ImportVmBackup.js')
+const { Task } = require('../../Task.js')
+const MergeWorker = require('../../merge-worker/index.js')
 
 const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
 
-exports.MixinBackupWriter = (BaseClass = Object) =>
-  class MixinBackupWriter extends BaseClass {
+exports.MixinRemoteWriter = (BaseClass = Object) =>
+  class MixinRemoteWriter extends BaseClass {
     #lock
 
-    constructor({ remoteId, ...rest }) {
+    constructor({ remoteId, adapter, ...rest }) {
       super(rest)
 
-      this._adapter = rest.backup.remoteAdapters[remoteId]
+      this._adapter = adapter
       this._remoteId = remoteId
 
-      this._vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
+      this._vmBackupDir = getVmBackupDir(rest.vmUuid)
     }
 
     async _cleanVm(options) {
@@ -38,7 +38,7 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
               Task.warning(message, data)
             },
             lock: false,
-            mergeBlockConcurrency: this._backup.config.mergeBlockConcurrency,
+            mergeBlockConcurrency: this._config.mergeBlockConcurrency,
           })
         })
       } catch (error) {
@@ -55,10 +55,10 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
     }
 
     async afterBackup() {
-      const { disableMergeWorker } = this._backup.config
+      const { disableMergeWorker } = this._config
       // merge worker only compatible with local remotes
       const { handler } = this._adapter
-      const willMergeInWorker = !disableMergeWorker && typeof handler._getRealPath === 'function'
+      const willMergeInWorker = !disableMergeWorker && typeof handler.getRealPath === 'function'
 
       const { merge } = await this._cleanVm({ remove: true, merge: !willMergeInWorker })
       await this.#lock.dispose()
@@ -71,12 +71,14 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
           Math.random().toString(36).slice(2)
 
         await handler.outputFile(taskFile, this._backup.vm.uuid)
-        const remotePath = handler._getRealPath()
+        const remotePath = handler.getRealPath()
         await MergeWorker.run(remotePath)
       }
     }
 
-    healthCheck(sr) {
+    healthCheck() {
+      const sr = this._healthCheckSr
+      assert.notStrictEqual(sr, undefined, 'SR should be defined before making a health check')
       assert.notStrictEqual(
         this._metadataFileName,
         undefined,
@@ -109,4 +111,16 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
         }
       )
     }
+
+    _isAlreadyTransferred(timestamp) {
+      const vmUuid = this._vmUuid
+      const adapter = this._adapter
+      const backupDir = getVmBackupDir(vmUuid)
+      try {
+        const actualMetadata = JSON.parse(
+          adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
+        )
+        return actualMetadata
+      } catch (error) {}
+    }
   }

package/{writers/_MixinReplicationWriter.js → _runners/_writers/_MixinXapiWriter.js}

@@ -1,26 +1,22 @@
 'use strict'
 
-const { Task } = require('../Task')
+const { extractOpaqueRef } = require('@xen-orchestra/xapi')
+
+const { Task } = require('../../Task')
 const assert = require('node:assert/strict')
-const { HealthCheckVmBackup } = require('../HealthCheckVmBackup')
+const { HealthCheckVmBackup } = require('../../HealthCheckVmBackup')
 
-function extractOpaqueRef(str) {
-  const OPAQUE_REF_RE = /OpaqueRef:[0-9a-z-]+/
-  const matches = OPAQUE_REF_RE.exec(str)
-  if (!matches) {
-    throw new Error('no opaque ref found')
-  }
-  return matches[0]
-}
-exports.MixinReplicationWriter = (BaseClass = Object) =>
-  class MixinReplicationWriter extends BaseClass {
+exports.MixinXapiWriter = (BaseClass = Object) =>
+  class MixinXapiWriter extends BaseClass {
     constructor({ sr, ...rest }) {
       super(rest)
 
       this._sr = sr
     }
 
-    healthCheck(sr) {
+    healthCheck() {
+      const sr = this._healthCheckSr
+      assert.notStrictEqual(sr, undefined, 'SR should be defined before making a health check')
       assert.notEqual(this._targetVmRef, undefined, 'A vm should have been transfered to be health checked')
       // copy VM
       return Task.run(

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.36.1",
+  "version": "0.38.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -27,7 +27,7 @@
     "@vates/nbd-client": "^1.2.0",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^3.3.4",
+    "@xen-orchestra/fs": "^4.0.0",
     "@xen-orchestra/log": "^0.6.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^5.0.1",
@@ -42,17 +42,17 @@
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
     "uuid": "^9.0.0",
-    "vhd-lib": "^4.4.0",
+    "vhd-lib": "^4.5.0",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
-    "rimraf": "^4.1.1",
+    "rimraf": "^5.0.1",
     "sinon": "^15.0.1",
     "test": "^3.2.1",
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^2.2.0"
+    "@xen-orchestra/xapi": "^2.2.1"
   },
   "license": "AGPL-3.0-or-later",
   "author": {