@xen-orchestra/backups 0.66.0 → 0.67.1

package/ImportVmBackup.mjs

@@ -29,10 +29,10 @@ export class ImportVmBackup {
     metadata,
     srUuid,
     xapi,
-    settings: { additionnalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
+    settings: { additionalVmTag, newMacAddresses, mapVdisSrs = {}, useDifferentialRestore = false } = {},
   }) {
     this._adapter = adapter
-    this._importIncrementalVmSettings = { additionnalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
+    this._importIncrementalVmSettings = { additionalVmTag, newMacAddresses, mapVdisSrs, useDifferentialRestore }
     this._metadata = metadata
     this._srUuid = srUuid
     this._xapi = xapi
@@ -61,11 +61,11 @@ export class ImportVmBackup {
     const { mapVdisSrs } = this._importIncrementalVmSettings
     const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
     const disks = {}
-    const metdataDir = dirname(metadata._filename)
+    const metadataDir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
 
     for (const [vdiRef, vdi] of Object.entries(vdis)) {
-      const vhdPath = join(metdataDir, vhds[vdiRef])
+      const vhdPath = join(metadataDir, vhds[vdiRef])
 
       let xapiDisk
       try {
@@ -77,7 +77,7 @@ export class ImportVmBackup {
 
       let snapshotCandidate, backupCandidate
       if (xapiDisk !== undefined) {
-        debug('found disks, wlll search its snapshots', { snapshots: xapiDisk.snapshots })
+        debug('found disks, will search its snapshots', { snapshots: xapiDisk.snapshots })
         for (const snapshotRef of xapiDisk.snapshots) {
           const snapshot = await this._xapi.getRecord('VDI', snapshotRef)
 
@@ -96,7 +96,7 @@ export class ImportVmBackup {
           // have a corresponding backup more recent than metadata ?
           const pathToSnapshotData = await this.#getPathOfVdiSnapshot(snapshot.uuid)
           if (pathToSnapshotData === undefined) {
-            debug('no backup linked to this snaphot')
+            debug('no backup linked to this snapshot')
             continue
           }
           if (snapshot.$SR.uuid !== (mapVdisSrs[vdi.$snapshot_of$uuid] ?? this._srUuid)) {
@@ -112,7 +112,7 @@ export class ImportVmBackup {
       }
 
       let disk
-      const backupWithSnapshotPath = join(metdataDir, backupCandidate ?? '')
+      const backupWithSnapshotPath = join(metadataDir, backupCandidate ?? '')
       if (vhdPath === backupWithSnapshotPath) {
         // all the data are already on the host
         debug('direct reuse of a snapshot')
@@ -194,7 +194,7 @@ export class ImportVmBackup {
   }
 
   async #decorateIncrementalVmMetadata() {
-    const { additionnalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
+    const { additionalVmTag, mapVdisSrs, useDifferentialRestore } = this._importIncrementalVmSettings
 
     const ignoredVdis = new Set(
       Object.entries(mapVdisSrs)
@@ -211,8 +211,8 @@ export class ImportVmBackup {
 
     const cache = new Map()
     const mapVdisSrRefs = {}
-    if (additionnalVmTag !== undefined) {
-      backup.vm.tags.push(additionnalVmTag)
+    if (additionalVmTag !== undefined) {
+      backup.vm.tags.push(additionalVmTag)
     }
     for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
       mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')

package/RemoteAdapter.mjs

@@ -27,7 +27,7 @@ import { formatFilenameDate } from './_filenameDate.mjs'
 import { getTmpDir } from './_getTmpDir.mjs'
 import { isMetadataFile } from './_backupType.mjs'
 import { isValidXva } from './_isValidXva.mjs'
-import { listPartitions, LVM_PARTITION_TYPE } from './_listPartitions.mjs'
+import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from './_listPartitions.mjs'
 import { lvs, pvs } from './_lvm.mjs'
 import { watchStreamSize } from './_watchStreamSize.mjs'
 
@@ -40,7 +40,7 @@ export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 
 export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 
-const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
+const IMMUTABILITY_METADATA_FILENAME = '/immutability.json'
 
 const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
 
@@ -312,8 +312,8 @@ export class RemoteAdapter {
   }
 
   async deleteVmBackups(files) {
-    const metadatas = await asyncMap(files, file => this.readVmBackupMetadata(file))
-    const { delta, full, ...others } = groupBy(metadatas, 'mode')
+    const metadata = await asyncMap(files, file => this.readVmBackupMetadata(file))
+    const { delta, full, ...others } = groupBy(metadata, 'mode')
 
     const unsupportedModes = Object.keys(others)
     if (unsupportedModes.length !== 0) {
@@ -498,7 +498,7 @@ export class RemoteAdapter {
 
       const results = []
       await asyncMapSettled(partitions, partition =>
-        partition.type === LVM_PARTITION_TYPE
+        partition.type === LVM_PARTITION_TYPE_MBR || partition.type === LVM_PARTITION_TYPE_GPT
           ? this._listLvmLogicalVolumes(devicePath, partition, results)
           : results.push(partition)
       )
@@ -575,7 +575,7 @@ export class RemoteAdapter {
     }
   }
 
-  async #getCachabledDataListVmBackups(dir) {
+  async #getCacheableDataListVmBackups(dir) {
     debug('generating cache', { path: dir })
 
     const handler = this._handler
@@ -622,7 +622,7 @@ export class RemoteAdapter {
     }
 
     // nothing cached, or cache unreadable => regenerate it
-    const backups = await this.#getCachabledDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
+    const backups = await this.#getCacheableDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
     if (backups === undefined) {
       return
     }
@@ -709,7 +709,7 @@ export class RemoteAdapter {
       })
     } else {
       const stream = await toVhdStream(disk)
-      await this.outputStream(path, stream, { validator })
+      await this.outputStream(path, stream, { validator, checksum: false })
       await validator(path)
     }
   }
@@ -779,7 +779,7 @@ export class RemoteAdapter {
     // if the remote is immutable, check if this metadata is also immutable
     try {
       // this file is not encrypted
-      await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
+      await this._handler._readFile(IMMUTABILITY_METADATA_FILENAME)
       remoteIsImmutable = true
     } catch (error) {
       if (error.code !== 'ENOENT') {

package/_backupType.mjs

@@ -1,4 +1,5 @@
 export const isMetadataFile = filename => filename.endsWith('.json')
 export const isVhdFile = filename => filename.endsWith('.vhd')
+export const isVhdSumFile = filename => filename.endsWith('.vhd.checksum')
 export const isXvaFile = filename => filename.endsWith('.xva')
 export const isXvaSumFile = filename => filename.endsWith('.xva.checksum')

package/_cleanVm.mjs

@@ -4,7 +4,7 @@ import { asyncMap } from '@xen-orchestra/async-map'
 import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
 import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
 import { basename, dirname, resolve } from 'node:path'
-import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
+import { isMetadataFile, isVhdFile, isVhdSumFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { mergeVhdChain } from 'vhd-lib/merge.js'
 
@@ -68,10 +68,12 @@ async function _mergeVhdChain(handler, chain, { logInfo, remove, mergeBlockConcu
 const noop = Function.prototype
 
 const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
+
 const listVhds = async (handler, vmDir, logWarn) => {
   const vhds = new Set()
   const aliases = {}
   const interruptedVhds = new Map()
+  const checksums = new Set()
 
   await asyncMap(
     await handler.list(`${vmDir}/vdis`, {
@@ -85,11 +87,15 @@ const listVhds = async (handler, vmDir, logWarn) => {
         }),
         async vdiDir => {
           const list = await handler.list(vdiDir, {
-            filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
+            filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file) || isVhdSumFile(file),
           })
           aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
 
           await asyncMap(list, async file => {
+            if (isVhdSumFile(file)) {
+              checksums.add(`${vdiDir}/${file}`)
+              return
+            }
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
               vhds.add(`${vdiDir}/${file}`)
@@ -111,7 +117,7 @@ const listVhds = async (handler, vmDir, logWarn) => {
       )
   )
 
-  return { vhds, interruptedVhds, aliases }
+  return { vhds, interruptedVhds, aliases, checksums }
 }
 
 export async function checkAliases(
@@ -228,7 +234,16 @@ export async function cleanVm(
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
-  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir, logWarn)
+  const { vhds, interruptedVhds, aliases, checksums } = await listVhds(handler, vmDir, logWarn)
+
+  // from 5.110 to 5.113 we computed checksum for vhd file
+  // but never used nor removed them
+  await asyncMap(checksums, async path => {
+    if (remove) {
+      logInfo('deleting checksum file ', { path })
+      return handler.unlink(path)
+    }
+  })
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {

package/_getOldEntries.mjs

@@ -97,40 +97,59 @@ export function getOldEntries(minRetentionCount, entries, { longTermRetention =
       remaining: retention,
       lastMatchingBucket: null,
       formatter: LTR_DEFINITIONS[duration].makeDateFormatter({ ...settings, dateCreator }),
+      entries: {}, // bucket => entry id
     }
   }
   const nb = entries.length
-  const oldEntries = []
-
+  const minDurationEntries = []
+  let previousTimestamp = -1
   for (let i = nb - 1; i >= 0; i--) {
     const entry = entries[i]
-    const entryDate = dateCreator(entry.timestamp)
-    let shouldBeKept = false
-    for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
-      if (remaining === 0) {
-        continue
-      }
-      const bucket = formatter(entryDate)
-      if (lastMatchingBucket !== bucket) {
-        if (lastMatchingBucket !== null) {
-          assert.strictEqual(
-            lastMatchingBucket > bucket,
-            true,
-            `entries must be sorted in asc order ${lastMatchingBucket} ${bucket}`
-          )
+    if (entry.timestamp !== undefined) {
+      // we go through the entries from the last (most recent) to the first (oldest)
+      assert.ok(
+        previousTimestamp === -1 || entry.timestamp < previousTimestamp,
+        `entries must be sorted in desc order ${new Date(entry.timestamp)} , previous :  > ${new Date(previousTimestamp)} `
+      )
+      previousTimestamp = entry.timestamp
+      const entryDate = dateCreator(entry.timestamp)
+      for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
+        const bucket = formatter(entryDate)
+        if (lastMatchingBucket !== bucket) {
+          if (remaining === 0) {
+            continue
+          }
+          dateBuckets[duration].lastMatchingBucket = bucket
+          dateBuckets[duration].remaining -= 1
         }
-        shouldBeKept = true
-        dateBuckets[duration].remaining -= 1
-        dateBuckets[duration].lastMatchingBucket = bucket
+        dateBuckets[duration].entries[bucket] = entry
       }
+    } else {
+      // replicated VM entries or snapshot retention don't have a timestamp
+      // but also can't have LTR
+      assert.deepStrictEqual(
+        longTermRetention,
+        {},
+        "Can't compute long term retention if entries don't have a timestamp"
+      )
     }
+
+    // also keep it on retention
     if (i >= nb - minRetentionCount) {
-      shouldBeKept = true
+      minDurationEntries.push(entry)
     }
-    if (!shouldBeKept) {
-      oldEntries.push(entry)
+  }
+  const kept = new Set(minDurationEntries)
+  for (const { entries } of Object.values(dateBuckets)) {
+    for (const entry of Object.values(entries)) {
+      kept.add(entry)
     }
   }
-  // we expect the entries to be in the right order
-  return oldEntries.reverse()
+
+  // ensure order is the same as the source
+  const oldEntries = entries.filter(entry => {
+    return !kept.has(entry)
+  })
+
+  return oldEntries
 }

package/_listPartitions.mjs

@@ -20,13 +20,31 @@ const IGNORED_PARTITION_TYPES = {
   0xd5: true,
 
   0x82: true, // swap
+  // GPT Linux swap GUID
+  '0657fd6d-a4ab-43c4-84e5-0933c84b4f4f': true,
 }
 
-export const LVM_PARTITION_TYPE = 0x8e
+// MBR LVM type
+export const LVM_PARTITION_TYPE_MBR = 0x8e
+// GPT LVM type
+export const LVM_PARTITION_TYPE_GPT = 'e6d6d379-f507-44c2-a23c-238f2a3df928'
 
 const parsePartxLine = createParser({
   keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
-  valueTransform: (value, key) => (key === 'start' || key === 'size' || key === 'type' ? +value : value),
+  valueTransform: (value, key) => {
+    if (key === 'start' || key === 'size') {
+      return +value
+    }
+    // For GPT partitions, type is a UUID string
+    if (key === 'type' && !value.startsWith('0x')) {
+      return value.toLowerCase()
+    }
+    // For MBR partitions, type is a hex number as string (e.g., "0x8e")
+    if (key === 'type' && value.startsWith('0x')) {
+      return parseInt(value, 16)
+    }
+    return value
+  },
 })
 
 // returns an empty array in case of a non-partitioned disk

package/_runners/_vmRunners/_AbstractRemote.mjs

@@ -74,28 +74,28 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
   }
 
   async #computeTransferListPerJob(sourceBackups, remotesBackups) {
-    const localMetada = new Map()
+    const localMetadata = new Map()
     sourceBackups.forEach(metadata => {
       const timestamp = metadata.timestamp
-      localMetada.set(timestamp, metadata)
+      localMetadata.set(timestamp, metadata)
     })
     const nbRemotes = remotesBackups.length
-    const remoteMetadatas = {}
+    const remoteMetadata = {}
     remotesBackups.forEach(async remoteBackups => {
       remoteBackups.forEach(metadata => {
         const timestamp = metadata.timestamp
-        remoteMetadatas[timestamp] = (remoteMetadatas[timestamp] ?? 0) + 1
+        remoteMetadata[timestamp] = (remoteMetadata[timestamp] ?? 0) + 1
       })
     })
 
     let transferList = []
-    const timestamps = [...localMetada.keys()]
+    const timestamps = [...localMetadata.keys()]
     timestamps.sort()
     for (const timestamp of timestamps) {
-      if (remoteMetadatas[timestamp] !== nbRemotes) {
+      if (remoteMetadata[timestamp] !== nbRemotes) {
         // this backup is not present in all the remote
         // should be retransferred if not found later
-        transferList.push(localMetada.get(timestamp))
+        transferList.push(localMetadata.get(timestamp))
       } else {
         // backup is present in local and remote : the chain has already been transferred
         transferList = []
@@ -103,17 +103,10 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
     }
     if (transferList.length > 0) {
       const filteredTransferList = this._filterTransferList(transferList)
-      if (filteredTransferList.length > 0) {
-        return filteredTransferList
-      } else {
-        Task.info('This VM is excluded by the job filter')
-        return []
-      }
+      return { transferList: filteredTransferList, filtered: true }
     } else {
-      Task.info('No new data to upload for this VM')
+      return { transferList, filtered: false }
     }
-
-    return []
   }
 
   /**
@@ -127,12 +120,21 @@ export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstrac
       Object.values(this.remoteAdapters).map(remoteAdapter => remoteAdapter.listVmBackups(this._vmUuid, vmPredicate))
     )
     const sourceBackupByJobId = groupBy(sourceBackups, 'jobId')
-    const transferByJobs = await Promise.all(
+    const transferObjectsList = await Promise.all(
       Object.values(sourceBackupByJobId).map(vmBackupsByJob =>
         this.#computeTransferListPerJob(vmBackupsByJob, remotesBackups)
       )
     )
-    return transferByJobs.flat(1)
+    const transferList = transferObjectsList.map(transferObject => transferObject.transferList).flat(1)
+    if (transferList.length === 0 && Object.values(sourceBackupByJobId).length > 0) {
+      if (transferObjectsList.some(transferObject => transferObject.filtered)) {
+        // Happens if any job transfer list wasn't empty before filter and is empty after filter.
+        Task.info('This VM is excluded by the job filter')
+      } else {
+        Task.info('No new data to upload for this VM')
+      }
+    }
+    return transferList
   }
 
   async run($defer) {

package/_runners/_writers/FullRemoteWriter.mjs

@@ -33,7 +33,7 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
     let metadata = await this._isAlreadyTransferred(timestamp)
     if (metadata !== undefined) {
       // @todo : should skip backup while being vigilant to not stuck the forked stream
-      Task.info('This backup has already been transfered')
+      Task.info('This backup has already been transferred')
     }
 
     const oldBackups = getOldEntries(

package/_runners/_writers/_MixinRemoteWriter.mjs

@@ -99,7 +99,7 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
             srUuid,
             xapi,
             settings: {
-              additionnalVmTag: 'xo:no-bak=Health Check',
+              additionalVmTag: 'xo:no-bak=Health Check',
             },
           }).run()
           let restoredVm

package/_runners/_writers/_MixinXapiWriter.mjs

@@ -29,7 +29,7 @@ export const MixinXapiWriter = (BaseClass = Object) =>
       // the SR that the VM has been replicated on
       const sr = this._sr
       assert.notStrictEqual(sr, undefined, 'SR should be defined before making a health check')
-      assert.notEqual(this._targetVmRef, undefined, 'A vm should have been transfered to be health checked')
+      assert.notEqual(this._targetVmRef, undefined, 'A vm should have been transferred to be health checked')
       // copy VM
       return Task.run(
         {

package/formatVmBackups.mjs

@@ -43,6 +43,7 @@ function formatVmBackup(backup) {
       uuid: backup.vm.uuid,
       name_description: backup.vm.name_description,
       name_label: backup.vm.name_label,
+      tags: backup.vm.tags,
     },
 
     differencingVhds,

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.66.0",
+  "version": "0.67.1",
   "engines": {
     "node": ">=14.18"
   },
@@ -26,11 +26,11 @@
     "@vates/disposable": "^0.1.6",
     "@vates/fuse-vhd": "^2.1.2",
     "@vates/generator-toolbox": "^1.1.0",
-    "@vates/nbd-client": "^3.2.1",
+    "@vates/nbd-client": "^3.2.2",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/disk-transform": "^1.2.0",
-    "@xen-orchestra/fs": "^4.6.4",
+    "@xen-orchestra/fs": "^4.6.5",
     "@xen-orchestra/log": "^0.7.1",
     "@xen-orchestra/qcow2": "^1.1.0",
     "@xen-orchestra/template": "^0.1.0",
@@ -51,8 +51,8 @@
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^4.14.3",
-    "xen-api": "^4.7.4",
+    "vhd-lib": "^4.14.5",
+    "xen-api": "^4.7.5",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -62,7 +62,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^8.5.0"
+    "@xen-orchestra/xapi": "^8.6.1"
   },
   "license": "AGPL-3.0-or-later",
   "author": {