@xen-orchestra/backups 0.67.1 → 0.68.0

package/ImportVmBackup.mjs

@@ -11,6 +11,7 @@ import pickBy from 'lodash/pickBy.js'
 import { defer } from 'golike-defer'
 import { NegativeDisk } from '@xen-orchestra/disk-transform'
 import { openDiskChain } from './disks/openDiskChain.mjs'
+import { resetVmOtherConfig } from './_otherConfig.mjs'
 
 const { debug, info, warn } = createLogger('xo:backups:importVmBackup')
 async function resolveUuid(xapi, cache, uuid, type) {
@@ -270,6 +271,7 @@ export class ImportVmBackup {
             `${metadata.vm.name_label} (${formatFilenameDate(metadata.timestamp)})`
           ),
           xapi.call('VM.set_name_description', vmRef, desc),
+          resetVmOtherConfig(xapi, vmRef),
         ])
 
         return {

package/RemoteAdapter.mjs

@@ -18,7 +18,7 @@ import groupBy from 'lodash/groupBy.js'
 import pDefer from 'promise-toolbox/defer'
 import pickBy from 'lodash/pickBy.js'
 import reduce from 'lodash/reduce.js'
-import tar from 'tar'
+import * as tar from 'tar'
 import zlib from 'zlib'
 
 import { BACKUP_DIR } from './_getVmBackupDir.mjs'

package/_getOldEntries.mjs

@@ -1,5 +1,58 @@
+// @ts-check
+
 import moment from 'moment-timezone'
 import assert from 'node:assert'
+/**
+ * @typedef {import('moment-timezone').Moment} Moment
+ */
+/**
+ * @typedef {Object} LTRSettings
+ * @property {number} [firstHourOfTheDay] - First hour of the day (for daily retention)
+ * @property {number} [firstDayOfWeek] - First day of the week (for weekly retention)
+ * @property {number} [firstDayOfMonth] - First day of the month (for monthly retention)
+ * @property {number} [firstDayOfYear] - First day of the year (for yearly retention)
+ */
+
+/**
+ * @typedef {Object} LTRConfig
+ * @property {number} retention - Number of time buckets to retain
+ * @property {LTRSettings} [settings] - Optional settings for the retention period
+ */
+
+/**
+ * @typedef {Object} LongTermRetention
+ * @property {LTRConfig} [daily] - Daily retention configuration
+ * @property {LTRConfig} [weekly] - Weekly retention configuration
+ * @property {LTRConfig} [monthly] - Monthly retention configuration
+ * @property {LTRConfig} [yearly] - Yearly retention configuration
+ */
+
+/**
+ * @typedef {Object} Entry
+ * @property {number | undefined} timestamp - Unix timestamp of the entry
+ * @property {string} id
+ * @property {*} [key] - Any other properties
+ */
+
+/**
+ * @typedef {Object} DateBucket
+ * @property {number} remaining - Number of remaining buckets to fill
+ * @property {string|null} lastMatchingBucket - Last bucket key that was matched
+ * @property {(date: Moment) => string} formatter - Function to format date into bucket key
+ * @property {Object.<string, Entry>} entries - Map of bucket keys to entries
+ */
+
+/**
+ * @typedef {Object} LTRDefinition
+ * @property {(options: {firstHourOfTheDay?: number, firstDayOfWeek?: number, firstDayOfMonth?: number, firstDayOfYear?: number, dateCreator: (date: Moment) => Moment}) => (date: Moment) => string} makeDateFormatter
+ * @property {string} [ancestor] - Name of the parent retention period
+ */
+
+/**
+ * Creates a function that converts dates to moment objects in a specific timezone
+ * @param {string|undefined} timezone - IANA timezone string (e.g., 'America/New_York')
+ * @returns {(date: Moment | number) => Moment} Function that creates timezoned moment objects
+ */
 
 function instantiateTimezonedDateCreator(timezone) {
   return date => {
@@ -9,9 +62,12 @@ function instantiateTimezonedDateCreator(timezone) {
   }
 }
 
+/**
+ * @type {Object.<string, LTRDefinition>}
+ */
 const LTR_DEFINITIONS = {
   daily: {
-    makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator } = {}) => {
+    makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator }) => {
       return date => {
         const copy = dateCreator(date)
         copy.hour(copy.hour() - firstHourOfTheDay)
@@ -20,11 +76,11 @@ const LTR_DEFINITIONS = {
     },
   },
   weekly: {
-    makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator } = {}) => {
+    makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator }) => {
       return date => {
         const copy = dateCreator(date)
 
-        copy.date(date.date() - firstDayOfWeek)
+        copy.date(copy.date() - firstDayOfWeek)
         // warning, the year in term of week may different from YYYY
         // since the computation of the first week of a year is timezone dependent
         return copy.format('gggg-ww')
@@ -33,7 +89,7 @@ const LTR_DEFINITIONS = {
     ancestor: 'daily',
   },
   monthly: {
-    makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator } = {}) => {
+    makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator }) => {
       return date => {
         const copy = dateCreator(date)
         copy.date(copy.date() - firstDayOfMonth)
@@ -43,7 +99,7 @@ const LTR_DEFINITIONS = {
     ancestor: 'weekly',
   },
   yearly: {
-    makeDateFormatter: ({ firstDayOfYear = 0, dateCreator } = {}) => {
+    makeDateFormatter: ({ firstDayOfYear = 0, dateCreator }) => {
       return date => {
         const copy = dateCreator(date)
         copy.date(copy.date() - firstDayOfYear)
@@ -53,27 +109,23 @@ const LTR_DEFINITIONS = {
     ancestor: 'monthly',
   },
 }
-
-// returns all entries but the last retention-th
 /**
- * return the entries too old to be kept
- *  if multiple entries are i the same time bucket : keep only the most recent one
- *  if an entry is valid in any of the bucket OR the  minRetentionCount : keep it
- *  if a bucket is completely empty : it does not count as one, thus it may extend the retention
- * @returns Array<Backup>
+ * Groups entries into date buckets based on long-term retention configuration
+ * @template {Entry} T
+ * @param {Array<T>} entries - Array of entries sorted in descending order by timestamp
+ * @param {LongTermRetention} longTermRetention - Configuration for retention periods
+ * @param {string | undefined} timezone - IANA timezone string
+ * @returns {Object.<string, DateBucket>} Map of retention period names to their date buckets
  */
-export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
-  assert.strictEqual(
-    typeof minRetentionCount,
-    'number',
-    `minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
-  )
-  assert.strictEqual(
-    minRetentionCount >= 0,
-    true,
-    `minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
-  )
+
+export function getLtrEntries(entries, longTermRetention, timezone) {
+  /**
+   * @type  {Object.<string, DateBucket>}
+   */
   const dateBuckets = {}
+  if (!longTermRetention || Object.keys(longTermRetention).length === 0) {
+    return {}
+  }
   const dateCreator = instantiateTimezonedDateCreator(timezone)
   // only check buckets that have a retention set
   for (const [duration, { retention, settings }] of Object.entries(longTermRetention)) {
@@ -81,7 +133,7 @@ export function getOldEntries(minRetentionCount, entries, { longTermRetention =
     assert.notStrictEqual(
       timezone,
       undefined,
-      `timezone must defined for ltr, got ${JSON.stringify({ minRetentionCount, longTermRetention, timezone })}`
+      `timezone must defined for ltr, got ${JSON.stringify({ longTermRetention, timezone })}`
     )
     assert.strictEqual(
       typeof retention,
@@ -101,45 +153,92 @@ export function getOldEntries(minRetentionCount, entries, { longTermRetention =
     }
   }
   const nb = entries.length
-  const minDurationEntries = []
   let previousTimestamp = -1
   for (let i = nb - 1; i >= 0; i--) {
     const entry = entries[i]
-    if (entry.timestamp !== undefined) {
-      // we go through the entries from the last (most recent) to the first (oldest)
-      assert.ok(
-        previousTimestamp === -1 || entry.timestamp < previousTimestamp,
-        `entries must be sorted in desc order ${new Date(entry.timestamp)} , previous :  > ${new Date(previousTimestamp)} `
+    // force it so the compiler is sure we have a timestamp
+    if (entry.timestamp === undefined) {
+      assert.notStrictEqual(
+        entry.timestamp,
+        undefined,
+        `Can't compute long term retention if entries don't have a timestamp`
       )
-      previousTimestamp = entry.timestamp
-      const entryDate = dateCreator(entry.timestamp)
-      for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
-        const bucket = formatter(entryDate)
-        if (lastMatchingBucket !== bucket) {
-          if (remaining === 0) {
-            continue
-          }
-          dateBuckets[duration].lastMatchingBucket = bucket
-          dateBuckets[duration].remaining -= 1
+      continue
+    }
+    // we go through the entries from the last (most recent) to the first (oldest)
+    assert.ok(
+      previousTimestamp === -1 || entry.timestamp < previousTimestamp,
+      `entries must be sorted in desc order ${new Date(entry.timestamp)} , previous :  > ${new Date(previousTimestamp)} `
+    )
+    previousTimestamp = entry.timestamp
+    const entryDate = dateCreator(entry.timestamp)
+    for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
+      const bucket = formatter(entryDate)
+      if (lastMatchingBucket !== bucket) {
+        if (remaining === 0) {
+          continue
         }
-        dateBuckets[duration].entries[bucket] = entry
+        dateBuckets[duration].lastMatchingBucket = bucket
+        dateBuckets[duration].remaining -= 1
       }
-    } else {
-      // replicated VM entries or snapshot retention don't have a timestamp
-      // but also can't have LTR
-      assert.deepStrictEqual(
-        longTermRetention,
-        {},
-        "Can't compute long term retention if entries don't have a timestamp"
-      )
+      dateBuckets[duration].entries[bucket] = entry
     }
+  }
+
+  return dateBuckets
+}
 
-    // also keep it on retention
-    if (i >= nb - minRetentionCount) {
-      minDurationEntries.push(entry)
+/**
+ * return the status of an entry
+ * @param {Array<Entry>} entries
+ * @param {Entry} entry
+ * @param {LongTermRetention} longTermRetention - Configuration for retention periods
+ * @param {string | undefined} timezone - IANA timezone string
+ * @returns {Array<{duration: string, dateBucket:string}>}
+ */
+export function getEntryStatus(entries, entry, longTermRetention, timezone) {
+  const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
+  const entryBuckets = []
+  for (const [duration, { entries }] of Object.entries(dateBuckets)) {
+    for (const [dateBucket, bucketEntry] of Object.entries(entries)) {
+      if (entry.id === bucketEntry.id) {
+        entryBuckets.push({ duration, dateBucket })
+      }
     }
   }
-  const kept = new Set(minDurationEntries)
+  return entryBuckets
+}
+
+/**
+ * return the entries too old to be kept
+ *  if multiple entries are in the same time bucket: keep only the oldest one
+ *  if an entry is valid in any of the bucket OR the minRetentionCount: keep it
+ *  if a bucket is completely empty: it does not count as one, thus it may extend the retention
+ * if an entry is within the most recents minRetentionCount entries : keep it
+ *
+ * @param {number} minRetentionCount - Minimum number of most recent entries to always keep
+ * @param {Array<Entry>} entries - Array of entries sorted in descending order by timestamp
+ * @param {Object} options - Additional options
+ * @param {LongTermRetention} [options.longTermRetention={}] - Configuration for retention periods
+ * @param {string | undefined} [options.timezone] - IANA timezone string
+ * @returns {Array<Entry>} Array of entries that can be removed
+ */
+export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
+  assert.strictEqual(
+    typeof minRetentionCount,
+    'number',
+    `minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
+  )
+  assert.ok(
+    minRetentionCount >= 0,
+    `minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
+  )
+  const dateBuckets = getLtrEntries(entries, longTermRetention, timezone)
+
+  const kept = new Set(entries.filter((_, index) => index >= entries.length - minRetentionCount))
+  /**
+   * @type {Set<Entry>}
+   */
   for (const { entries } of Object.values(dateBuckets)) {
     for (const entry of Object.values(entries)) {
       kept.add(entry)
@@ -147,7 +246,7 @@ export function getOldEntries(minRetentionCount, entries, { longTermRetention =
   }
 
   // ensure order is the same as the source
-  const oldEntries = entries.filter(entry => {
+  const oldEntries = entries.filter((entry, index) => {
     return !kept.has(entry)
   })
 

package/_otherConfig.mjs

@@ -70,7 +70,8 @@ export async function getVmDeltaChainLength(xapi, vmRef) {
 
 /**
  *
- * Reset the other_config field of a VM and its VDIs
+ * Reset the other_config field related to backups of a VM and its VDIs
+ *
  *
  * @param {Xapi} xapi
  * @param {String} vmRef
@@ -90,39 +91,6 @@ export function resetVmOtherConfig(xapi, vmRef) {
   })
 }
 
-/**
- *
- * used to ensure compatibility with the previous snapshots that were having the config stored only into VM
- *
- * @param {Xapi} xapi
- * @param {String} vmRef
- * @returns {Promise}
- */
-export async function populateVdisOtherConfig(xapi, vmRef) {
-  const otherConfig = await xapi.getField('VM', vmRef, 'other_config')
-  const {
-    [DATETIME]: datetime,
-    [DELTA_CHAIN_LENGTH]: chainLength,
-    [EXPORTED_SUCCESSFULLY]: successfully,
-    [JOB_ID]: jobId,
-    [REPLICATED_TO_SR_UUID]: replicatedTo,
-    [SCHEDULE_ID]: scheduleId,
-    [VM_UUID]: vmUuid,
-  } = otherConfig
-
-  return applyToVmAndVdis(xapi, vmRef, (type, ref) =>
-    xapi.setFieldEntries(type, ref, 'other_config', {
-      [DATETIME]: datetime,
-      [DELTA_CHAIN_LENGTH]: chainLength,
-      [EXPORTED_SUCCESSFULLY]: successfully,
-      [JOB_ID]: jobId,
-      [REPLICATED_TO_SR_UUID]: replicatedTo,
-      [SCHEDULE_ID]: scheduleId,
-      [VM_UUID]: vmUuid,
-    })
-  )
-}
-
 /**
  *
  * set the other_config key related to a backup of a VM and its associated VDIs

package/_runners/_vmRunners/FullXapi.mjs

@@ -64,9 +64,16 @@ export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
       const vdi = await this._xapi.getRecord('VDI', vdiRef)
 
       // the size a of fully allocated vdi will be virtual_size exactly, it's a gross over evaluation
-      // of the real stream size in general, since a disk is never completely full
+      // of the real stream size in general, since a disk is rarely completely full
       // vdi.physical_size seems to underevaluate a lot the real disk usage of a VDI, as of 2023-10-30
-      maxStreamLength += vdi.virtual_size
+
+      // xva files are tar archive with data cut in 1MB blocks
+      // each block is signed with a hash (in xxhash format)
+      // each of these blocks + hash add one tar header entry in the file
+      const XVA_BLOCK_SIZE = 1024 * 1024
+      const nbBlocks = Math.ceil(vdi.virtual_size / XVA_BLOCK_SIZE)
+      const headersSize = nbBlocks * (512 /* file header */ + 512 /* xxhash header */ + 512) /* xhxhash size */
+      maxStreamLength += XVA_BLOCK_SIZE * nbBlocks + headersSize
     }
 
     const sizeContainer = watchStreamSize(stream)

package/_runners/_vmRunners/_AbstractXapi.mjs

@@ -1,5 +1,6 @@
 import assert from 'node:assert'
 import groupBy from 'lodash/groupBy.js'
+import { createLogger } from '@xen-orchestra/log'
 import ignoreErrors from 'promise-toolbox/ignoreErrors'
 import { asyncMap } from '@xen-orchestra/async-map'
 import { decorateMethodsWith } from '@vates/decorate-with'
@@ -8,14 +9,9 @@ import { defer } from 'golike-defer'
 import { getOldEntries } from '../../_getOldEntries.mjs'
 import { Task } from '../../Task.mjs'
 import { Abstract } from './_Abstract.mjs'
-import {
-  DATETIME,
-  JOB_ID,
-  SCHEDULE_ID,
-  populateVdisOtherConfig,
-  resetVmOtherConfig,
-  setVmOtherConfig,
-} from '../../_otherConfig.mjs'
+import { DATETIME, JOB_ID, SCHEDULE_ID, VM_UUID, resetVmOtherConfig, setVmOtherConfig } from '../../_otherConfig.mjs'
+
+const { warn, info } = createLogger('xo:backups:AbstractXapi')
 
 export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
   constructor({
@@ -179,43 +175,113 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     }
   }
 
+  // handle snapshot by VDI since snapshot of cbtDestroySnapshotData jobs
+  // aren't attached to any VM
+
+  // look at all the vdi snapshots ( snapshot_of not empty )
+  // with the same vm_uuid and job_uuid
+  //   for cbt_metadata list them unconditionnaly to remove older one
+  //   for other: only list them if they are attached to a VM snapshot
+  //  and if this vm snapshot is also part of the backup
+  //   ensure they are attached to only one vm snapshot
+  //   ensure any VM-snapshot harvested by this has all its disk harvested (no mix of vdi snapshot from this job and not)
+
   async _fetchJobSnapshots() {
     const jobId = this._jobId
-    const vmRef = this._vm.$ref
     const xapi = this._xapi
 
-    // to ensure compatibility with snapshots older than CBT implementation
-    // update vdi data to ensure the vdi are correctly fetched in _jobSnapshotVdis
-    // remove by then end of 2024
-    const vmSnapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
-    const vmSnapshotsOtherConfig = await asyncMap(vmSnapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
+    const vdiCandidates = {}
 
-    const vmSnapshots = []
-    vmSnapshotsOtherConfig.forEach((other_config, i) => {
-      if (other_config[JOB_ID] === jobId) {
-        vmSnapshots.push({ other_config, $ref: vmSnapshotsRef[i] })
+    Object.values(xapi.objects.indexes.type.VDI)
+      .filter(_ => !!_) // filter nullish
+      .filter(({ other_config, $snapshot_of }) => {
+        return $snapshot_of !== undefined && other_config[JOB_ID] === jobId && other_config[VM_UUID] === this._vm.uuid
+      })
+      .forEach(vdi => {
+        vdiCandidates[vdi.uuid] = vdi
+      })
+
+    // check that user snapshots are clean
+
+    for (const vdi of Object.values(vdiCandidates)) {
+      // cbt metadata are always considered linked to a backup job
+      // if they have the right other_config
+      if (vdi.type === 'cbt_metadata') {
+        continue
       }
-    })
-    await Promise.all(vmSnapshots.map(snapshot => populateVdisOtherConfig(xapi, snapshot.$ref)))
-    // end of compatibility handling
-
-    // handle snapshot by VDI
-    this._jobSnapshotVdis = []
-    const srcVdis = await xapi.getRecords('VDI', await this._vm.$getDisks())
-    for (const srcVdi of srcVdis) {
-      const snapshots = await xapi.getRecords('VDI', srcVdi.snapshots)
-      for (const snapshot of snapshots) {
-        // only keep the snapshot related to this backup job
-        // and only if the job is still using  purge snapshot data or if the disk
-        // is not a cbt metadata disk ( expect a type: user for normal disks)
-        if (
-          snapshot.other_config[JOB_ID] === jobId &&
-          (this._settings.cbtDestroySnapshotData || snapshot.type !== 'cbt_metadata')
-        ) {
-          this._jobSnapshotVdis.push(snapshot)
-        }
+      const vbds = vdi.$VBDs
+        .filter(({ $VM }) => !!$VM) // filter empty VMs
+        .filter(({ $VM }) => $VM.is_control_domain === false) // do not handle control domain
+      if (vbds.length === 0) {
+        // orphan vdi snapshot
+        info(
+          `disk snapshot ${vdi.name_label} is orphan or attached only to control domain,
+          it will be removed at the end of a successful backup run`,
+          { vdi, attachedto: vdi.$VBDs.map(vbd => vbd?.$VM) }
+        )
+        continue
       }
+      const userVms = vbds.map(({ $VM }) => $VM)
+      if (vbds.length > 1) {
+        warn(
+          `vdi ${vdi.name_label} (${vdi.uuid}) is linked to multipe vms :  ${userVms.map(({ name_label, uuid }) => `${name_label} ${uuid}`).join(', ')}. 
+              This disk snapshot will be excluded from the backup cleaning`,
+          { vdi, userVms }
+        )
+        delete vdiCandidates[vdi.uuid]
+        continue
+      }
+
+      const vm = vbds[0].$VM
+
+      // xapi has some issue regarding vdi snapshot attached to non snapshot VM
+      // it can also be some user action forcibely linking a vdi snapshot to a vm
+      // => we exclude these from the backup processing
+      if (vm.$snapshot_of === undefined) {
+        warn(
+          `vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a non snapshot vm ${vm.name_label} ${vm.uuid}. 
+          This disk snapshot will be excluded from the backup cleaning`,
+          { vdi, vm }
+        )
+        delete vdiCandidates[vdi.uuid]
+        continue
+      }
+
+      // vdi is attached only to a snapshot that is not a backup snapshot
+      // we don't check scheduleId since we are looking for all the snapshot of this job
+      // => excludes from the list to be cleared
+      if (
+        vm.other_config[DATETIME] !== vdi.other_config[DATETIME] ||
+        vm.other_config[JOB_ID] !== vdi.other_config[JOB_ID] ||
+        vm.other_config[VM_UUID] !== vdi.other_config[VM_UUID]
+      ) {
+        warn(
+          `vdi ${vdi.name_label} (${vdi.uuid}) is a snapshot linked to a snapshot vm ${vm.name_label} ${vm.uuid} out of this backup job scope. 
+          This disk snapshot will be excluded from the backup cleaning`,
+          { vdi, vm }
+        )
+        delete vdiCandidates[vdi.uuid]
+        continue
+      }
+
+      // check if all the disks of these VM snapshot have been harvested
+      // if not => remove it from the list to ensure we won't half destroy VM later
+      vm.$VBDs
+        .filter(({ $VDI }) => !!$VDI) // filter missing keys
+        .filter(({ $VDI }) => $VDI && vdiCandidates[$VDI.uuid] === undefined)
+        .forEach(({ $VDI: outOfSnapshotsVdi, ...other }) => {
+          warn(
+            `vdi ${vdi.name_label} ${vdi.uuid} is recognized as a snapshot of the backup job,
+           linked to vm ${vm.name_label} ${vm.uuid} but vdi ${outOfSnapshotsVdi.name_label} ${outOfSnapshotsVdi.uuid} 
+           is not linked to the job. This disk snapshot will be excluded from the backup cleaning`,
+            { vdi, vm, vbds, outOfSnapshotsVdi }
+          )
+          // this will be called multiple time but it is not really an issue
+          delete vdiCandidates[vdi.uuid]
+        })
     }
+
+    this._jobSnapshotVdis = Object.values(vdiCandidates)
   }
 
   async _removeUnusedSnapshots() {
@@ -255,28 +321,47 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
           return
         }
         const vdis = snapshotPerDatetime[datetime]
-        let vmRef
+        let vm
         // if there is an attached VM => destroy the VM (Non CBT backups)
         for (const vdi of vdis) {
           const vbds = vdi.$VBDs.filter(({ $VM }) => $VM.is_control_domain === false)
           if (vbds.length > 0) {
             // only one VM linked to this vdi
             // this will throw error for VDI still attached to control domain
+            // since we won't be able to remove an attached VDI
             assert.strictEqual(vbds.length, 1, 'VDI must be free or attached to exactly one VM')
-            const vm = vbds[0].$VM
-            assert.strictEqual(vm.is_a_snapshot, true, `VM must be a snapshot`) // don't delete a VM (especially a control domain)
-
-            const vmRefVdi = vm.$ref
-            // same vm than other vdi of the same batch
-            assert.ok(
-              vmRef === undefined || vmRef === vmRefVdi,
-              '_removeUnusedSnapshots don t handle vdi related to multiple VMs '
-            )
-            vmRef = vmRefVdi
+            const vdiVm = vbds[0].$VM
+            if (vdiVm.$snapshot_of === undefined) {
+              // don't delete a VM (especially a control domain)
+              warn(
+                `VM ${vdiVm.uuid} (${vdiVm.name_label}) linked to VDI ${vdi.uuid} (${vdi.name_label}) should be a snapshot`
+              )
+              throw new Error(`VM must be a snapshot`)
+            }
+            if (vm !== undefined && vm.$ref !== vdiVm.$ref) {
+              // this VDI is attached to another VM than the other vdi of
+              // this batch
+              // in doubt, do not delete anything
+              warn("_removeUnusedSnapshots don't handle vdi related to multiple VMs ", {
+                vm1: {
+                  label: vm.name_label,
+                  id: vm.$id,
+                },
+                vm2: {
+                  label: vdiVm.name_label,
+                  id: vdiVm.$id,
+                },
+                vdis: vdis.map(({ name_label, $id }) => ({ name_label, $id })),
+              })
+              throw new Error(
+                `_removeUnusedSnapshots don't handle vdi related to multiple VMs ${vm.name_label} and ${vdiVm.name_label}`
+              )
+            }
+            vm = vdiVm
           }
         }
-        if (vmRef !== undefined) {
-          return xapi.VM_destroy(vmRef)
+        if (vm?.$ref !== undefined) {
+          return xapi.VM_destroy(vm.$ref)
         } else {
           return asyncMap(
             vdis.map(async ({ $ref }) => {
@@ -293,8 +378,9 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
     // going back to a previous version of XO not supporting CBT will create a full backup
     // this will only do something after snapshot and transfer
     if (
+      this._exportedVm !== undefined &&
       // don't modify the VM
-      this._exportedVm?.is_a_snapshot &&
+      this._exportedVm.$snapshot_of !== undefined &&
       // user don't want to keep the snapshot data
       this._settings.snapshotRetention === 0 &&
       // preferNbd is not a guarantee that the backup used NBD, depending on the network configuration,

package/_runners/_writers/FullRemoteWriter.mjs

@@ -73,6 +73,7 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
       return { size: sizeContainer.size }
     })
     metadata.size = sizeContainer.size
+    metadata.tags = await this.getLongTermRetentionTags(metadata)
     this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadata)
 
     if (!deleteFirst) {

package/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -244,6 +244,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
 
       return { size }
     })
+    metadataContent.tags = await this.getLongTermRetentionTags(metadataContent)
     metadataContent.size = size // @todo return exactly the size written by this writer
     this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
 

package/_runners/_writers/_AbstractWriter.mjs

@@ -1,6 +1,3 @@
-import { formatFilenameDate } from '../../_filenameDate.mjs'
-import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
-
 export class AbstractWriter {
   constructor({ config, healthCheckSr, job, vmUuid, scheduleId, settings }) {
     this._config = config
@@ -16,14 +13,4 @@ export class AbstractWriter {
   afterBackup() {}
 
   healthCheck(sr) {}
-
-  _isAlreadyTransferred(timestamp) {
-    const vmUuid = this._vmUuid
-    const adapter = this._adapter
-    const backupDir = getVmBackupDir(vmUuid)
-    try {
-      const actualMetadata = JSON.parse(adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`))
-      return actualMetadata
-    } catch (error) {}
-  }
 }

package/_runners/_writers/_MixinRemoteWriter.mjs

@@ -9,6 +9,7 @@ import { ImportVmBackup } from '../../ImportVmBackup.mjs'
 import { Task } from '../../Task.mjs'
 import * as MergeWorker from '../../merge-worker/index.mjs'
 import ms from 'ms'
+import { getEntryStatus } from '../../_getOldEntries.mjs'
 
 const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
 
@@ -47,6 +48,19 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
       }
     }
 
+    async getLongTermRetentionTags(currentEntry) {
+      const settings = this._settings
+      const scheduleId = this._scheduleId
+      const vmUuid = this._vmUuid
+      const adapter = this._adapter
+
+      const entries = await adapter.listVmBackups(vmUuid, _ => _.scheduleId === scheduleId)
+      entries.push(currentEntry)
+      const buckets = getEntryStatus(entries, currentEntry, settings.longTermRetention, settings.timezone)
+
+      return buckets.map(({ duration, dateBucket }) => `${duration}=${dateBucket}`)
+    }
+
     async beforeBackup() {
       const { handler } = this._adapter
       const vmBackupDir = this._vmBackupDir

package/formatVmBackups.mjs

@@ -45,7 +45,7 @@ function formatVmBackup(backup) {
       name_label: backup.vm.name_label,
       tags: backup.vm.tags,
     },
-
+    tags: backup.tags ?? [],
     differencingVhds,
     dynamicVhds,
     withMemory,

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.67.1",
+  "version": "0.68.0",
   "engines": {
     "node": ">=14.18"
   },
@@ -29,10 +29,10 @@
     "@vates/nbd-client": "^3.2.2",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/disk-transform": "^1.2.0",
+    "@xen-orchestra/disk-transform": "^1.2.1",
     "@xen-orchestra/fs": "^4.6.5",
     "@xen-orchestra/log": "^0.7.1",
-    "@xen-orchestra/qcow2": "^1.1.0",
+    "@xen-orchestra/qcow2": "^1.1.2",
     "@xen-orchestra/template": "^0.1.0",
     "app-conf": "^3.0.0",
     "compare-versions": "^6.0.0",
@@ -48,11 +48,11 @@
     "parse-pairs": "^2.0.0",
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
-    "tar": "^6.1.15",
+    "tar": "^7.5.3",
     "uuid": "^9.0.0",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^4.14.5",
-    "xen-api": "^4.7.5",
+    "vhd-lib": "^4.14.7",
+    "xen-api": "^4.7.6",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -62,7 +62,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^8.6.1"
+    "@xen-orchestra/xapi": "^8.6.4"
   },
   "license": "AGPL-3.0-or-later",
   "author": {