@xen-orchestra/backups 0.54.3 → 0.55.0

package/ImportVmBackup.mjs

@@ -58,7 +58,7 @@ export class ImportVmBackup {
   async _reuseNearestSnapshot($defer, ignoredVdis) {
     const metadata = this._metadata
     const { mapVdisSrs } = this._importIncrementalVmSettings
-    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
+    const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
     const streams = {}
     const metdataDir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
@@ -191,6 +191,7 @@ export class ImportVmBackup {
       version: '1.0.0',
       vifs,
       vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
+      vtpms,
     }
   }
 

package/RemoteAdapter.mjs

@@ -744,7 +744,7 @@ export class RemoteAdapter {
 
   async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
     const handler = this._handler
-    const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
+    const { vbds, vhds, vifs, vm, vmSnapshot, vtpms } = metadata
     const dir = dirname(metadata._filename)
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
 
@@ -760,6 +760,7 @@ export class RemoteAdapter {
       version: '1.0.0',
       vifs,
       vm: { ...vm, suspend_VDI: vmSnapshot.suspend_VDI },
+      vtpms,
     }
   }
 

package/_getOldEntries.mjs

@@ -1,4 +1,136 @@
+import moment from 'moment-timezone'
+import assert from 'node:assert'
+
+function instantiateTimezonedDateCreator(timezone) {
+  return date => {
+    const transformed = timezone ? moment.tz(date, timezone) : moment(date)
+    assert.ok(transformed.isValid(), `date ${date} , timezone ${timezone} is invalid`)
+    return transformed
+  }
+}
+
+const LTR_DEFINITIONS = {
+  daily: {
+    makeDateFormatter: ({ firstHourOfTheDay = 0, dateCreator } = {}) => {
+      return date => {
+        const copy = dateCreator(date)
+        copy.hour(copy.hour() - firstHourOfTheDay)
+        return copy.format('YYYY-MM-DD')
+      }
+    },
+  },
+  weekly: {
+    makeDateFormatter: ({ firstDayOfWeek = 0 /* relative to timezone week start */, dateCreator } = {}) => {
+      return date => {
+        const copy = dateCreator(date)
+
+        copy.date(date.date() - firstDayOfWeek)
+        // warning, the year in term of week may different from YYYY
+        // since the computation of the first week of a year is timezone dependant
+        return copy.format('gggg-WW')
+      }
+    },
+    ancestor: 'daily',
+  },
+  monthly: {
+    makeDateFormatter: ({ firstDayOfMonth = 0, dateCreator } = {}) => {
+      return date => {
+        const copy = dateCreator(date)
+        copy.date(copy.date() - firstDayOfMonth)
+        return copy.format('YYYY-MM')
+      }
+    },
+    ancestor: 'weekly',
+  },
+  yearly: {
+    makeDateFormatter: ({ firstDayOfYear = 0, dateCreator } = {}) => {
+      return date => {
+        const copy = dateCreator(date)
+        copy.date(copy.date() - firstDayOfYear)
+        return copy.format('YYYY')
+      }
+    },
+    ancestor: 'monthly',
+  },
+}
+
 // returns all entries but the last retention-th
-export function getOldEntries(retention, entries) {
-  return entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries
+/**
+ * return the entries too old to be kept
+ *  if multiple entries are i the same time bucket : keep only the most recent one
+ *  if an entry is valid in any of the bucket OR the  minRetentionCount : keep it
+ *  if a bucket is completly empty : it does not count as one, thus it may extend the retention
+ * @returns Array<Backup>
+ */
+export function getOldEntries(minRetentionCount, entries, { longTermRetention = {}, timezone } = {}) {
+  assert.strictEqual(
+    typeof minRetentionCount,
+    'number',
+    `minRetentionCount must be a number, got ${JSON.stringify(minRetentionCount)}`
+  )
+  assert.strictEqual(
+    minRetentionCount >= 0,
+    true,
+    `minRetentionCount must be a positive number, got ${JSON.stringify(minRetentionCount)}`
+  )
+  const dateBuckets = {}
+  const dateCreator = instantiateTimezonedDateCreator(timezone)
+  // only check buckets that have a retention set
+  for (const [duration, { retention, settings }] of Object.entries(longTermRetention)) {
+    assert.notStrictEqual(LTR_DEFINITIONS[duration], undefined, `Retention of type ${duration} is not defined`)
+    assert.notStrictEqual(
+      timezone,
+      undefined,
+      `timezone must defined for ltr, got ${JSON.stringify({ minRetentionCount, longTermRetention, timezone })}`
+    )
+    assert.strictEqual(
+      typeof retention,
+      'number',
+      `retention  of type ${duration} must be a number, got ${JSON.stringify(retention)}`
+    )
+    assert.strictEqual(
+      retention > 0,
+      true,
+      `retention  of type ${duration} must be a positive number, got ${JSON.stringify(retention)}`
+    )
+    dateBuckets[duration] = {
+      remaining: retention,
+      lastMatchingBucket: null,
+      formatter: LTR_DEFINITIONS[duration].makeDateFormatter({ ...settings, dateCreator }),
+    }
+  }
+  const nb = entries.length
+  const oldEntries = []
+
+  for (let i = nb - 1; i >= 0; i--) {
+    const entry = entries[i]
+    const entryDate = dateCreator(entry.timestamp)
+    let shouldBeKept = false
+    for (const [duration, { remaining, lastMatchingBucket, formatter }] of Object.entries(dateBuckets)) {
+      if (remaining === 0) {
+        continue
+      }
+      const bucket = formatter(entryDate)
+      if (lastMatchingBucket !== bucket) {
+        if (lastMatchingBucket !== null) {
+          assert.strictEqual(
+            lastMatchingBucket > bucket,
+            true,
+            `entries must be sorted in asc order ${lastMatchingBucket} ${bucket}`
+          )
+        }
+        shouldBeKept = true
+        dateBuckets[duration].remaining -= 1
+        dateBuckets[duration].lastMatchingBucket = bucket
+      }
+    }
+    if (i >= nb - minRetentionCount) {
+      shouldBeKept = true
+    }
+    if (!shouldBeKept) {
+      oldEntries.push(entry)
+    }
+  }
+  // we expect the entries to be in the right order
+  return oldEntries.reverse()
 }

package/_incrementalVm.mjs

@@ -104,6 +104,18 @@ export async function exportIncrementalVm(
     }
   })
 
+  const vtpms = await Promise.all(
+    vm.$VTPMs.map(async vtpm => {
+      let content
+      try {
+        content = await vm.$xapi.call('VTPM.get_contents', vtpm.$ref)
+      } catch (err) {
+        console.error(err)
+      }
+      return content
+    })
+  )
+
   return Object.defineProperty(
     {
       version: '1.1.0',
@@ -113,6 +125,7 @@ export async function exportIncrementalVm(
       vm: {
         ...vm,
       },
+      vtpms,
     },
     'streams',
     {
@@ -281,6 +294,12 @@ export const importIncrementalVm = defer(async function importIncrementalVm(
       }
     }),
   ])
+  // recreate VTPMs
+  await Promise.all(
+    (incrementalVm.vtpms ?? []).map(async contents => {
+      await xapi.VTPM_create({ VM: vmRef, contents })
+    })
+  )
 
   await Promise.all([
     incrementalVm.vm.ha_always_run && xapi.setField('VM', vmRef, 'ha_always_run', true),

package/_runners/_writers/FullRemoteWriter.mjs

@@ -38,7 +38,8 @@ export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
 
     const oldBackups = getOldEntries(
       settings.exportRetention - 1,
-      await adapter.listVmBackups(vm.uuid, _ => _.mode === 'full' && _.scheduleId === scheduleId)
+      await adapter.listVmBackups(vm.uuid, _ => _.mode === 'full' && _.scheduleId === scheduleId),
+      { longTermRetention: settings.longTermRetention, timezone: settings.timezone }
     )
     const deleteOldBackups = () => adapter.deleteFullVmBackups(oldBackups)
 

package/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -95,7 +95,8 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
 
     const oldEntries = getOldEntries(
       settings.exportRetention - 1,
-      await adapter.listVmBackups(vmUuid, _ => _.mode === 'delta' && _.scheduleId === scheduleId)
+      await adapter.listVmBackups(vmUuid, _ => _.mode === 'delta' && _.scheduleId === scheduleId),
+      { longTermRetention: settings.longTermRetention, timezone: settings.timezone }
     )
     this._oldEntries = oldEntries
 
@@ -220,6 +221,7 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
       vhds,
       vm,
       vmSnapshot,
+      vtpms: deltaExport.vtpms,
     }
 
     const { size } = await Task.run({ name: 'transfer' }, async () => {

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.54.3",
+  "version": "0.55.0",
   "engines": {
     "node": ">=14.18"
   },
@@ -28,7 +28,7 @@
     "@vates/nbd-client": "^3.1.2",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^4.2.1",
+    "@xen-orchestra/fs": "^4.3.0",
     "@xen-orchestra/log": "^0.7.1",
     "@xen-orchestra/template": "^0.1.0",
     "app-conf": "^3.0.0",
@@ -39,6 +39,7 @@
     "human-format": "^1.2.0",
     "limit-concurrency-decorator": "^0.6.0",
     "lodash": "^4.17.20",
+    "moment-timezone": "^0.5.46",
     "ms": "^2.1.3",
     "node-zone": "^0.4.0",
     "parse-pairs": "^2.0.0",
@@ -47,7 +48,7 @@
     "tar": "^6.1.15",
     "uuid": "^9.0.0",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^4.11.1",
+    "vhd-lib": "^4.11.2",
     "xen-api": "^4.5.0",
     "yazl": "^2.5.1"
   },
@@ -58,7 +59,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^7.7.1"
+    "@xen-orchestra/xapi": "^7.8.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {