@xen-orchestra/backups 0.26.0 → 0.27.2

package/RemoteAdapter.js

@@ -279,7 +279,7 @@ class RemoteAdapter {
     const dirs = new Set(files.map(file => dirname(file)))
     for (const dir of dirs) {
       // don't merge in main process, unused VHDs will be merged in the next backup run
-      await this.cleanVm(dir, { remove: true, onLog: warn })
+      await this.cleanVm(dir, { remove: true, logWarn: warn })
     }
 
     const dedupedVmUuid = new Set(metadatas.map(_ => _.vm.uuid))

package/_cleanVm.js

@@ -2,6 +2,7 @@
 
 const assert = require('assert')
 const sum = require('lodash/sum')
+const UUID = require('uuid')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
 const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
@@ -50,16 +51,21 @@ const computeVhdsSize = (handler, vhdPaths) =>
 async function mergeVhdChain(chain, { handler, logInfo, remove, merge }) {
   assert(chain.length >= 2)
   const chainCopy = [...chain]
-  const parent = chainCopy.pop()
+  const parent = chainCopy.shift()
   const children = chainCopy
 
   if (merge) {
-    logInfo(`merging children into parent`, { childrenCount: children.length, parent })
+    logInfo('will merge children into parent', { children, parent })
 
     let done, total
     const handle = setInterval(() => {
       if (done !== undefined) {
-        logInfo(`merging children in progress`, { children, parent, doneCount: done, totalCount: total })
+        logInfo('merge in progress', {
+          done,
+          parent,
+          progress: Math.round((100 * done) / total),
+          total,
+        })
       }
     }, 10e3)
 
@@ -121,15 +127,15 @@ async function checkAliases(
   { handler, logInfo = noop, logWarn = console.warn, remove = false }
 ) {
   const aliasFound = []
-  for (const path of aliasPaths) {
-    const target = await resolveVhdAlias(handler, path)
+  for (const alias of aliasPaths) {
+    const target = await resolveVhdAlias(handler, alias)
 
     if (!isVhdFile(target)) {
-      logWarn('alias references non VHD target', { path, target })
+      logWarn('alias references non VHD target', { alias, target })
       if (remove) {
-        logInfo('removing alias and non VHD target', { path, target })
+        logInfo('removing alias and non VHD target', { alias, target })
         await handler.unlink(target)
-        await handler.unlink(path)
+        await handler.unlink(alias)
       }
       continue
     }
@@ -142,13 +148,13 @@ async function checkAliases(
         // error during dispose should not trigger a deletion
       }
     } catch (error) {
-      logWarn('missing or broken alias target', { target, path, error })
+      logWarn('missing or broken alias target', { alias, target, error })
       if (remove) {
         try {
-          await VhdAbstract.unlink(handler, path)
+          await VhdAbstract.unlink(handler, alias)
         } catch (error) {
           if (error.code !== 'ENOENT') {
-            logWarn('error deleting alias target', { target, path, error })
+            logWarn('error deleting alias target', { alias, target, error })
           }
         }
       }
@@ -158,17 +164,17 @@ async function checkAliases(
     aliasFound.push(resolve('/', target))
   }
 
-  const entries = await handler.list(targetDataRepository, {
+  const vhds = await handler.list(targetDataRepository, {
     ignoreMissing: true,
     prependDir: true,
   })
 
-  entries.forEach(async entry => {
-    if (!aliasFound.includes(entry)) {
-      logWarn('no alias references VHD', { entry })
+  await asyncMap(vhds, async path => {
+    if (!aliasFound.includes(path)) {
+      logWarn('no alias references VHD', { path })
       if (remove) {
-        logInfo('deleting unaliased VHD')
-        await VhdAbstract.unlink(handler, entry)
+        logInfo('deleting unused VHD', { path })
+        await VhdAbstract.unlink(handler, path)
       }
     }
   })
@@ -187,6 +193,7 @@ exports.cleanVm = async function cleanVm(
   const handler = this._handler
 
   const vhdsToJSons = new Set()
+  const vhdById = new Map()
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
@@ -208,12 +215,33 @@ exports.cleanVm = async function cleanVm(
           }
           vhdChildren[parent] = path
         }
+        // Detect VHDs with the same UUIDs
+        //
+        // Due to a bug introduced in a1bcd35e2
+        const duplicate = vhdById.get(UUID.stringify(vhd.footer.uuid))
+        let vhdKept = vhd
+        if (duplicate !== undefined) {
+          logWarn('uuid is duplicated', { uuid: UUID.stringify(vhd.footer.uuid) })
+          if (duplicate.containsAllDataOf(vhd)) {
+            logWarn(`should delete ${path}`)
+            vhdKept = duplicate
+            vhds.delete(path)
+          } else if (vhd.containsAllDataOf(duplicate)) {
+            logWarn(`should delete ${duplicate._path}`)
+            vhds.delete(duplicate._path)
+          } else {
+            logWarn('same ids but different content')
+          }
+        } else {
+          logInfo('not duplicate', UUID.stringify(vhd.footer.uuid), path)
+        }
+        vhdById.set(UUID.stringify(vhdKept.footer.uuid), vhdKept)
       })
     } catch (error) {
       vhds.delete(path)
       logWarn('VHD check error', { path, error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
-        logInfo('deleting broken path', { path })
+        logInfo('deleting broken VHD', { path })
         return VhdAbstract.unlink(handler, path)
       }
     }
@@ -261,9 +289,9 @@ exports.cleanVm = async function cleanVm(
       if (!vhds.has(parent)) {
         vhds.delete(vhdPath)
 
-        logWarn('parent VHD is missing', { parent, vhdPath })
+        logWarn('parent VHD is missing', { parent, child: vhdPath })
         if (remove) {
-          logInfo('deleting orphan VHD', { vhdPath })
+          logInfo('deleting orphan VHD', { path: vhdPath })
           deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
@@ -314,7 +342,7 @@ exports.cleanVm = async function cleanVm(
     try {
       metadata = JSON.parse(await handler.readFile(json))
     } catch (error) {
-      logWarn('failed to read metadata file', { json, error })
+      logWarn('failed to read backup metadata', { path: json, error })
       jsons.delete(json)
       return
     }
@@ -325,9 +353,9 @@ exports.cleanVm = async function cleanVm(
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
       } else {
-        logWarn('metadata XVA is missing', { json })
+        logWarn('the XVA linked to the backup is missing', { backup: json, xva: linkedXva })
         if (remove) {
-          logInfo('deleting incomplete backup', { json })
+          logInfo('deleting incomplete backup', { path: json })
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -348,9 +376,9 @@ exports.cleanVm = async function cleanVm(
           vhdsToJSons[path] = json
         })
       } else {
-        logWarn('some metadata VHDs are missing', { json, missingVhds })
+        logWarn('some VHDs linked to the backup are missing', { backup: json, missingVhds })
         if (remove) {
-          logInfo('deleting incomplete backup', { json })
+          logInfo('deleting incomplete backup', { path: json })
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -362,7 +390,7 @@ exports.cleanVm = async function cleanVm(
   const unusedVhdsDeletion = []
   const toMerge = []
   {
-    // VHD chains (as list from child to ancestor) to merge indexed by last
+    // VHD chains (as list from oldest to most recent) to merge indexed by most recent
     // ancestor
     const vhdChainsToMerge = { __proto__: null }
 
@@ -386,14 +414,14 @@ exports.cleanVm = async function cleanVm(
       if (child !== undefined) {
         const chain = getUsedChildChainOrDelete(child)
         if (chain !== undefined) {
-          chain.push(vhd)
+          chain.unshift(vhd)
           return chain
         }
       }
 
-      logWarn('unused VHD', { vhd })
+      logWarn('unused VHD', { path: vhd })
       if (remove) {
-        logInfo('deleting unused VHD', { vhd })
+        logInfo('deleting unused VHD', { path: vhd })
         unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
@@ -474,11 +502,15 @@ exports.cleanVm = async function cleanVm(
 
         // don't warn if the size has changed after a merge
         if (!merged && fileSystemSize !== size) {
-          logWarn('incorrect size in metadata', { size: size ?? 'none', fileSystemSize })
+          logWarn('incorrect backup size in metadata', {
+            path: metadataPath,
+            actual: size ?? 'none',
+            expected: fileSystemSize,
+          })
         }
       }
     } catch (error) {
-      logWarn('failed to get metadata size', { metadataPath, error })
+      logWarn('failed to get backup size', { backup: metadataPath, error })
       return
     }
 
@@ -488,7 +520,7 @@ exports.cleanVm = async function cleanVm(
       try {
         await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
       } catch (error) {
-        logWarn('metadata size update failed', { metadataPath, error })
+        logWarn('failed to update backup size in metadata', { path: metadataPath, error })
       }
     }
   })

package/merge-worker/cli.js

@@ -64,7 +64,7 @@ const main = Disposable.wrap(async function* main(args) {
     try {
       const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
       try {
-        await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+        await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
       } catch (error) {
         // consider the clean successful if the VM dir is missing
         if (error.code !== 'ENOENT') {

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.26.0",
+  "version": "0.27.2",
   "engines": {
     "node": ">=14.6"
   },
@@ -22,7 +22,7 @@
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^1.0.3",
+    "@xen-orchestra/fs": "^2.0.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
@@ -38,7 +38,7 @@
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
     "uuid": "^8.3.2",
-    "vhd-lib": "^3.2.0",
+    "vhd-lib": "^3.3.3",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -46,7 +46,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^1.2.0"
+    "@xen-orchestra/xapi": "^1.4.1"
   },
   "license": "AGPL-3.0-or-later",
   "author": {