@xen-orchestra/backups 0.27.0 → 0.27.3

package/RemoteAdapter.js

@@ -279,7 +279,7 @@ class RemoteAdapter {
     const dirs = new Set(files.map(file => dirname(file)))
     for (const dir of dirs) {
       // don't merge in main process, unused VHDs will be merged in the next backup run
-      await this.cleanVm(dir, { remove: true, onLog: warn })
+      await this.cleanVm(dir, { remove: true, logWarn: warn })
     }
 
     const dedupedVmUuid = new Set(metadatas.map(_ => _.vm.uuid))

package/_cleanVm.js

@@ -13,6 +13,7 @@ const { limitConcurrency } = require('limit-concurrency-decorator')
 
 const { Task } = require('./Task.js')
 const { Disposable } = require('promise-toolbox')
+const handlerPath = require('@xen-orchestra/fs/path')
 
 // checking the size of a vhd directory is costly
 // 1 Http Query per 1000 blocks
@@ -55,12 +56,17 @@ async function mergeVhdChain(chain, { handler, logInfo, remove, merge }) {
   const children = chainCopy
 
   if (merge) {
-    logInfo(`merging children into parent`, { childrenCount: children.length, parent })
+    logInfo('will merge children into parent', { children, parent })
 
     let done, total
     const handle = setInterval(() => {
       if (done !== undefined) {
-        logInfo(`merging children in progress`, { children, parent, doneCount: done, totalCount: total })
+        logInfo('merge in progress', {
+          done,
+          parent,
+          progress: Math.round((100 * done) / total),
+          total,
+        })
       }
     }, 10e3)
 
@@ -81,7 +87,7 @@ async function mergeVhdChain(chain, { handler, logInfo, remove, merge }) {
 const noop = Function.prototype
 
 const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
-const listVhds = async (handler, vmDir) => {
+const listVhds = async (handler, vmDir, logWarn) => {
   const vhds = new Set()
   const aliases = {}
   const interruptedVhds = new Map()
@@ -101,12 +107,23 @@ const listVhds = async (handler, vmDir) => {
             filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
           })
           aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
-          list.forEach(file => {
+
+          await asyncMap(list, async file => {
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
               vhds.add(`${vdiDir}/${file}`)
             } else {
-              interruptedVhds.set(`${vdiDir}/${res[1]}`, `${vdiDir}/${file}`)
+              try {
+                const mergeState = JSON.parse(await handler.readFile(file))
+                interruptedVhds.set(`${vdiDir}/${res[1]}`, {
+                  statePath: `${vdiDir}/${file}`,
+                  chain: mergeState.chain,
+                })
+              } catch (error) {
+                // fall back to a non resuming merge
+                vhds.add(`${vdiDir}/${file}`)
+                logWarn('failed to read existing merge state', { path: file, error })
+              }
             }
           })
         }
@@ -122,15 +139,15 @@ async function checkAliases(
   { handler, logInfo = noop, logWarn = console.warn, remove = false }
 ) {
   const aliasFound = []
-  for (const path of aliasPaths) {
-    const target = await resolveVhdAlias(handler, path)
+  for (const alias of aliasPaths) {
+    const target = await resolveVhdAlias(handler, alias)
 
     if (!isVhdFile(target)) {
-      logWarn('alias references non VHD target', { path, target })
+      logWarn('alias references non VHD target', { alias, target })
       if (remove) {
-        logInfo('removing alias and non VHD target', { path, target })
+        logInfo('removing alias and non VHD target', { alias, target })
         await handler.unlink(target)
-        await handler.unlink(path)
+        await handler.unlink(alias)
       }
       continue
     }
@@ -143,13 +160,13 @@ async function checkAliases(
         // error during dispose should not trigger a deletion
       }
     } catch (error) {
-      logWarn('missing or broken alias target', { target, path, error })
+      logWarn('missing or broken alias target', { alias, target, error })
       if (remove) {
         try {
-          await VhdAbstract.unlink(handler, path)
+          await VhdAbstract.unlink(handler, alias)
         } catch (error) {
           if (error.code !== 'ENOENT') {
-            logWarn('error deleting alias target', { target, path, error })
+            logWarn('error deleting alias target', { alias, target, error })
           }
         }
       }
@@ -159,17 +176,17 @@ async function checkAliases(
     aliasFound.push(resolve('/', target))
   }
 
-  const entries = await handler.list(targetDataRepository, {
+  const vhds = await handler.list(targetDataRepository, {
     ignoreMissing: true,
     prependDir: true,
   })
 
-  entries.forEach(async entry => {
-    if (!aliasFound.includes(entry)) {
-      logWarn('no alias references VHD', { entry })
+  await asyncMap(vhds, async path => {
+    if (!aliasFound.includes(path)) {
+      logWarn('no alias references VHD', { path })
       if (remove) {
-        logInfo('deleting unaliased VHD')
-        await VhdAbstract.unlink(handler, entry)
+        logInfo('deleting unused VHD', { path })
+        await VhdAbstract.unlink(handler, path)
       }
     }
   })
@@ -192,7 +209,7 @@ exports.cleanVm = async function cleanVm(
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
-  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir)
+  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir, logWarn)
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {
@@ -225,10 +242,8 @@ exports.cleanVm = async function cleanVm(
             logWarn(`should delete ${duplicate._path}`)
             vhds.delete(duplicate._path)
           } else {
-            logWarn(`same ids but different content`)
+            logWarn('same ids but different content')
           }
-        } else {
-          logInfo('not duplicate', UUID.stringify(vhd.footer.uuid), path)
         }
         vhdById.set(UUID.stringify(vhdKept.footer.uuid), vhdKept)
       })
@@ -236,7 +251,7 @@ exports.cleanVm = async function cleanVm(
       vhds.delete(path)
       logWarn('VHD check error', { path, error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
-        logInfo('deleting broken path', { path })
+        logInfo('deleting broken VHD', { path })
         return VhdAbstract.unlink(handler, path)
       }
     }
@@ -245,7 +260,7 @@ exports.cleanVm = async function cleanVm(
   // remove interrupted merge states for missing VHDs
   for (const interruptedVhd of interruptedVhds.keys()) {
     if (!vhds.has(interruptedVhd)) {
-      const statePath = interruptedVhds.get(interruptedVhd)
+      const { statePath } = interruptedVhds.get(interruptedVhd)
       interruptedVhds.delete(interruptedVhd)
 
       logWarn('orphan merge state', {
@@ -284,9 +299,9 @@ exports.cleanVm = async function cleanVm(
       if (!vhds.has(parent)) {
         vhds.delete(vhdPath)
 
-        logWarn('parent VHD is missing', { parent, vhdPath })
+        logWarn('parent VHD is missing', { parent, child: vhdPath })
         if (remove) {
-          logInfo('deleting orphan VHD', { vhdPath })
+          logInfo('deleting orphan VHD', { path: vhdPath })
           deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
@@ -337,7 +352,7 @@ exports.cleanVm = async function cleanVm(
     try {
       metadata = JSON.parse(await handler.readFile(json))
     } catch (error) {
-      logWarn('failed to read metadata file', { json, error })
+      logWarn('failed to read backup metadata', { path: json, error })
       jsons.delete(json)
       return
     }
@@ -348,9 +363,9 @@ exports.cleanVm = async function cleanVm(
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
       } else {
-        logWarn('metadata XVA is missing', { json })
+        logWarn('the XVA linked to the backup is missing', { backup: json, xva: linkedXva })
         if (remove) {
-          logInfo('deleting incomplete backup', { json })
+          logInfo('deleting incomplete backup', { path: json })
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -371,9 +386,9 @@ exports.cleanVm = async function cleanVm(
           vhdsToJSons[path] = json
         })
       } else {
-        logWarn('some metadata VHDs are missing', { json, missingVhds })
+        logWarn('some VHDs linked to the backup are missing', { backup: json, missingVhds })
         if (remove) {
-          logInfo('deleting incomplete backup', { json })
+          logInfo('deleting incomplete backup', { path: json })
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -414,9 +429,9 @@ exports.cleanVm = async function cleanVm(
         }
       }
 
-      logWarn('unused VHD', { vhd })
+      logWarn('unused VHD', { path: vhd })
       if (remove) {
-        logInfo('deleting unused VHD', { vhd })
+        logInfo('deleting unused VHD', { path: vhd })
         unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
@@ -427,7 +442,13 @@ exports.cleanVm = async function cleanVm(
 
     // merge interrupted VHDs
     for (const parent of interruptedVhds.keys()) {
-      vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
+      // before #6349 the chain wasn't in the mergeState
+      const { chain, statePath } = interruptedVhds.get(parent)
+      if (chain === undefined) {
+        vhdChainsToMerge[parent] = [parent, vhdChildren[parent]]
+      } else {
+        vhdChainsToMerge[parent] = chain.map(vhdPath => handlerPath.resolveFromFile(statePath, vhdPath))
+      }
     }
 
     Object.values(vhdChainsToMerge).forEach(chain => {
@@ -442,7 +463,7 @@ exports.cleanVm = async function cleanVm(
     await asyncMap(toMerge, async chain => {
       const merged = await limitedMergeVhdChain(chain, { handler, logInfo, logWarn, remove, merge })
       if (merged !== undefined) {
-        const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
+        const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
         metadataWithMergedVhd[metadataPath] = true
       }
     })
@@ -497,11 +518,15 @@ exports.cleanVm = async function cleanVm(
 
         // don't warn if the size has changed after a merge
         if (!merged && fileSystemSize !== size) {
-          logWarn('incorrect size in metadata', { size: size ?? 'none', fileSystemSize })
+          logWarn('incorrect backup size in metadata', {
+            path: metadataPath,
+            actual: size ?? 'none',
+            expected: fileSystemSize,
+          })
         }
       }
     } catch (error) {
-      logWarn('failed to get metadata size', { metadataPath, error })
+      logWarn('failed to get backup size', { backup: metadataPath, error })
       return
     }
 
@@ -511,7 +536,7 @@ exports.cleanVm = async function cleanVm(
       try {
         await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
       } catch (error) {
-        logWarn('metadata size update failed', { metadataPath, error })
+        logWarn('failed to update backup size in metadata', { path: metadataPath, error })
       }
     }
   })

package/merge-worker/cli.js

@@ -64,7 +64,7 @@ const main = Disposable.wrap(async function* main(args) {
     try {
       const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
       try {
-        await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+        await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
       } catch (error) {
         // consider the clean successful if the VM dir is missing
         if (error.code !== 'ENOENT') {

package/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.27.0",
+  "version": "0.27.3",
   "engines": {
     "node": ">=14.6"
   },
@@ -22,7 +22,7 @@
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^1.1.0",
+    "@xen-orchestra/fs": "^2.0.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
@@ -38,7 +38,7 @@
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
     "uuid": "^8.3.2",
-    "vhd-lib": "^3.3.1",
+    "vhd-lib": "^3.3.4",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -46,7 +46,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^1.4.0"
+    "@xen-orchestra/xapi": "^1.4.1"
   },
   "license": "AGPL-3.0-or-later",
   "author": {