@xen-orchestra/backups 0.73.2 → 0.73.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/RemoteAdapter.mjs CHANGED
@@ -1,34 +1,23 @@
1
1
  import { asyncEach } from '@vates/async-each'
2
2
  import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
3
- import { compose } from '@vates/compose'
4
3
  import { createLogger } from '@xen-orchestra/log'
5
4
  import { VhdDirectory, VhdSynthetic } from 'vhd-lib'
6
5
  import { decorateMethodsWith } from '@vates/decorate-with'
7
- import { deduped } from '@vates/disposable/deduped.js'
8
6
  import { dirname, join, resolve } from 'node:path'
9
- import { execFile } from 'child_process'
10
- import { mount } from '@vates/fuse-vhd'
11
- import { readdir, lstat } from 'node:fs/promises'
12
7
  import { synchronized } from 'decorator-synchronized'
13
- import { ZipFile } from 'yazl'
14
8
  import Disposable from 'promise-toolbox/Disposable'
15
9
  import fromCallback from 'promise-toolbox/fromCallback'
16
- import fromEvent from 'promise-toolbox/fromEvent'
17
10
  import groupBy from 'lodash/groupBy.js'
18
- import pDefer from 'promise-toolbox/defer'
19
11
  import pickBy from 'lodash/pickBy.js'
20
12
  import reduce from 'lodash/reduce.js'
21
- import * as tar from 'tar'
22
13
  import zlib from 'zlib'
23
14
 
24
15
  import { BACKUP_DIR } from './_getVmBackupDir.mjs'
25
16
  import { VmBackupDirectory } from '@xen-orchestra/backup-archive'
17
+ import { fileRestoreDecorators, fileRestoreMethods } from './_fileRestore.mjs'
26
18
  import { formatFilenameDate } from './_filenameDate.mjs'
27
- import { getTmpDir } from './_getTmpDir.mjs'
28
19
  import { isMetadataFile } from './_backupType.mjs'
29
20
  import { isValidXva } from './_isValidXva.mjs'
30
- import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from './_listPartitions.mjs'
31
- import { lvs, pvs } from './_lvm.mjs'
32
21
  import { watchStreamSize } from './_watchStreamSize.mjs'
33
22
 
34
23
  import { RemoteVhdDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
@@ -48,25 +37,6 @@ export const compareTimestamp = (a, b) => a.timestamp - b.timestamp
48
37
  const noop = Function.prototype
49
38
 
50
39
  const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)
51
- const makeRelative = path => resolve('/', path).slice(1)
52
- const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
53
-
54
- async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
55
- for (const relativePath of relativePaths) {
56
- const realPath = join(realBasePath, relativePath)
57
- const virtualPath = join(virtualBasePath, relativePath)
58
-
59
- const stats = await lstat(realPath)
60
- const { mode, mtime } = stats
61
- const opts = { mode, mtime }
62
- if (stats.isDirectory()) {
63
- zip.addEmptyDirectory(virtualPath, opts)
64
- await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
65
- } else if (stats.isFile()) {
66
- zip.addFile(realPath, virtualPath, opts)
67
- }
68
- }
69
- }
70
40
 
71
41
  const createSafeReaddir = (handler, methodName) => (path, options) =>
72
42
  handler.list(path, options).catch(error => {
@@ -76,11 +46,6 @@ const createSafeReaddir = (handler, methodName) => (path, options) =>
76
46
  return []
77
47
  })
78
48
 
79
- const debounceResourceFactory = factory =>
80
- function () {
81
- return this._debounceResource(factory.apply(this, arguments))
82
- }
83
-
84
49
  export class RemoteAdapter {
85
50
  constructor(
86
51
  handler,
@@ -98,112 +63,6 @@ export class RemoteAdapter {
98
63
  return this._handler
99
64
  }
100
65
 
101
- async _findPartition(devicePath, partitionId) {
102
- const partitions = await listPartitions(devicePath)
103
- const partition = partitions.find(_ => _.id === partitionId)
104
- if (partition === undefined) {
105
- throw new Error(`partition ${partitionId} not found`)
106
- }
107
- return partition
108
- }
109
-
110
- async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
111
- yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
112
-
113
- debug('activate LVM volume group', { vgName })
114
- await fromCallback(execFile, 'vgchange', ['-ay', vgName])
115
- try {
116
- debug('get LVM volume group name and path', { vgName })
117
- yield lvs(['lv_name', 'lv_path'], vgName)
118
- } finally {
119
- debug('deactivate LVM volume group', { vgName })
120
- await fromCallback(execFile, 'vgchange', ['-an', vgName])
121
- }
122
- }
123
-
124
- async *_getLvmPhysicalVolume(devicePath, partition) {
125
- const args = []
126
- if (partition !== undefined) {
127
- args.push('-o', partition.start * 512, '--sizelimit', partition.size)
128
- }
129
- args.push('--show', '-f', devicePath)
130
-
131
- debug('attach loop device', { devicePath, partition })
132
- const path = (await fromCallback(execFile, 'losetup', args)).trim()
133
- try {
134
- debug('list LVM physical volume', { path })
135
- await fromCallback(execFile, 'pvscan', ['--cache', path])
136
-
137
- yield path
138
- } finally {
139
- try {
140
- const vgNames = await pvs('vg_name', path)
141
-
142
- debug('deactivate LVM volume groups', { vgNames })
143
- await fromCallback(execFile, 'vgchange', ['-an', ...vgNames])
144
- } finally {
145
- debug('detach loop device', { path })
146
- await fromCallback(execFile, 'losetup', ['-d', path])
147
- }
148
- }
149
- }
150
-
151
- async *_getPartition(devicePath, partition) {
152
- // the norecovery option is necessary because if the partition is dirty,
153
- // mount will try to fix it which is impossible if because the device is read-only
154
- const options = ['loop', 'ro', 'norecovery']
155
-
156
- if (partition !== undefined) {
157
- const { size, start } = partition
158
- options.push(`sizelimit=${size}`)
159
- if (start !== undefined) {
160
- options.push(`offset=${start * 512}`)
161
- }
162
- }
163
-
164
- const path = yield getTmpDir()
165
- const mount = options => {
166
- debug('mount device', { devicePath, mountPath: path })
167
- return fromCallback(execFile, 'mount', [
168
- `--options=${options.join(',')}`,
169
- `--source=${devicePath}`,
170
- `--target=${path}`,
171
- ])
172
- }
173
-
174
- // `norecovery` option is used for ext3/ext4/xfs, if it fails it might be
175
- // another fs, try without
176
- try {
177
- await mount([...options, 'norecovery'])
178
- } catch (error) {
179
- await mount(options)
180
- }
181
- try {
182
- yield path
183
- } finally {
184
- debug('umount device', { devicePath, mountPath: path })
185
- await fromCallback(execFile, 'umount', ['--lazy', path])
186
- }
187
- }
188
-
189
- _listLvmLogicalVolumes(devicePath, partition, results = []) {
190
- return Disposable.use(this._getLvmPhysicalVolume(devicePath, partition), async path => {
191
- const lvs = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], path)
192
- const partitionId = partition !== undefined ? partition.id : ''
193
- lvs.forEach((lv, i) => {
194
- const name = lv.lv_name
195
- if (name !== '') {
196
- results.push({
197
- id: `${partitionId}/${lv.vg_name}/${name}`,
198
- name,
199
- size: lv.lv_size,
200
- })
201
- }
202
- })
203
- return results
204
- })
205
- }
206
-
207
66
  // check if we will be allowed to merge a vhd created in this adapter
208
67
  // with the vhd at path `path`
209
68
  async isMergeableParent(packedParentUid, path) {
@@ -221,34 +80,6 @@ export class RemoteAdapter {
221
80
  })
222
81
  }
223
82
 
224
- fetchPartitionFiles(diskId, partitionId, paths, format) {
225
- const { promise, reject, resolve } = pDefer()
226
- Disposable.use(
227
- async function* () {
228
- const path = yield this.getPartition(diskId, partitionId)
229
- let outputStream
230
-
231
- if (format === 'tgz') {
232
- outputStream = tar.c({ cwd: path, gzip: true }, paths.map(makeRelative))
233
- } else if (format === 'zip') {
234
- const zip = new ZipFile()
235
- await addZipEntries(zip, path, '', paths.map(makeRelative))
236
- zip.end()
237
- ;({ outputStream } = zip)
238
- } else {
239
- throw new Error('unsupported format ' + format)
240
- }
241
-
242
- resolve(outputStream)
243
- await fromEvent(outputStream, 'end')
244
- }.bind(this)
245
- ).catch(error => {
246
- warn(error)
247
- reject(error)
248
- })
249
- return promise
250
- }
251
-
252
83
  async #removeVmBackupsFromCache(backups) {
253
84
  await asyncEach(
254
85
  Object.entries(
@@ -386,76 +217,6 @@ export class RemoteAdapter {
386
217
  return this.useVhdDirectory()
387
218
  }
388
219
 
389
- async *#getDiskLegacy(diskId) {
390
- const RE_VHDI = /^vhdi(\d+)$/
391
- const handler = this._handler
392
-
393
- const diskPath = handler.getFilePath('/' + diskId)
394
- const mountDir = yield getTmpDir()
395
-
396
- debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir })
397
- await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
398
- try {
399
- let max = 0
400
- let maxEntry
401
- const entries = await readdir(mountDir)
402
- entries.forEach(entry => {
403
- const matches = RE_VHDI.exec(entry)
404
- if (matches !== null) {
405
- const value = +matches[1]
406
- if (value > max) {
407
- max = value
408
- maxEntry = entry
409
- }
410
- }
411
- })
412
- if (max === 0) {
413
- throw new Error('no disks found')
414
- }
415
-
416
- yield `${mountDir}/${maxEntry}`
417
- } finally {
418
- debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir })
419
- await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
420
- }
421
- }
422
-
423
- async *getDisk(diskId) {
424
- if (this._useGetDiskLegacy) {
425
- yield* this.#getDiskLegacy(diskId)
426
- return
427
- }
428
- const handler = this._handler
429
- // this is a disposable
430
- const mountDir = yield getTmpDir()
431
- // this is also a disposable
432
- yield mount(handler, diskId, mountDir)
433
- // this will yield disk path to caller
434
- yield `${mountDir}/vhd0`
435
- }
436
-
437
- // partitionId values:
438
- //
439
- // - undefined: raw disk
440
- // - `<partitionId>`: partitioned disk
441
- // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
442
- // - `/<vgName>/lvName>`: LVM on a raw disk
443
- async *getPartition(diskId, partitionId) {
444
- const devicePath = yield this.getDisk(diskId)
445
- if (partitionId === undefined) {
446
- return yield this._getPartition(devicePath)
447
- }
448
-
449
- const isLvmPartition = partitionId.includes('/')
450
- if (isLvmPartition) {
451
- const [pvId, vgName, lvName] = partitionId.split('/')
452
- const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName)
453
- return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path)
454
- }
455
-
456
- return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
457
- }
458
-
459
220
  // if we use alias on this remote, we have to name the file alias.vhd
460
221
  getVhdFileName(baseName) {
461
222
  if (this.#useAlias()) {
@@ -496,56 +257,6 @@ export class RemoteAdapter {
496
257
  return backups
497
258
  }
498
259
 
499
- listPartitionFiles(diskId, partitionId, path) {
500
- return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
501
- path = resolveSubpath(rootPath, path)
502
- const entriesMap = {}
503
- await asyncEach(
504
- await readdir(path),
505
- async name => {
506
- try {
507
- const stats = await lstat(`${path}/${name}`)
508
- if (stats.isDirectory()) {
509
- entriesMap[name + '/'] = {}
510
- } else if (stats.isFile()) {
511
- entriesMap[name] = {}
512
- }
513
- } catch (error) {
514
- if (error == null || error.code !== 'ENOENT') {
515
- throw error
516
- }
517
- }
518
- },
519
- { concurrency: 1 }
520
- )
521
-
522
- return entriesMap
523
- })
524
- }
525
-
526
- listPartitions(diskId) {
527
- return Disposable.use(this.getDisk(diskId), async devicePath => {
528
- const partitions = await listPartitions(devicePath)
529
-
530
- if (partitions.length === 0) {
531
- try {
532
- // handle potential raw LVM physical volume
533
- return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
534
- } catch (error) {
535
- return []
536
- }
537
- }
538
-
539
- const results = []
540
- await asyncMapSettled(partitions, partition =>
541
- partition.type === LVM_PARTITION_TYPE_MBR || partition.type === LVM_PARTITION_TYPE_GPT
542
- ? this._listLvmLogicalVolumes(devicePath, partition, results)
543
- : results.push(partition)
544
- )
545
- return results
546
- })
547
- }
548
-
549
260
  async listPoolMetadataBackups() {
550
261
  const handler = this._handler
551
262
  const safeReaddir = createSafeReaddir(handler, 'listPoolMetadataBackups')
@@ -932,26 +643,8 @@ Object.assign(RemoteAdapter.prototype, {
932
643
  isValidXva,
933
644
  })
934
645
 
935
- decorateMethodsWith(RemoteAdapter, {
936
- _getLvmLogicalVolumes: compose([
937
- Disposable.factory,
938
- [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
939
- debounceResourceFactory,
940
- ]),
941
-
942
- _getLvmPhysicalVolume: compose([
943
- Disposable.factory,
944
- [deduped, (devicePath, partition) => [devicePath, partition?.id]],
945
- debounceResourceFactory,
946
- ]),
947
-
948
- _getPartition: compose([
949
- Disposable.factory,
950
- [deduped, (devicePath, partition) => [devicePath, partition?.id]],
951
- debounceResourceFactory,
952
- ]),
646
+ // File-level-restore methods live in ./_fileRestore.mjs; mix them onto the prototype
647
+ // before decorating so decorateMethodsWith can wrap them.
648
+ Object.assign(RemoteAdapter.prototype, fileRestoreMethods)
953
649
 
954
- getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
955
-
956
- getPartition: Disposable.factory,
957
- })
650
+ decorateMethodsWith(RemoteAdapter, fileRestoreDecorators)
@@ -0,0 +1,476 @@
1
+ // File-level-restore (FLR) methods for RemoteAdapter.
2
+ //
3
+ // These were extracted from RemoteAdapter.mjs to keep that file focused on backup
4
+ // CRUD/cache/metadata. They are mixed back onto `RemoteAdapter.prototype` via
5
+ // `Object.assign` and decorated via `decorateMethodsWith` in RemoteAdapter.mjs, so `this`
6
+ // is the RemoteAdapter instance at call time (`this._handler`, `this._debounceResource`,
7
+ // `this._useGetDiskLegacy` keep working unchanged). The logger keeps the original
8
+ // `xo:backups:RemoteAdapter` namespace so existing debug filters are unaffected.
9
+ import { asyncEach } from '@vates/async-each'
10
+ import { asyncMapSettled } from '@xen-orchestra/async-map'
11
+ import { compose } from '@vates/compose'
12
+ import { createLogger } from '@xen-orchestra/log'
13
+ import { deduped } from '@vates/disposable/deduped.js'
14
+ import { randomBytes } from 'node:crypto'
15
+ import { join, resolve } from 'node:path'
16
+ import { execFile } from 'child_process'
17
+ import { finished } from 'node:stream/promises'
18
+ import { lstat, open, readdir, unlink } from 'node:fs/promises'
19
+ import { tmpdir } from 'node:os'
20
+ import { mount } from '@vates/fuse-vhd'
21
+ import { ZipFile } from 'yazl'
22
+ import Disposable from 'promise-toolbox/Disposable'
23
+ import fromCallback from 'promise-toolbox/fromCallback'
24
+ import pDefer from 'promise-toolbox/defer'
25
+ import * as tar from 'tar'
26
+
27
+ import { getTmpDir } from './_getTmpDir.mjs'
28
+ import {
29
+ listPartitions,
30
+ LINUX_DATA_PARTITION_TYPE_GPT,
31
+ LINUX_DATA_PARTITION_TYPE_MBR,
32
+ LVM_PARTITION_TYPE_GPT,
33
+ LVM_PARTITION_TYPE_MBR,
34
+ } from './_listPartitions.mjs'
35
+ import { lvs, pvs } from './_lvm.mjs'
36
+
37
+ const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
38
+
39
+ const noop = Function.prototype
40
+
41
+ // Restrict LVM scanning to a single device. Backup PVs are clones: the very same PVID
42
+ // appears at once on the raw loop device, the dm-snapshot overlaid on it, and every
43
+ // other restored copy of the same VM. An unscoped pvs/pvscan/vgimportclone/vgchange then
44
+ // aborts with "duplicate PV ... for PVID ..." and the VG can neither be renamed nor
45
+ // activated. Accepting only the device in hand removes every duplicate from LVM's view.
46
+ export const lvmOnlyDevice = devicePath => `devices { global_filter=[ "a|^${devicePath}$|", "r|.*|" ] }`
47
+
48
+ // Partition-type predicates (exported for unit tests).
49
+ export const isLvmPartitionType = type => type === LVM_PARTITION_TYPE_MBR || type === LVM_PARTITION_TYPE_GPT
50
+
51
+ // Some installers (e.g. Ubuntu subiquity) place an LVM PV on a generic Linux-data partition
52
+ // instead of the LVM type, so those are the only non-LVM types worth the (expensive) probe.
53
+ export const isProbeableForLvm = type =>
54
+ type === LINUX_DATA_PARTITION_TYPE_MBR || type === LINUX_DATA_PARTITION_TYPE_GPT
55
+
56
+ // Build the partition entries for a PV's logical volumes (exported for unit tests).
57
+ // Shows "ubuntu-vg/ubuntu-lv" for readability; skips unnamed LVs.
58
+ export const toLvPartitions = (partitionId, originalVgName, lvItems) =>
59
+ lvItems
60
+ .filter(lv => lv.lv_name !== '')
61
+ .map(lv => ({
62
+ id: `${partitionId}/${lv.vg_name}/${lv.lv_name}`,
63
+ name: originalVgName !== undefined ? `${originalVgName}/${lv.lv_name}` : lv.lv_name,
64
+ size: lv.lv_size,
65
+ }))
66
+
67
+ const makeRelative = path => resolve('/', path).slice(1)
68
+ const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
69
+
70
+ async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
71
+ for (const relativePath of relativePaths) {
72
+ const realPath = join(realBasePath, relativePath)
73
+ const virtualPath = join(virtualBasePath, relativePath)
74
+
75
+ const stats = await lstat(realPath)
76
+ const { mode, mtime } = stats
77
+ const opts = { mode, mtime }
78
+ if (stats.isDirectory()) {
79
+ zip.addEmptyDirectory(virtualPath, opts)
80
+ await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
81
+ } else if (stats.isFile()) {
82
+ zip.addFile(realPath, virtualPath, opts)
83
+ }
84
+ }
85
+ }
86
+
87
+ const debounceResourceFactory = factory =>
88
+ function () {
89
+ return this._debounceResource(factory.apply(this, arguments))
90
+ }
91
+
92
+ // legacy disk mount via vhdimount; extracted from a private method so it can live in this
93
+ // module (mixin methods cannot reference class-private members).
94
+ async function* getDiskLegacy(handler, diskId) {
95
+ const RE_VHDI = /^vhdi(\d+)$/
96
+
97
+ const diskPath = handler.getFilePath('/' + diskId)
98
+ const mountDir = yield getTmpDir()
99
+
100
+ debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir })
101
+ await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
102
+ try {
103
+ let max = 0
104
+ let maxEntry
105
+ const entries = await readdir(mountDir)
106
+ entries.forEach(entry => {
107
+ const matches = RE_VHDI.exec(entry)
108
+ if (matches !== null) {
109
+ const value = +matches[1]
110
+ if (value > max) {
111
+ max = value
112
+ maxEntry = entry
113
+ }
114
+ }
115
+ })
116
+ if (max === 0) {
117
+ throw new Error('no disks found')
118
+ }
119
+
120
+ yield `${mountDir}/${maxEntry}`
121
+ } finally {
122
+ debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir })
123
+ await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
124
+ }
125
+ }
126
+
127
+ // Mixed onto RemoteAdapter.prototype — `this` is the RemoteAdapter instance.
128
+ export const fileRestoreMethods = {
129
+ async _findPartition(devicePath, partitionId) {
130
+ const partitions = await listPartitions(devicePath)
131
+ const partition = partitions.find(_ => _.id === partitionId)
132
+ if (partition === undefined) {
133
+ throw new Error(`partition ${partitionId} not found`)
134
+ }
135
+ return partition
136
+ },
137
+
138
+ async *_getLvmLogicalVolumes(devicePath, pvId, requestedVgName) {
139
+ const { vgName, lvmConfig } = yield this._getLvmPhysicalVolume(
140
+ devicePath,
141
+ pvId && (await this._findPartition(devicePath, pvId))
142
+ )
143
+
144
+ // vgName is the unique name vgimportclone just assigned to this PV; prefer it over the
145
+ // (possibly stale) name embedded in the partitionId. lvmConfig scopes every command to
146
+ // this device so a colliding/duplicate VG on the host or another copy can't shadow it.
147
+ const effectiveVgName = vgName ?? requestedVgName
148
+
149
+ debug('activate LVM volume group', { effectiveVgName, requestedVgName })
150
+ await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-ay', effectiveVgName])
151
+ try {
152
+ debug('get LVM logical volumes', { effectiveVgName })
153
+ yield lvs(['lv_name', 'lv_path'], '--config', lvmConfig, effectiveVgName)
154
+ } finally {
155
+ debug('deactivate LVM volume group', { effectiveVgName })
156
+ await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', effectiveVgName])
157
+ }
158
+ },
159
+
160
+ async *_getLvmPhysicalVolume(devicePath, partition) {
161
+ const loopArgs = []
162
+ if (partition !== undefined) {
163
+ loopArgs.push('-o', partition.start * 512, '--sizelimit', partition.size)
164
+ }
165
+ loopArgs.push('--show', '-f', devicePath)
166
+
167
+ debug('attach loop device', { devicePath, partition })
168
+ const loopDevice = (await fromCallback(execFile, 'losetup', loopArgs)).trim()
169
+
170
+ let cowPath, cowLoop, mapperName, vgName, lvmConfig
171
+ try {
172
+ // Cheap pre-check, scoped to this loop so a duplicate PVID (another copy of the same
173
+ // VM restored concurrently) can't make it fail: is there an LVM PV here at all?
174
+ // Non-PV partitions (/boot, EFI, raw disks) skip the whole dm-snapshot machinery.
175
+ const [originalVgName] = (
176
+ await pvs('vg_name', '--config', lvmOnlyDevice(loopDevice), loopDevice).catch(() => [])
177
+ ).filter(Boolean)
178
+ if (originalVgName === undefined) {
179
+ const where = partition !== undefined ? ` partition ${partition.id}` : ''
180
+ throw new Error(`no LVM physical volume on ${devicePath}${where}`)
181
+ }
182
+
183
+ // The backup is read-only, so overlay a writable dm-snapshot to let vgimportclone
184
+ // rewrite metadata, and rename the VG to a unique name so concurrent clones (identical
185
+ // PVID and VG name) don't collide on device-mapper node names. Every LVM command is
186
+ // scoped to the snapshot device (lvmConfig) to keep duplicate PVIDs out of LVM's view.
187
+ mapperName = `xo-pv-${randomBytes(4).toString('hex')}`
188
+ const mapperPath = `/dev/mapper/${mapperName}`
189
+ lvmConfig = lvmOnlyDevice(mapperPath)
190
+
191
+ // ~4 MB sparse COW is enough to hold the LVM metadata rewrites
192
+ cowPath = join(tmpdir(), `${mapperName}.cow`)
193
+ const fh = await open(cowPath, 'w')
194
+ await fh.truncate(4 * 1024 * 1024)
195
+ await fh.close()
196
+ cowLoop = (await fromCallback(execFile, 'losetup', ['--show', '-f', cowPath])).trim()
197
+
198
+ const sectors = (await fromCallback(execFile, 'blockdev', ['--getsz', loopDevice])).trim()
199
+ await fromCallback(execFile, 'dmsetup', [
200
+ 'create',
201
+ mapperName,
202
+ '--table',
203
+ `0 ${sectors} snapshot ${loopDevice} ${cowLoop} P 8`,
204
+ ])
205
+
206
+ // Unique random VG name: list and mount each query/yield the name from the device, so
207
+ // it need not be deterministic — only collision-free across concurrent clones.
208
+ vgName = `xo${randomBytes(8).toString('hex')}`
209
+ debug('import LVM volume group with unique name via dm-snapshot', { vgName, originalVgName })
210
+ await fromCallback(execFile, 'vgimportclone', ['--config', lvmConfig, '--basevgname', vgName, mapperPath])
211
+
212
+ yield { path: mapperPath, originalVgName, vgName, lvmConfig }
213
+ } finally {
214
+ if (mapperName !== undefined) {
215
+ // best-effort deactivate (a no-op if it was never activated) before removing the snapshot
216
+ await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', vgName]).catch(noop)
217
+ debug('remove dm-snapshot', { mapperName })
218
+ await fromCallback(execFile, 'dmsetup', ['remove', mapperName]).catch(err =>
219
+ warn('failed to remove dm-snapshot', { mapperName, error: err })
220
+ )
221
+ }
222
+ if (cowLoop !== undefined) {
223
+ await fromCallback(execFile, 'losetup', ['-d', cowLoop]).catch(noop)
224
+ }
225
+ if (cowPath !== undefined) {
226
+ await unlink(cowPath).catch(noop)
227
+ }
228
+ debug('detach loop device', { loopDevice })
229
+ await fromCallback(execFile, 'losetup', ['-d', loopDevice])
230
+ }
231
+ },
232
+
233
+ async *_getPartition(devicePath, partition) {
234
+ const options = ['loop', 'ro']
235
+
236
+ if (partition !== undefined) {
237
+ const { size, start } = partition
238
+ options.push(`sizelimit=${size}`)
239
+ if (start !== undefined) {
240
+ options.push(`offset=${start * 512}`)
241
+ }
242
+ }
243
+
244
+ const path = yield getTmpDir()
245
+ const mount = options => {
246
+ debug('mount device', { devicePath, mountPath: path })
247
+ return fromCallback(execFile, 'mount', [
248
+ `--options=${options.join(',')}`,
249
+ `--source=${devicePath}`,
250
+ `--target=${path}`,
251
+ ]).catch(error => {
252
+ if (error.stderr) {
253
+ error.message = `${error.message}: ${error.stderr.trim()}`
254
+ }
255
+ throw error
256
+ })
257
+ }
258
+
259
+ // norecovery prevents mount from attempting journal replay on a read-only device (ext3/ext4/xfs).
260
+ // Other filesystems don't support it, so fall back without it on failure.
261
+ try {
262
+ await mount([...options, 'norecovery'])
263
+ } catch (error) {
264
+ await mount(options)
265
+ }
266
+
267
+ try {
268
+ yield path
269
+ } finally {
270
+ debug('umount device', { devicePath, mountPath: path })
271
+ await fromCallback(execFile, 'umount', ['--lazy', path])
272
+ }
273
+ },
274
+
275
+ _listLvmLogicalVolumes(devicePath, partition, results = []) {
276
+ return Disposable.use(
277
+ this._getLvmPhysicalVolume(devicePath, partition),
278
+ async ({ path, originalVgName, lvmConfig }) => {
279
+ const lvItems = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], '--config', lvmConfig, path)
280
+ const partitionId = partition !== undefined ? partition.id : ''
281
+ results.push(...toLvPartitions(partitionId, originalVgName, lvItems))
282
+ return results
283
+ }
284
+ )
285
+ },
286
+
287
+ fetchPartitionFiles(diskId, partitionId, paths, format) {
288
+ const { promise, reject, resolve } = pDefer()
289
+ const self = this
290
+ Disposable.use(async function* () {
291
+ const path = yield self.getPartition(diskId, partitionId)
292
+ let outputStream
293
+
294
+ if (format === 'tgz') {
295
+ // process one entry at a time with { job: 1}. node-tar defaults to 4
296
+ // concurrent jobs, which on a FUSE-backed restore mount means up to 4
297
+ // simultaneous reads. Those saturate the libuv threadpool and starve
298
+ // the underlying vhd/CIFS reads NTFS-3g depends on (FUSE-on-FUSE
299
+ // threadpool deadlock). Serializing keeps a worker free for them.
300
+ outputStream = tar.c({ cwd: path, gzip: true, jobs: 1 }, paths.map(makeRelative))
301
+ resolve(outputStream)
302
+ } else if (format === 'zip') {
303
+ const zip = new ZipFile()
304
+ // Resolve with the stream before enumeration so the client can start
305
+ // receiving data immediately — addZipEntries over FUSE/S3 can take
306
+ // minutes for large trees (e.g. node_modules) and would otherwise
307
+ // appear as a freeze with no response sent.
308
+ outputStream = zip.outputStream
309
+ resolve(zip.outputStream)
310
+ await addZipEntries(zip, path, '', paths.map(makeRelative))
311
+ zip.end()
312
+ } else {
313
+ throw new Error('unsupported format ' + format)
314
+ }
315
+
316
+ await finished(outputStream).catch(noop)
317
+ }).catch(error => {
318
+ warn(error)
319
+ reject(error)
320
+ })
321
+ return promise
322
+ },
323
+
324
+ async *getDisk(diskId) {
325
+ if (this._useGetDiskLegacy) {
326
+ yield* getDiskLegacy(this._handler, diskId)
327
+ return
328
+ }
329
+ const handler = this._handler
330
+ // this is a disposable
331
+ const mountDir = yield getTmpDir()
332
+ // this is also a disposable
333
+ yield mount(handler, diskId, mountDir)
334
+ // this will yield disk path to caller
335
+ yield `${mountDir}/vhd0`
336
+ },
337
+
338
+ // partitionId values:
339
+ //
340
+ // - undefined: raw disk
341
+ // - `<partitionId>`: partitioned disk
342
+ // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
343
+ // - `/<vgName>/lvName>`: LVM on a raw disk
344
+ async *getPartition(diskId, partitionId) {
345
+ const devicePath = yield this.getDisk(diskId)
346
+ if (partitionId === undefined) {
347
+ debug(
348
+ 'no partition specified, attempting raw disk mount — call listPartitions first if disk has partitions or LVM'
349
+ )
350
+ return yield this._getPartition(devicePath)
351
+ }
352
+
353
+ const isLvmPartition = partitionId.includes('/')
354
+ if (isLvmPartition) {
355
+ const [pvId, vgName, lvName] = partitionId.split('/')
356
+ const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName)
357
+ return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path)
358
+ }
359
+
360
+ return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
361
+ },
362
+
363
+ listPartitionFiles(diskId, partitionId, path) {
364
+ return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
365
+ path = resolveSubpath(rootPath, path)
366
+ const entriesMap = {}
367
+ await asyncEach(
368
+ await readdir(path),
369
+ async name => {
370
+ try {
371
+ const stats = await lstat(`${path}/${name}`)
372
+ if (stats.isDirectory()) {
373
+ entriesMap[name + '/'] = {}
374
+ } else if (stats.isFile()) {
375
+ entriesMap[name] = {}
376
+ }
377
+ } catch (error) {
378
+ if (error == null || error.code !== 'ENOENT') {
379
+ throw error
380
+ }
381
+ }
382
+ },
383
+ { concurrency: 1 }
384
+ )
385
+
386
+ return entriesMap
387
+ })
388
+ },
389
+
390
+ listPartitions(diskId) {
391
+ return Disposable.use(this.getDisk(diskId), async devicePath => {
392
+ // partx may return empty on FUSE-backed files (vhd0); a loop device
393
+ // presents proper block-device semantics that partx reads reliably.
394
+ // losetup may itself fail if the FUSE mount isn't fully ready yet —
395
+ // fall back to the direct path so listPartitions returns [] instead of throwing.
396
+ let loopForParts, partitions
397
+ try {
398
+ loopForParts = (await fromCallback(execFile, 'losetup', ['--show', '-f', devicePath])).trim()
399
+ partitions = await listPartitions(loopForParts)
400
+ } catch (error) {
401
+ debug('partition probe via loop device failed, falling back to direct path', { error })
402
+ partitions = await listPartitions(devicePath)
403
+ } finally {
404
+ if (loopForParts !== undefined) {
405
+ await fromCallback(execFile, 'losetup', ['-d', loopForParts]).catch(noop)
406
+ }
407
+ }
408
+
409
+ if (partitions.length === 0) {
410
+ try {
411
+ // handle potential raw LVM physical volume
412
+ return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
413
+ } catch (error) {
414
+ return []
415
+ }
416
+ }
417
+
418
+ const results = []
419
+ await asyncMapSettled(partitions, async partition => {
420
+ if (isLvmPartitionType(partition.type)) {
421
+ return this._listLvmLogicalVolumes(devicePath, partition, results)
422
+ }
423
+
424
+ // Only generic Linux-data partitions can hide an LVM PV (subiquity-style); other
425
+ // types — BIOS boot, EFI, swap, … — are never PVs, so list them directly without
426
+ // the (expensive) loop + dm-snapshot + vgimportclone probe.
427
+ if (!isProbeableForLvm(partition.type)) {
428
+ results.push(partition)
429
+ return
430
+ }
431
+
432
+ const lvResults = []
433
+ try {
434
+ await this._listLvmLogicalVolumes(devicePath, partition, lvResults)
435
+ } catch (error) {
436
+ debug('LVM probe failed for Linux-data partition, treating as regular partition', {
437
+ partition,
438
+ error,
439
+ })
440
+ }
441
+ if (lvResults.length > 0) {
442
+ results.push(...lvResults)
443
+ } else {
444
+ results.push(partition)
445
+ }
446
+ })
447
+ return results
448
+ })
449
+ },
450
+ }
451
+
452
+ // Applied by RemoteAdapter.mjs via decorateMethodsWith(RemoteAdapter, fileRestoreDecorators).
453
+ // debounceResourceFactory/deduped reference `this._debounceResource` at call time.
454
+ export const fileRestoreDecorators = {
455
+ _getLvmLogicalVolumes: compose([
456
+ Disposable.factory,
457
+ [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
458
+ debounceResourceFactory,
459
+ ]),
460
+
461
+ _getLvmPhysicalVolume: compose([
462
+ Disposable.factory,
463
+ [deduped, (devicePath, partition) => [devicePath, partition?.id]],
464
+ debounceResourceFactory,
465
+ ]),
466
+
467
+ _getPartition: compose([
468
+ Disposable.factory,
469
+ [deduped, (devicePath, partition) => [devicePath, partition?.id]],
470
+ debounceResourceFactory,
471
+ ]),
472
+
473
+ getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
474
+
475
+ getPartition: Disposable.factory,
476
+ }
@@ -29,6 +29,15 @@ export const LVM_PARTITION_TYPE_MBR = 0x8e
29
29
  // GPT LVM type
30
30
  export const LVM_PARTITION_TYPE_GPT = 'e6d6d379-f507-44c2-a23c-238f2a3df928'
31
31
 
32
+ // Generic "Linux filesystem data" types. These are the only non-LVM-typed
33
+ // partitions worth probing for a hidden LVM PV, because some installers (e.g.
34
+ // Ubuntu subiquity) place a PV on a partition left with this generic type.
35
+ // Other types (BIOS boot, EFI, swap, …) are never LVM PVs.
36
+ // MBR Linux native
37
+ export const LINUX_DATA_PARTITION_TYPE_MBR = 0x83
38
+ // GPT Linux filesystem data
39
+ export const LINUX_DATA_PARTITION_TYPE_GPT = '0fc63daf-8483-4772-8e79-3d69d8477de4'
40
+
32
41
  const parsePartxLine = createParser({
33
42
  keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
34
43
  valueTransform: (value, key) => {
@@ -1,4 +1,4 @@
1
- import { asyncMap } from '@xen-orchestra/async-map'
1
+ import { asyncEach } from '@vates/async-each'
2
2
  import { Task } from '@vates/task'
3
3
  import Disposable from 'promise-toolbox/Disposable'
4
4
  import ignoreErrors from 'promise-toolbox/ignoreErrors'
@@ -82,10 +82,11 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
82
82
  // remove pools that failed (already handled)
83
83
  pools = pools.filter(_ => _ !== undefined)
84
84
 
85
- const promises = []
86
- if (pools.length !== 0 && settings.retentionPoolMetadata !== 0) {
87
- promises.push(
88
- asyncMap(pools, async pool =>
85
+ const tasks = []
86
+
87
+ if (settings.retentionPoolMetadata !== 0) {
88
+ for (const pool of pools) {
89
+ tasks.push(async () =>
89
90
  Task.run(
90
91
  {
91
92
  properties: {
@@ -107,11 +108,11 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
107
108
  }).run()
108
109
  ).catch(noop)
109
110
  )
110
- )
111
+ }
111
112
  }
112
113
 
113
114
  if (job.xoMetadata !== undefined && settings.retentionXoMetadata !== 0) {
114
- promises.push(
115
+ tasks.push(() =>
115
116
  Task.run(
116
117
  {
117
118
  properties: {
@@ -130,7 +131,16 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
130
131
  ).catch(noop)
131
132
  )
132
133
  }
133
- await Promise.all(promises)
134
+
135
+ const total = tasks.length
136
+ let transferred = 0
137
+
138
+ await asyncEach(tasks, async task => {
139
+ await task()
140
+ transferred++
141
+ Task.set('progress', Math.round(transferred / total))
142
+ })
143
+ Task.set('progress', 1)
134
144
  }
135
145
  )
136
146
  }
@@ -63,6 +63,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
63
63
  const vmsUuids = await sourceRemoteAdapter.listAllVms()
64
64
 
65
65
  Task.info('vms', { vms: vmsUuids })
66
+ const nbVms = vmsUuids.length
67
+ let nbVmsDone = 0
66
68
 
67
69
  remoteAdapters = getAdaptersByRemote(remoteAdapters)
68
70
  const allSettings = this._job.settings
@@ -123,11 +125,14 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
123
125
  )
124
126
  .then(result => {
125
127
  if (taskError === undefined) {
128
+ nbVmsDone++
126
129
  return task.success(result)
127
130
  }
128
131
  if (isLastRun) {
132
+ nbVmsDone++
129
133
  return task.failure(taskError)
130
134
  }
135
+ Task.set('progress', Math.round((nbVmsDone * 100) / nbVms))
131
136
  // don't end the task
132
137
  task.warning(`Retry the VM mirror backup due to an error`, {
133
138
  attempt: nTriesByVmId[vmUuid],
@@ -148,6 +153,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
148
153
 
149
154
  await asyncMapSettled(vmIds, _handleVm)
150
155
  }
156
+ Task.set('progress', 100)
151
157
  }
152
158
  )
153
159
  }
@@ -101,10 +101,14 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
101
101
 
102
102
  const handleVm = vmUuid => {
103
103
  const getVmTask = () => {
104
- if (taskByVmId[vmUuid] === undefined) {
104
+ const started = taskByVmId[vmUuid] !== undefined
105
+ if (!started) {
105
106
  taskByVmId[vmUuid] = new Task(taskStart)
106
107
  }
107
- return taskByVmId[vmUuid]
108
+ return {
109
+ task: taskByVmId[vmUuid],
110
+ started,
111
+ }
108
112
  }
109
113
  const vmBackupFailed = async (error, task) => {
110
114
  if (isLastRun) {
@@ -135,7 +139,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
135
139
  taskStart.properties.name_label = vm.name_label
136
140
  }
137
141
 
138
- const task = getVmTask()
142
+ const { task } = getVmTask()
139
143
  // error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
140
144
  let taskError
141
145
  return task
@@ -174,12 +178,19 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
174
178
  task.success(result)
175
179
  } else {
176
180
  // ending the task with error or not ending the task
177
- vmBackupFailed(taskError, task)
181
+ return vmBackupFailed(taskError, task)
178
182
  }
179
183
  })
180
184
  .catch(noop) // errors are handled by logs
181
185
  }),
182
- error => vmBackupFailed(error, getVmTask())
186
+ error => {
187
+ const { task: vmTask, started } = getVmTask()
188
+ if (!started) {
189
+ // the task is not started (except if it's a retry), and an unstarted task can't be failed
190
+ vmTask.start()
191
+ }
192
+ return vmBackupFailed(error, vmTask)
193
+ }
183
194
  )
184
195
  }
185
196
  const { concurrency } = settings
@@ -3,6 +3,7 @@ import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
3
3
  import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
4
4
  import { watchStreamSize } from '../../_watchStreamSize.mjs'
5
5
  import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
6
+ import { Task } from '@vates/task'
6
7
 
7
8
  export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
8
9
  _getRemoteWriters() {
@@ -15,7 +16,8 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
15
16
 
16
17
  async _run() {
17
18
  const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
18
-
19
+ const nbTransferrableVms = transferList.length
20
+ let nbTransferredVms = 0
19
21
  for (const metadata of transferList) {
20
22
  const stream = this._throttleStream(await this._sourceRemoteAdapter.readFullVmBackup(metadata))
21
23
  const sizeContainer = watchStreamSize(stream)
@@ -27,7 +29,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
27
29
  stream: forkStreamUnpipe(stream),
28
30
  // stream will be forked and transformed, it's not safe to attach additional properties to it
29
31
  streamLength: stream.length,
30
- maxStreamLength: stream.maxStreamLength, // on encrypted source
32
+ maxStreamLength: stream.maxStreamLength, // for encrypted destination/source without length
31
33
  timestamp: metadata.timestamp,
32
34
  vm: metadata.vm,
33
35
  vmSnapshot: metadata.vmSnapshot,
@@ -37,7 +39,10 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
37
39
  )
38
40
  // for healthcheck
39
41
  this._tags = metadata.vm.tags
42
+ nbTransferredVms++
43
+ Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
40
44
  }
45
+ Task.set('progress', 100)
41
46
  this._hasTransferredData = transferList.length > 0
42
47
  }
43
48
  }
@@ -12,6 +12,8 @@ import { openVhd } from 'vhd-lib'
12
12
  import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
13
13
  import { SynchronizedDisk, ThrottledDisk } from '@xen-orchestra/disk-transform'
14
14
  import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
15
+ import { Task } from '@vates/task'
16
+ import { TaskProgressHandler } from './_TaskProgressHandler.mjs'
15
17
 
16
18
  const { warn } = createLogger('xo:backups:Incrementalremote')
17
19
  class IncrementalRemoteVmBackupRunner extends AbstractRemote {
@@ -60,7 +62,8 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
60
62
  }
61
63
  async _run() {
62
64
  const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
63
-
65
+ const nbTransferrableVms = transferList.length
66
+ let nbTransferredVms = 0
64
67
  for (const metadata of transferList) {
65
68
  assert.strictEqual(metadata.mode, 'delta')
66
69
  const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
@@ -74,6 +77,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
74
77
  for (const key in incrementalExport.disks) {
75
78
  let disk = incrementalExport.disks[key]
76
79
  isVhdDifferencing[key] = disk.isDifferencing()
80
+ disk.addProgressHandler(new TaskProgressHandler())
77
81
  disk = new ThrottledDisk(disk, this._throttleGenerator)
78
82
  incrementalExport.disks[key] = new SynchronizedDisk(disk)
79
83
  }
@@ -116,7 +120,10 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
116
120
  await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
117
121
  // for healthcheck
118
122
  this._tags = metadata.vm.tags
123
+ nbTransferredVms++
124
+ Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
119
125
  }
126
+ Task.set('progress', 100)
120
127
  this._hasTransferredData = transferList.length > 0
121
128
  }
122
129
  }
@@ -18,6 +18,7 @@ import {
18
18
  import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
19
19
  import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
20
20
  import { AggregatedIncrementalXapiWriter } from '../_writers/AggregatedIncrementalXapiWriter.mjs'
21
+ import { TaskProgressHandler } from './_TaskProgressHandler.mjs'
21
22
 
22
23
  const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
23
24
 
@@ -52,6 +53,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
52
53
  let useNbd = false
53
54
  for (const key in deltaExport.disks) {
54
55
  let disk = deltaExport.disks[key]
56
+ disk.addProgressHandler(new TaskProgressHandler())
55
57
  isVhdDifferencing[key] = disk.isDifferencing()
56
58
  if (!isFull && !isVhdDifferencing[key] && key !== exportedVm.$suspend_VDI?.$ref) {
57
59
  Task.warning('Backup fell back to a full')
@@ -0,0 +1,29 @@
1
+ import { Task } from '@vates/task'
2
+
3
+ const MAX_DURATION_BETWEEN_PROGRESS_EMIT = 5e3
4
+ const MIN_THRESHOLD_PERCENT_BETWEEN_PROGRESS_EMIT = 0.01
5
+
6
+ export class TaskProgressHandler {
7
+ #lastProgressDate
8
+ #lastProgressValue
9
+ constructor() {
10
+ Task.set('progress', 0)
11
+ }
12
+ async setProgress(progress) {
13
+ if (progress < 0 || progress > 1) {
14
+ return
15
+ }
16
+ if (
17
+ this.#lastProgressDate !== undefined &&
18
+ this.#lastProgressValue !== undefined &&
19
+ Date.now() - this.#lastProgressDate < MAX_DURATION_BETWEEN_PROGRESS_EMIT &&
20
+ progress - this.#lastProgressValue < MIN_THRESHOLD_PERCENT_BETWEEN_PROGRESS_EMIT
21
+ ) {
22
+ return
23
+ }
24
+ this.#lastProgressDate = Date.now()
25
+ this.#lastProgressValue = progress
26
+ Task.set('progress', Math.round(progress * 100))
27
+ }
28
+ done() {}
29
+ }
@@ -1,6 +1,16 @@
1
+ import { TaskProgressHandler } from './_runners/_vmRunners/_TaskProgressHandler.mjs'
1
2
  export function watchStreamSize(stream, container = { size: 0 }) {
3
+ const progressHandler = container.progressHandler ?? new TaskProgressHandler()
2
4
  stream.on('data', data => {
3
5
  container.size += data.length
6
+ if (stream.length) {
7
+ // empty for xva exported by xapi
8
+ progressHandler.setProgress(container.size / stream.length)
9
+ }
10
+ })
11
+
12
+ stream.on('finish', () => {
13
+ progressHandler.setProgress(1)
4
14
  })
5
15
  stream.pause()
6
16
  return container
package/package.json CHANGED
@@ -8,7 +8,7 @@
8
8
  "type": "git",
9
9
  "url": "https://github.com/vatesfr/xen-orchestra.git"
10
10
  },
11
- "version": "0.73.2",
11
+ "version": "0.73.4",
12
12
  "engines": {
13
13
  "node": ">=14.18"
14
14
  },
@@ -25,19 +25,19 @@
25
25
  "@vates/compose": "^2.1.0",
26
26
  "@vates/decorate-with": "^2.1.0",
27
27
  "@vates/disposable": "^0.1.6",
28
- "@vates/fuse-vhd": "^2.1.2",
29
- "@vates/generator-toolbox": "^1.1.2",
28
+ "@vates/fuse-vhd": "^2.1.3",
29
+ "@vates/generator-toolbox": "^1.1.3",
30
30
  "@vates/nbd-client": "^3.4.0",
31
31
  "@vates/parse-duration": "^0.1.1",
32
32
  "@vates/task": "^0.7.0",
33
- "@vates/types": "^1.26.0",
33
+ "@vates/types": "^1.27.0",
34
34
  "@xen-orchestra/async-map": "^0.1.3",
35
- "@xen-orchestra/disk-transform": "^1.3.0",
36
- "@xen-orchestra/fs": "^4.9.0",
35
+ "@xen-orchestra/disk-transform": "^1.3.1",
36
+ "@xen-orchestra/fs": "^4.9.1",
37
37
  "@xen-orchestra/log": "^0.7.2",
38
- "@xen-orchestra/qcow2": "^1.3.0",
38
+ "@xen-orchestra/qcow2": "^1.3.1",
39
39
  "@xen-orchestra/template": "^0.1.1",
40
- "@xen-orchestra/backup-archive": "^1.0.1",
40
+ "@xen-orchestra/backup-archive": "^1.0.2",
41
41
  "app-conf": "^3.0.0",
42
42
  "compare-versions": "^6.0.0",
43
43
  "d3-time-format": "^4.1.0",
@@ -56,7 +56,7 @@
56
56
  "uuid": "^9.0.0",
57
57
  "value-matcher": "^0.2.0",
58
58
  "vhd-lib": "^4.16.0",
59
- "xen-api": "^4.7.7",
59
+ "xen-api": "^4.7.8",
60
60
  "yazl": "^2.5.1"
61
61
  },
62
62
  "devDependencies": {
@@ -67,7 +67,7 @@
67
67
  "typescript": "^5.9.3"
68
68
  },
69
69
  "peerDependencies": {
70
- "@xen-orchestra/xapi": "^8.8.0"
70
+ "@xen-orchestra/xapi": "^8.9.0"
71
71
  },
72
72
  "license": "AGPL-3.0-or-later",
73
73
  "author": {
@@ -4,22 +4,10 @@ import { fork } from 'child_process'
4
4
  const { warn } = createLogger('xo:backups:backupWorker')
5
5
 
6
6
  const PATH = new URL('_backupWorker.mjs', import.meta.url).pathname
7
- const DEFAULT_INSPECTOR_PORT = 9229
8
7
 
9
8
  export function runBackupWorker(params, onLog) {
10
9
  return new Promise((resolve, reject) => {
11
- // run Node inspector on port+1 if --inspect or --inspect-brk
12
- const inspectArg = process.execArgv.find(arg => arg.startsWith('--inspect') || arg.startsWith('--inspect-brk'))
13
- const execArgv = inspectArg
14
- ? [
15
- inspectArg.replace(/^(--inspect(-brk)?)(=([^:]+:)?(\d+))?$/, (_, prefix, brk, _fullMatch, host, port) => {
16
- const basePort = port ? parseInt(port) : DEFAULT_INSPECTOR_PORT
17
- return `${prefix}=${host || ''}${basePort + 1}`
18
- }),
19
- ]
20
- : []
21
-
22
- const worker = fork(PATH, [], { execArgv })
10
+ const worker = fork(PATH, [])
23
11
 
24
12
  worker.on('exit', (code, signal) => reject(new Error(`worker exited with code ${code} and signal ${signal}`)))
25
13
  worker.on('error', reject)