@xen-orchestra/backups 0.73.2 → 0.73.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/RemoteAdapter.mjs +5 -312
- package/_fileRestore.mjs +476 -0
- package/_listPartitions.mjs +9 -0
- package/_runners/Metadata.mjs +18 -8
- package/_runners/VmsRemote.mjs +6 -0
- package/_runners/VmsXapi.mjs +16 -5
- package/_runners/_vmRunners/FullRemote.mjs +7 -2
- package/_runners/_vmRunners/IncrementalRemote.mjs +8 -1
- package/_runners/_vmRunners/IncrementalXapi.mjs +2 -0
- package/_runners/_vmRunners/_TaskProgressHandler.mjs +29 -0
- package/_watchStreamSize.mjs +10 -0
- package/package.json +10 -10
- package/runBackupWorker.mjs +1 -13
package/RemoteAdapter.mjs
CHANGED
|
@@ -1,34 +1,23 @@
|
|
|
1
1
|
import { asyncEach } from '@vates/async-each'
|
|
2
2
|
import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
|
|
3
|
-
import { compose } from '@vates/compose'
|
|
4
3
|
import { createLogger } from '@xen-orchestra/log'
|
|
5
4
|
import { VhdDirectory, VhdSynthetic } from 'vhd-lib'
|
|
6
5
|
import { decorateMethodsWith } from '@vates/decorate-with'
|
|
7
|
-
import { deduped } from '@vates/disposable/deduped.js'
|
|
8
6
|
import { dirname, join, resolve } from 'node:path'
|
|
9
|
-
import { execFile } from 'child_process'
|
|
10
|
-
import { mount } from '@vates/fuse-vhd'
|
|
11
|
-
import { readdir, lstat } from 'node:fs/promises'
|
|
12
7
|
import { synchronized } from 'decorator-synchronized'
|
|
13
|
-
import { ZipFile } from 'yazl'
|
|
14
8
|
import Disposable from 'promise-toolbox/Disposable'
|
|
15
9
|
import fromCallback from 'promise-toolbox/fromCallback'
|
|
16
|
-
import fromEvent from 'promise-toolbox/fromEvent'
|
|
17
10
|
import groupBy from 'lodash/groupBy.js'
|
|
18
|
-
import pDefer from 'promise-toolbox/defer'
|
|
19
11
|
import pickBy from 'lodash/pickBy.js'
|
|
20
12
|
import reduce from 'lodash/reduce.js'
|
|
21
|
-
import * as tar from 'tar'
|
|
22
13
|
import zlib from 'zlib'
|
|
23
14
|
|
|
24
15
|
import { BACKUP_DIR } from './_getVmBackupDir.mjs'
|
|
25
16
|
import { VmBackupDirectory } from '@xen-orchestra/backup-archive'
|
|
17
|
+
import { fileRestoreDecorators, fileRestoreMethods } from './_fileRestore.mjs'
|
|
26
18
|
import { formatFilenameDate } from './_filenameDate.mjs'
|
|
27
|
-
import { getTmpDir } from './_getTmpDir.mjs'
|
|
28
19
|
import { isMetadataFile } from './_backupType.mjs'
|
|
29
20
|
import { isValidXva } from './_isValidXva.mjs'
|
|
30
|
-
import { listPartitions, LVM_PARTITION_TYPE_MBR, LVM_PARTITION_TYPE_GPT } from './_listPartitions.mjs'
|
|
31
|
-
import { lvs, pvs } from './_lvm.mjs'
|
|
32
21
|
import { watchStreamSize } from './_watchStreamSize.mjs'
|
|
33
22
|
|
|
34
23
|
import { RemoteVhdDisk, openDiskChain } from '@xen-orchestra/backup-archive/disks'
|
|
@@ -48,25 +37,6 @@ export const compareTimestamp = (a, b) => a.timestamp - b.timestamp
|
|
|
48
37
|
const noop = Function.prototype
|
|
49
38
|
|
|
50
39
|
const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)
|
|
51
|
-
const makeRelative = path => resolve('/', path).slice(1)
|
|
52
|
-
const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
|
|
53
|
-
|
|
54
|
-
async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
|
|
55
|
-
for (const relativePath of relativePaths) {
|
|
56
|
-
const realPath = join(realBasePath, relativePath)
|
|
57
|
-
const virtualPath = join(virtualBasePath, relativePath)
|
|
58
|
-
|
|
59
|
-
const stats = await lstat(realPath)
|
|
60
|
-
const { mode, mtime } = stats
|
|
61
|
-
const opts = { mode, mtime }
|
|
62
|
-
if (stats.isDirectory()) {
|
|
63
|
-
zip.addEmptyDirectory(virtualPath, opts)
|
|
64
|
-
await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
|
|
65
|
-
} else if (stats.isFile()) {
|
|
66
|
-
zip.addFile(realPath, virtualPath, opts)
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
40
|
|
|
71
41
|
const createSafeReaddir = (handler, methodName) => (path, options) =>
|
|
72
42
|
handler.list(path, options).catch(error => {
|
|
@@ -76,11 +46,6 @@ const createSafeReaddir = (handler, methodName) => (path, options) =>
|
|
|
76
46
|
return []
|
|
77
47
|
})
|
|
78
48
|
|
|
79
|
-
const debounceResourceFactory = factory =>
|
|
80
|
-
function () {
|
|
81
|
-
return this._debounceResource(factory.apply(this, arguments))
|
|
82
|
-
}
|
|
83
|
-
|
|
84
49
|
export class RemoteAdapter {
|
|
85
50
|
constructor(
|
|
86
51
|
handler,
|
|
@@ -98,112 +63,6 @@ export class RemoteAdapter {
|
|
|
98
63
|
return this._handler
|
|
99
64
|
}
|
|
100
65
|
|
|
101
|
-
async _findPartition(devicePath, partitionId) {
|
|
102
|
-
const partitions = await listPartitions(devicePath)
|
|
103
|
-
const partition = partitions.find(_ => _.id === partitionId)
|
|
104
|
-
if (partition === undefined) {
|
|
105
|
-
throw new Error(`partition ${partitionId} not found`)
|
|
106
|
-
}
|
|
107
|
-
return partition
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
|
|
111
|
-
yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
|
|
112
|
-
|
|
113
|
-
debug('activate LVM volume group', { vgName })
|
|
114
|
-
await fromCallback(execFile, 'vgchange', ['-ay', vgName])
|
|
115
|
-
try {
|
|
116
|
-
debug('get LVM volume group name and path', { vgName })
|
|
117
|
-
yield lvs(['lv_name', 'lv_path'], vgName)
|
|
118
|
-
} finally {
|
|
119
|
-
debug('deactivate LVM volume group', { vgName })
|
|
120
|
-
await fromCallback(execFile, 'vgchange', ['-an', vgName])
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
async *_getLvmPhysicalVolume(devicePath, partition) {
|
|
125
|
-
const args = []
|
|
126
|
-
if (partition !== undefined) {
|
|
127
|
-
args.push('-o', partition.start * 512, '--sizelimit', partition.size)
|
|
128
|
-
}
|
|
129
|
-
args.push('--show', '-f', devicePath)
|
|
130
|
-
|
|
131
|
-
debug('attach loop device', { devicePath, partition })
|
|
132
|
-
const path = (await fromCallback(execFile, 'losetup', args)).trim()
|
|
133
|
-
try {
|
|
134
|
-
debug('list LVM physical volume', { path })
|
|
135
|
-
await fromCallback(execFile, 'pvscan', ['--cache', path])
|
|
136
|
-
|
|
137
|
-
yield path
|
|
138
|
-
} finally {
|
|
139
|
-
try {
|
|
140
|
-
const vgNames = await pvs('vg_name', path)
|
|
141
|
-
|
|
142
|
-
debug('deactivate LVM volume groups', { vgNames })
|
|
143
|
-
await fromCallback(execFile, 'vgchange', ['-an', ...vgNames])
|
|
144
|
-
} finally {
|
|
145
|
-
debug('detach loop device', { path })
|
|
146
|
-
await fromCallback(execFile, 'losetup', ['-d', path])
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
async *_getPartition(devicePath, partition) {
|
|
152
|
-
// the norecovery option is necessary because if the partition is dirty,
|
|
153
|
-
// mount will try to fix it which is impossible if because the device is read-only
|
|
154
|
-
const options = ['loop', 'ro', 'norecovery']
|
|
155
|
-
|
|
156
|
-
if (partition !== undefined) {
|
|
157
|
-
const { size, start } = partition
|
|
158
|
-
options.push(`sizelimit=${size}`)
|
|
159
|
-
if (start !== undefined) {
|
|
160
|
-
options.push(`offset=${start * 512}`)
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
const path = yield getTmpDir()
|
|
165
|
-
const mount = options => {
|
|
166
|
-
debug('mount device', { devicePath, mountPath: path })
|
|
167
|
-
return fromCallback(execFile, 'mount', [
|
|
168
|
-
`--options=${options.join(',')}`,
|
|
169
|
-
`--source=${devicePath}`,
|
|
170
|
-
`--target=${path}`,
|
|
171
|
-
])
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
// `norecovery` option is used for ext3/ext4/xfs, if it fails it might be
|
|
175
|
-
// another fs, try without
|
|
176
|
-
try {
|
|
177
|
-
await mount([...options, 'norecovery'])
|
|
178
|
-
} catch (error) {
|
|
179
|
-
await mount(options)
|
|
180
|
-
}
|
|
181
|
-
try {
|
|
182
|
-
yield path
|
|
183
|
-
} finally {
|
|
184
|
-
debug('umount device', { devicePath, mountPath: path })
|
|
185
|
-
await fromCallback(execFile, 'umount', ['--lazy', path])
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
_listLvmLogicalVolumes(devicePath, partition, results = []) {
|
|
190
|
-
return Disposable.use(this._getLvmPhysicalVolume(devicePath, partition), async path => {
|
|
191
|
-
const lvs = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], path)
|
|
192
|
-
const partitionId = partition !== undefined ? partition.id : ''
|
|
193
|
-
lvs.forEach((lv, i) => {
|
|
194
|
-
const name = lv.lv_name
|
|
195
|
-
if (name !== '') {
|
|
196
|
-
results.push({
|
|
197
|
-
id: `${partitionId}/${lv.vg_name}/${name}`,
|
|
198
|
-
name,
|
|
199
|
-
size: lv.lv_size,
|
|
200
|
-
})
|
|
201
|
-
}
|
|
202
|
-
})
|
|
203
|
-
return results
|
|
204
|
-
})
|
|
205
|
-
}
|
|
206
|
-
|
|
207
66
|
// check if we will be allowed to merge a vhd created in this adapter
|
|
208
67
|
// with the vhd at path `path`
|
|
209
68
|
async isMergeableParent(packedParentUid, path) {
|
|
@@ -221,34 +80,6 @@ export class RemoteAdapter {
|
|
|
221
80
|
})
|
|
222
81
|
}
|
|
223
82
|
|
|
224
|
-
fetchPartitionFiles(diskId, partitionId, paths, format) {
|
|
225
|
-
const { promise, reject, resolve } = pDefer()
|
|
226
|
-
Disposable.use(
|
|
227
|
-
async function* () {
|
|
228
|
-
const path = yield this.getPartition(diskId, partitionId)
|
|
229
|
-
let outputStream
|
|
230
|
-
|
|
231
|
-
if (format === 'tgz') {
|
|
232
|
-
outputStream = tar.c({ cwd: path, gzip: true }, paths.map(makeRelative))
|
|
233
|
-
} else if (format === 'zip') {
|
|
234
|
-
const zip = new ZipFile()
|
|
235
|
-
await addZipEntries(zip, path, '', paths.map(makeRelative))
|
|
236
|
-
zip.end()
|
|
237
|
-
;({ outputStream } = zip)
|
|
238
|
-
} else {
|
|
239
|
-
throw new Error('unsupported format ' + format)
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
resolve(outputStream)
|
|
243
|
-
await fromEvent(outputStream, 'end')
|
|
244
|
-
}.bind(this)
|
|
245
|
-
).catch(error => {
|
|
246
|
-
warn(error)
|
|
247
|
-
reject(error)
|
|
248
|
-
})
|
|
249
|
-
return promise
|
|
250
|
-
}
|
|
251
|
-
|
|
252
83
|
async #removeVmBackupsFromCache(backups) {
|
|
253
84
|
await asyncEach(
|
|
254
85
|
Object.entries(
|
|
@@ -386,76 +217,6 @@ export class RemoteAdapter {
|
|
|
386
217
|
return this.useVhdDirectory()
|
|
387
218
|
}
|
|
388
219
|
|
|
389
|
-
async *#getDiskLegacy(diskId) {
|
|
390
|
-
const RE_VHDI = /^vhdi(\d+)$/
|
|
391
|
-
const handler = this._handler
|
|
392
|
-
|
|
393
|
-
const diskPath = handler.getFilePath('/' + diskId)
|
|
394
|
-
const mountDir = yield getTmpDir()
|
|
395
|
-
|
|
396
|
-
debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir })
|
|
397
|
-
await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
|
|
398
|
-
try {
|
|
399
|
-
let max = 0
|
|
400
|
-
let maxEntry
|
|
401
|
-
const entries = await readdir(mountDir)
|
|
402
|
-
entries.forEach(entry => {
|
|
403
|
-
const matches = RE_VHDI.exec(entry)
|
|
404
|
-
if (matches !== null) {
|
|
405
|
-
const value = +matches[1]
|
|
406
|
-
if (value > max) {
|
|
407
|
-
max = value
|
|
408
|
-
maxEntry = entry
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
})
|
|
412
|
-
if (max === 0) {
|
|
413
|
-
throw new Error('no disks found')
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
yield `${mountDir}/${maxEntry}`
|
|
417
|
-
} finally {
|
|
418
|
-
debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir })
|
|
419
|
-
await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
async *getDisk(diskId) {
|
|
424
|
-
if (this._useGetDiskLegacy) {
|
|
425
|
-
yield* this.#getDiskLegacy(diskId)
|
|
426
|
-
return
|
|
427
|
-
}
|
|
428
|
-
const handler = this._handler
|
|
429
|
-
// this is a disposable
|
|
430
|
-
const mountDir = yield getTmpDir()
|
|
431
|
-
// this is also a disposable
|
|
432
|
-
yield mount(handler, diskId, mountDir)
|
|
433
|
-
// this will yield disk path to caller
|
|
434
|
-
yield `${mountDir}/vhd0`
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
// partitionId values:
|
|
438
|
-
//
|
|
439
|
-
// - undefined: raw disk
|
|
440
|
-
// - `<partitionId>`: partitioned disk
|
|
441
|
-
// - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
|
|
442
|
-
// - `/<vgName>/lvName>`: LVM on a raw disk
|
|
443
|
-
async *getPartition(diskId, partitionId) {
|
|
444
|
-
const devicePath = yield this.getDisk(diskId)
|
|
445
|
-
if (partitionId === undefined) {
|
|
446
|
-
return yield this._getPartition(devicePath)
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
const isLvmPartition = partitionId.includes('/')
|
|
450
|
-
if (isLvmPartition) {
|
|
451
|
-
const [pvId, vgName, lvName] = partitionId.split('/')
|
|
452
|
-
const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName)
|
|
453
|
-
return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path)
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
|
|
457
|
-
}
|
|
458
|
-
|
|
459
220
|
// if we use alias on this remote, we have to name the file alias.vhd
|
|
460
221
|
getVhdFileName(baseName) {
|
|
461
222
|
if (this.#useAlias()) {
|
|
@@ -496,56 +257,6 @@ export class RemoteAdapter {
|
|
|
496
257
|
return backups
|
|
497
258
|
}
|
|
498
259
|
|
|
499
|
-
listPartitionFiles(diskId, partitionId, path) {
|
|
500
|
-
return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
|
|
501
|
-
path = resolveSubpath(rootPath, path)
|
|
502
|
-
const entriesMap = {}
|
|
503
|
-
await asyncEach(
|
|
504
|
-
await readdir(path),
|
|
505
|
-
async name => {
|
|
506
|
-
try {
|
|
507
|
-
const stats = await lstat(`${path}/${name}`)
|
|
508
|
-
if (stats.isDirectory()) {
|
|
509
|
-
entriesMap[name + '/'] = {}
|
|
510
|
-
} else if (stats.isFile()) {
|
|
511
|
-
entriesMap[name] = {}
|
|
512
|
-
}
|
|
513
|
-
} catch (error) {
|
|
514
|
-
if (error == null || error.code !== 'ENOENT') {
|
|
515
|
-
throw error
|
|
516
|
-
}
|
|
517
|
-
}
|
|
518
|
-
},
|
|
519
|
-
{ concurrency: 1 }
|
|
520
|
-
)
|
|
521
|
-
|
|
522
|
-
return entriesMap
|
|
523
|
-
})
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
listPartitions(diskId) {
|
|
527
|
-
return Disposable.use(this.getDisk(diskId), async devicePath => {
|
|
528
|
-
const partitions = await listPartitions(devicePath)
|
|
529
|
-
|
|
530
|
-
if (partitions.length === 0) {
|
|
531
|
-
try {
|
|
532
|
-
// handle potential raw LVM physical volume
|
|
533
|
-
return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
|
|
534
|
-
} catch (error) {
|
|
535
|
-
return []
|
|
536
|
-
}
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
const results = []
|
|
540
|
-
await asyncMapSettled(partitions, partition =>
|
|
541
|
-
partition.type === LVM_PARTITION_TYPE_MBR || partition.type === LVM_PARTITION_TYPE_GPT
|
|
542
|
-
? this._listLvmLogicalVolumes(devicePath, partition, results)
|
|
543
|
-
: results.push(partition)
|
|
544
|
-
)
|
|
545
|
-
return results
|
|
546
|
-
})
|
|
547
|
-
}
|
|
548
|
-
|
|
549
260
|
async listPoolMetadataBackups() {
|
|
550
261
|
const handler = this._handler
|
|
551
262
|
const safeReaddir = createSafeReaddir(handler, 'listPoolMetadataBackups')
|
|
@@ -932,26 +643,8 @@ Object.assign(RemoteAdapter.prototype, {
|
|
|
932
643
|
isValidXva,
|
|
933
644
|
})
|
|
934
645
|
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
[deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
|
|
939
|
-
debounceResourceFactory,
|
|
940
|
-
]),
|
|
941
|
-
|
|
942
|
-
_getLvmPhysicalVolume: compose([
|
|
943
|
-
Disposable.factory,
|
|
944
|
-
[deduped, (devicePath, partition) => [devicePath, partition?.id]],
|
|
945
|
-
debounceResourceFactory,
|
|
946
|
-
]),
|
|
947
|
-
|
|
948
|
-
_getPartition: compose([
|
|
949
|
-
Disposable.factory,
|
|
950
|
-
[deduped, (devicePath, partition) => [devicePath, partition?.id]],
|
|
951
|
-
debounceResourceFactory,
|
|
952
|
-
]),
|
|
646
|
+
// File-level-restore methods live in ./_fileRestore.mjs; mix them onto the prototype
|
|
647
|
+
// before decorating so decorateMethodsWith can wrap them.
|
|
648
|
+
Object.assign(RemoteAdapter.prototype, fileRestoreMethods)
|
|
953
649
|
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
getPartition: Disposable.factory,
|
|
957
|
-
})
|
|
650
|
+
decorateMethodsWith(RemoteAdapter, fileRestoreDecorators)
|
package/_fileRestore.mjs
ADDED
|
@@ -0,0 +1,476 @@
|
|
|
1
|
+
// File-level-restore (FLR) methods for RemoteAdapter.
|
|
2
|
+
//
|
|
3
|
+
// These were extracted from RemoteAdapter.mjs to keep that file focused on backup
|
|
4
|
+
// CRUD/cache/metadata. They are mixed back onto `RemoteAdapter.prototype` via
|
|
5
|
+
// `Object.assign` and decorated via `decorateMethodsWith` in RemoteAdapter.mjs, so `this`
|
|
6
|
+
// is the RemoteAdapter instance at call time (`this._handler`, `this._debounceResource`,
|
|
7
|
+
// `this._useGetDiskLegacy` keep working unchanged). The logger keeps the original
|
|
8
|
+
// `xo:backups:RemoteAdapter` namespace so existing debug filters are unaffected.
|
|
9
|
+
import { asyncEach } from '@vates/async-each'
|
|
10
|
+
import { asyncMapSettled } from '@xen-orchestra/async-map'
|
|
11
|
+
import { compose } from '@vates/compose'
|
|
12
|
+
import { createLogger } from '@xen-orchestra/log'
|
|
13
|
+
import { deduped } from '@vates/disposable/deduped.js'
|
|
14
|
+
import { randomBytes } from 'node:crypto'
|
|
15
|
+
import { join, resolve } from 'node:path'
|
|
16
|
+
import { execFile } from 'child_process'
|
|
17
|
+
import { finished } from 'node:stream/promises'
|
|
18
|
+
import { lstat, open, readdir, unlink } from 'node:fs/promises'
|
|
19
|
+
import { tmpdir } from 'node:os'
|
|
20
|
+
import { mount } from '@vates/fuse-vhd'
|
|
21
|
+
import { ZipFile } from 'yazl'
|
|
22
|
+
import Disposable from 'promise-toolbox/Disposable'
|
|
23
|
+
import fromCallback from 'promise-toolbox/fromCallback'
|
|
24
|
+
import pDefer from 'promise-toolbox/defer'
|
|
25
|
+
import * as tar from 'tar'
|
|
26
|
+
|
|
27
|
+
import { getTmpDir } from './_getTmpDir.mjs'
|
|
28
|
+
import {
|
|
29
|
+
listPartitions,
|
|
30
|
+
LINUX_DATA_PARTITION_TYPE_GPT,
|
|
31
|
+
LINUX_DATA_PARTITION_TYPE_MBR,
|
|
32
|
+
LVM_PARTITION_TYPE_GPT,
|
|
33
|
+
LVM_PARTITION_TYPE_MBR,
|
|
34
|
+
} from './_listPartitions.mjs'
|
|
35
|
+
import { lvs, pvs } from './_lvm.mjs'
|
|
36
|
+
|
|
37
|
+
const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
|
|
38
|
+
|
|
39
|
+
const noop = Function.prototype
|
|
40
|
+
|
|
41
|
+
// Restrict LVM scanning to a single device. Backup PVs are clones: the very same PVID
|
|
42
|
+
// appears at once on the raw loop device, the dm-snapshot overlaid on it, and every
|
|
43
|
+
// other restored copy of the same VM. An unscoped pvs/pvscan/vgimportclone/vgchange then
|
|
44
|
+
// aborts with "duplicate PV ... for PVID ..." and the VG can neither be renamed nor
|
|
45
|
+
// activated. Accepting only the device in hand removes every duplicate from LVM's view.
|
|
46
|
+
export const lvmOnlyDevice = devicePath => `devices { global_filter=[ "a|^${devicePath}$|", "r|.*|" ] }`
|
|
47
|
+
|
|
48
|
+
// Partition-type predicates (exported for unit tests).
|
|
49
|
+
export const isLvmPartitionType = type => type === LVM_PARTITION_TYPE_MBR || type === LVM_PARTITION_TYPE_GPT
|
|
50
|
+
|
|
51
|
+
// Some installers (e.g. Ubuntu subiquity) place an LVM PV on a generic Linux-data partition
|
|
52
|
+
// instead of the LVM type, so those are the only non-LVM types worth the (expensive) probe.
|
|
53
|
+
export const isProbeableForLvm = type =>
|
|
54
|
+
type === LINUX_DATA_PARTITION_TYPE_MBR || type === LINUX_DATA_PARTITION_TYPE_GPT
|
|
55
|
+
|
|
56
|
+
// Build the partition entries for a PV's logical volumes (exported for unit tests).
|
|
57
|
+
// Shows "ubuntu-vg/ubuntu-lv" for readability; skips unnamed LVs.
|
|
58
|
+
export const toLvPartitions = (partitionId, originalVgName, lvItems) =>
|
|
59
|
+
lvItems
|
|
60
|
+
.filter(lv => lv.lv_name !== '')
|
|
61
|
+
.map(lv => ({
|
|
62
|
+
id: `${partitionId}/${lv.vg_name}/${lv.lv_name}`,
|
|
63
|
+
name: originalVgName !== undefined ? `${originalVgName}/${lv.lv_name}` : lv.lv_name,
|
|
64
|
+
size: lv.lv_size,
|
|
65
|
+
}))
|
|
66
|
+
|
|
67
|
+
const makeRelative = path => resolve('/', path).slice(1)
|
|
68
|
+
const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
|
|
69
|
+
|
|
70
|
+
async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
|
|
71
|
+
for (const relativePath of relativePaths) {
|
|
72
|
+
const realPath = join(realBasePath, relativePath)
|
|
73
|
+
const virtualPath = join(virtualBasePath, relativePath)
|
|
74
|
+
|
|
75
|
+
const stats = await lstat(realPath)
|
|
76
|
+
const { mode, mtime } = stats
|
|
77
|
+
const opts = { mode, mtime }
|
|
78
|
+
if (stats.isDirectory()) {
|
|
79
|
+
zip.addEmptyDirectory(virtualPath, opts)
|
|
80
|
+
await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
|
|
81
|
+
} else if (stats.isFile()) {
|
|
82
|
+
zip.addFile(realPath, virtualPath, opts)
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const debounceResourceFactory = factory =>
|
|
88
|
+
function () {
|
|
89
|
+
return this._debounceResource(factory.apply(this, arguments))
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// legacy disk mount via vhdimount; extracted from a private method so it can live in this
|
|
93
|
+
// module (mixin methods cannot reference class-private members).
|
|
94
|
+
async function* getDiskLegacy(handler, diskId) {
|
|
95
|
+
const RE_VHDI = /^vhdi(\d+)$/
|
|
96
|
+
|
|
97
|
+
const diskPath = handler.getFilePath('/' + diskId)
|
|
98
|
+
const mountDir = yield getTmpDir()
|
|
99
|
+
|
|
100
|
+
debug('mount VHD (vhdimount)', { diskPath, mountPath: mountDir })
|
|
101
|
+
await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
|
|
102
|
+
try {
|
|
103
|
+
let max = 0
|
|
104
|
+
let maxEntry
|
|
105
|
+
const entries = await readdir(mountDir)
|
|
106
|
+
entries.forEach(entry => {
|
|
107
|
+
const matches = RE_VHDI.exec(entry)
|
|
108
|
+
if (matches !== null) {
|
|
109
|
+
const value = +matches[1]
|
|
110
|
+
if (value > max) {
|
|
111
|
+
max = value
|
|
112
|
+
maxEntry = entry
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
})
|
|
116
|
+
if (max === 0) {
|
|
117
|
+
throw new Error('no disks found')
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
yield `${mountDir}/${maxEntry}`
|
|
121
|
+
} finally {
|
|
122
|
+
debug('umount VHD (fusermount)', { diskPath, mountPath: mountDir })
|
|
123
|
+
await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Mixed onto RemoteAdapter.prototype — `this` is the RemoteAdapter instance.
|
|
128
|
+
export const fileRestoreMethods = {
|
|
129
|
+
async _findPartition(devicePath, partitionId) {
|
|
130
|
+
const partitions = await listPartitions(devicePath)
|
|
131
|
+
const partition = partitions.find(_ => _.id === partitionId)
|
|
132
|
+
if (partition === undefined) {
|
|
133
|
+
throw new Error(`partition ${partitionId} not found`)
|
|
134
|
+
}
|
|
135
|
+
return partition
|
|
136
|
+
},
|
|
137
|
+
|
|
138
|
+
async *_getLvmLogicalVolumes(devicePath, pvId, requestedVgName) {
|
|
139
|
+
const { vgName, lvmConfig } = yield this._getLvmPhysicalVolume(
|
|
140
|
+
devicePath,
|
|
141
|
+
pvId && (await this._findPartition(devicePath, pvId))
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
// vgName is the unique name vgimportclone just assigned to this PV; prefer it over the
|
|
145
|
+
// (possibly stale) name embedded in the partitionId. lvmConfig scopes every command to
|
|
146
|
+
// this device so a colliding/duplicate VG on the host or another copy can't shadow it.
|
|
147
|
+
const effectiveVgName = vgName ?? requestedVgName
|
|
148
|
+
|
|
149
|
+
debug('activate LVM volume group', { effectiveVgName, requestedVgName })
|
|
150
|
+
await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-ay', effectiveVgName])
|
|
151
|
+
try {
|
|
152
|
+
debug('get LVM logical volumes', { effectiveVgName })
|
|
153
|
+
yield lvs(['lv_name', 'lv_path'], '--config', lvmConfig, effectiveVgName)
|
|
154
|
+
} finally {
|
|
155
|
+
debug('deactivate LVM volume group', { effectiveVgName })
|
|
156
|
+
await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', effectiveVgName])
|
|
157
|
+
}
|
|
158
|
+
},
|
|
159
|
+
|
|
160
|
+
async *_getLvmPhysicalVolume(devicePath, partition) {
|
|
161
|
+
const loopArgs = []
|
|
162
|
+
if (partition !== undefined) {
|
|
163
|
+
loopArgs.push('-o', partition.start * 512, '--sizelimit', partition.size)
|
|
164
|
+
}
|
|
165
|
+
loopArgs.push('--show', '-f', devicePath)
|
|
166
|
+
|
|
167
|
+
debug('attach loop device', { devicePath, partition })
|
|
168
|
+
const loopDevice = (await fromCallback(execFile, 'losetup', loopArgs)).trim()
|
|
169
|
+
|
|
170
|
+
let cowPath, cowLoop, mapperName, vgName, lvmConfig
|
|
171
|
+
try {
|
|
172
|
+
// Cheap pre-check, scoped to this loop so a duplicate PVID (another copy of the same
|
|
173
|
+
// VM restored concurrently) can't make it fail: is there an LVM PV here at all?
|
|
174
|
+
// Non-PV partitions (/boot, EFI, raw disks) skip the whole dm-snapshot machinery.
|
|
175
|
+
const [originalVgName] = (
|
|
176
|
+
await pvs('vg_name', '--config', lvmOnlyDevice(loopDevice), loopDevice).catch(() => [])
|
|
177
|
+
).filter(Boolean)
|
|
178
|
+
if (originalVgName === undefined) {
|
|
179
|
+
const where = partition !== undefined ? ` partition ${partition.id}` : ''
|
|
180
|
+
throw new Error(`no LVM physical volume on ${devicePath}${where}`)
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// The backup is read-only, so overlay a writable dm-snapshot to let vgimportclone
|
|
184
|
+
// rewrite metadata, and rename the VG to a unique name so concurrent clones (identical
|
|
185
|
+
// PVID and VG name) don't collide on device-mapper node names. Every LVM command is
|
|
186
|
+
// scoped to the snapshot device (lvmConfig) to keep duplicate PVIDs out of LVM's view.
|
|
187
|
+
mapperName = `xo-pv-${randomBytes(4).toString('hex')}`
|
|
188
|
+
const mapperPath = `/dev/mapper/${mapperName}`
|
|
189
|
+
lvmConfig = lvmOnlyDevice(mapperPath)
|
|
190
|
+
|
|
191
|
+
// ~4 MB sparse COW is enough to hold the LVM metadata rewrites
|
|
192
|
+
cowPath = join(tmpdir(), `${mapperName}.cow`)
|
|
193
|
+
const fh = await open(cowPath, 'w')
|
|
194
|
+
await fh.truncate(4 * 1024 * 1024)
|
|
195
|
+
await fh.close()
|
|
196
|
+
cowLoop = (await fromCallback(execFile, 'losetup', ['--show', '-f', cowPath])).trim()
|
|
197
|
+
|
|
198
|
+
const sectors = (await fromCallback(execFile, 'blockdev', ['--getsz', loopDevice])).trim()
|
|
199
|
+
await fromCallback(execFile, 'dmsetup', [
|
|
200
|
+
'create',
|
|
201
|
+
mapperName,
|
|
202
|
+
'--table',
|
|
203
|
+
`0 ${sectors} snapshot ${loopDevice} ${cowLoop} P 8`,
|
|
204
|
+
])
|
|
205
|
+
|
|
206
|
+
// Unique random VG name: list and mount each query/yield the name from the device, so
|
|
207
|
+
// it need not be deterministic — only collision-free across concurrent clones.
|
|
208
|
+
vgName = `xo${randomBytes(8).toString('hex')}`
|
|
209
|
+
debug('import LVM volume group with unique name via dm-snapshot', { vgName, originalVgName })
|
|
210
|
+
await fromCallback(execFile, 'vgimportclone', ['--config', lvmConfig, '--basevgname', vgName, mapperPath])
|
|
211
|
+
|
|
212
|
+
yield { path: mapperPath, originalVgName, vgName, lvmConfig }
|
|
213
|
+
} finally {
|
|
214
|
+
if (mapperName !== undefined) {
|
|
215
|
+
// best-effort deactivate (a no-op if it was never activated) before removing the snapshot
|
|
216
|
+
await fromCallback(execFile, 'vgchange', ['--config', lvmConfig, '-an', vgName]).catch(noop)
|
|
217
|
+
debug('remove dm-snapshot', { mapperName })
|
|
218
|
+
await fromCallback(execFile, 'dmsetup', ['remove', mapperName]).catch(err =>
|
|
219
|
+
warn('failed to remove dm-snapshot', { mapperName, error: err })
|
|
220
|
+
)
|
|
221
|
+
}
|
|
222
|
+
if (cowLoop !== undefined) {
|
|
223
|
+
await fromCallback(execFile, 'losetup', ['-d', cowLoop]).catch(noop)
|
|
224
|
+
}
|
|
225
|
+
if (cowPath !== undefined) {
|
|
226
|
+
await unlink(cowPath).catch(noop)
|
|
227
|
+
}
|
|
228
|
+
debug('detach loop device', { loopDevice })
|
|
229
|
+
await fromCallback(execFile, 'losetup', ['-d', loopDevice])
|
|
230
|
+
}
|
|
231
|
+
},
|
|
232
|
+
|
|
233
|
+
async *_getPartition(devicePath, partition) {
|
|
234
|
+
const options = ['loop', 'ro']
|
|
235
|
+
|
|
236
|
+
if (partition !== undefined) {
|
|
237
|
+
const { size, start } = partition
|
|
238
|
+
options.push(`sizelimit=${size}`)
|
|
239
|
+
if (start !== undefined) {
|
|
240
|
+
options.push(`offset=${start * 512}`)
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const path = yield getTmpDir()
|
|
245
|
+
const mount = options => {
|
|
246
|
+
debug('mount device', { devicePath, mountPath: path })
|
|
247
|
+
return fromCallback(execFile, 'mount', [
|
|
248
|
+
`--options=${options.join(',')}`,
|
|
249
|
+
`--source=${devicePath}`,
|
|
250
|
+
`--target=${path}`,
|
|
251
|
+
]).catch(error => {
|
|
252
|
+
if (error.stderr) {
|
|
253
|
+
error.message = `${error.message}: ${error.stderr.trim()}`
|
|
254
|
+
}
|
|
255
|
+
throw error
|
|
256
|
+
})
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// norecovery prevents mount from attempting journal replay on a read-only device (ext3/ext4/xfs).
|
|
260
|
+
// Other filesystems don't support it, so fall back without it on failure.
|
|
261
|
+
try {
|
|
262
|
+
await mount([...options, 'norecovery'])
|
|
263
|
+
} catch (error) {
|
|
264
|
+
await mount(options)
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
try {
|
|
268
|
+
yield path
|
|
269
|
+
} finally {
|
|
270
|
+
debug('umount device', { devicePath, mountPath: path })
|
|
271
|
+
await fromCallback(execFile, 'umount', ['--lazy', path])
|
|
272
|
+
}
|
|
273
|
+
},
|
|
274
|
+
|
|
275
|
+
_listLvmLogicalVolumes(devicePath, partition, results = []) {
|
|
276
|
+
return Disposable.use(
|
|
277
|
+
this._getLvmPhysicalVolume(devicePath, partition),
|
|
278
|
+
async ({ path, originalVgName, lvmConfig }) => {
|
|
279
|
+
const lvItems = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], '--config', lvmConfig, path)
|
|
280
|
+
const partitionId = partition !== undefined ? partition.id : ''
|
|
281
|
+
results.push(...toLvPartitions(partitionId, originalVgName, lvItems))
|
|
282
|
+
return results
|
|
283
|
+
}
|
|
284
|
+
)
|
|
285
|
+
},
|
|
286
|
+
|
|
287
|
+
fetchPartitionFiles(diskId, partitionId, paths, format) {
|
|
288
|
+
const { promise, reject, resolve } = pDefer()
|
|
289
|
+
const self = this
|
|
290
|
+
Disposable.use(async function* () {
|
|
291
|
+
const path = yield self.getPartition(diskId, partitionId)
|
|
292
|
+
let outputStream
|
|
293
|
+
|
|
294
|
+
if (format === 'tgz') {
|
|
295
|
+
// process one entry at a time with { job: 1}. node-tar defaults to 4
|
|
296
|
+
// concurrent jobs, which on a FUSE-backed restore mount means up to 4
|
|
297
|
+
// simultaneous reads. Those saturate the libuv threadpool and starve
|
|
298
|
+
// the underlying vhd/CIFS reads NTFS-3g depends on (FUSE-on-FUSE
|
|
299
|
+
// threadpool deadlock). Serializing keeps a worker free for them.
|
|
300
|
+
outputStream = tar.c({ cwd: path, gzip: true, jobs: 1 }, paths.map(makeRelative))
|
|
301
|
+
resolve(outputStream)
|
|
302
|
+
} else if (format === 'zip') {
|
|
303
|
+
const zip = new ZipFile()
|
|
304
|
+
// Resolve with the stream before enumeration so the client can start
|
|
305
|
+
// receiving data immediately — addZipEntries over FUSE/S3 can take
|
|
306
|
+
// minutes for large trees (e.g. node_modules) and would otherwise
|
|
307
|
+
// appear as a freeze with no response sent.
|
|
308
|
+
outputStream = zip.outputStream
|
|
309
|
+
resolve(zip.outputStream)
|
|
310
|
+
await addZipEntries(zip, path, '', paths.map(makeRelative))
|
|
311
|
+
zip.end()
|
|
312
|
+
} else {
|
|
313
|
+
throw new Error('unsupported format ' + format)
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
await finished(outputStream).catch(noop)
|
|
317
|
+
}).catch(error => {
|
|
318
|
+
warn(error)
|
|
319
|
+
reject(error)
|
|
320
|
+
})
|
|
321
|
+
return promise
|
|
322
|
+
},
|
|
323
|
+
|
|
324
|
+
async *getDisk(diskId) {
|
|
325
|
+
if (this._useGetDiskLegacy) {
|
|
326
|
+
yield* getDiskLegacy(this._handler, diskId)
|
|
327
|
+
return
|
|
328
|
+
}
|
|
329
|
+
const handler = this._handler
|
|
330
|
+
// this is a disposable
|
|
331
|
+
const mountDir = yield getTmpDir()
|
|
332
|
+
// this is also a disposable
|
|
333
|
+
yield mount(handler, diskId, mountDir)
|
|
334
|
+
// this will yield disk path to caller
|
|
335
|
+
yield `${mountDir}/vhd0`
|
|
336
|
+
},
|
|
337
|
+
|
|
338
|
+
// partitionId values:
|
|
339
|
+
//
|
|
340
|
+
// - undefined: raw disk
|
|
341
|
+
// - `<partitionId>`: partitioned disk
|
|
342
|
+
// - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
|
|
343
|
+
// - `/<vgName>/lvName>`: LVM on a raw disk
|
|
344
|
+
async *getPartition(diskId, partitionId) {
|
|
345
|
+
const devicePath = yield this.getDisk(diskId)
|
|
346
|
+
if (partitionId === undefined) {
|
|
347
|
+
debug(
|
|
348
|
+
'no partition specified, attempting raw disk mount — call listPartitions first if disk has partitions or LVM'
|
|
349
|
+
)
|
|
350
|
+
return yield this._getPartition(devicePath)
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
const isLvmPartition = partitionId.includes('/')
|
|
354
|
+
if (isLvmPartition) {
|
|
355
|
+
const [pvId, vgName, lvName] = partitionId.split('/')
|
|
356
|
+
const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName)
|
|
357
|
+
return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path)
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
|
|
361
|
+
},
|
|
362
|
+
|
|
363
|
+
listPartitionFiles(diskId, partitionId, path) {
|
|
364
|
+
return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
|
|
365
|
+
path = resolveSubpath(rootPath, path)
|
|
366
|
+
const entriesMap = {}
|
|
367
|
+
await asyncEach(
|
|
368
|
+
await readdir(path),
|
|
369
|
+
async name => {
|
|
370
|
+
try {
|
|
371
|
+
const stats = await lstat(`${path}/${name}`)
|
|
372
|
+
if (stats.isDirectory()) {
|
|
373
|
+
entriesMap[name + '/'] = {}
|
|
374
|
+
} else if (stats.isFile()) {
|
|
375
|
+
entriesMap[name] = {}
|
|
376
|
+
}
|
|
377
|
+
} catch (error) {
|
|
378
|
+
if (error == null || error.code !== 'ENOENT') {
|
|
379
|
+
throw error
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
},
|
|
383
|
+
{ concurrency: 1 }
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
return entriesMap
|
|
387
|
+
})
|
|
388
|
+
},
|
|
389
|
+
|
|
390
|
+
listPartitions(diskId) {
|
|
391
|
+
return Disposable.use(this.getDisk(diskId), async devicePath => {
|
|
392
|
+
// partx may return empty on FUSE-backed files (vhd0); a loop device
|
|
393
|
+
// presents proper block-device semantics that partx reads reliably.
|
|
394
|
+
// losetup may itself fail if the FUSE mount isn't fully ready yet —
|
|
395
|
+
// fall back to the direct path so listPartitions returns [] instead of throwing.
|
|
396
|
+
let loopForParts, partitions
|
|
397
|
+
try {
|
|
398
|
+
loopForParts = (await fromCallback(execFile, 'losetup', ['--show', '-f', devicePath])).trim()
|
|
399
|
+
partitions = await listPartitions(loopForParts)
|
|
400
|
+
} catch (error) {
|
|
401
|
+
debug('partition probe via loop device failed, falling back to direct path', { error })
|
|
402
|
+
partitions = await listPartitions(devicePath)
|
|
403
|
+
} finally {
|
|
404
|
+
if (loopForParts !== undefined) {
|
|
405
|
+
await fromCallback(execFile, 'losetup', ['-d', loopForParts]).catch(noop)
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
if (partitions.length === 0) {
|
|
410
|
+
try {
|
|
411
|
+
// handle potential raw LVM physical volume
|
|
412
|
+
return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
|
|
413
|
+
} catch (error) {
|
|
414
|
+
return []
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
const results = []
|
|
419
|
+
await asyncMapSettled(partitions, async partition => {
|
|
420
|
+
if (isLvmPartitionType(partition.type)) {
|
|
421
|
+
return this._listLvmLogicalVolumes(devicePath, partition, results)
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
// Only generic Linux-data partitions can hide an LVM PV (subiquity-style); other
|
|
425
|
+
// types — BIOS boot, EFI, swap, … — are never PVs, so list them directly without
|
|
426
|
+
// the (expensive) loop + dm-snapshot + vgimportclone probe.
|
|
427
|
+
if (!isProbeableForLvm(partition.type)) {
|
|
428
|
+
results.push(partition)
|
|
429
|
+
return
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
const lvResults = []
|
|
433
|
+
try {
|
|
434
|
+
await this._listLvmLogicalVolumes(devicePath, partition, lvResults)
|
|
435
|
+
} catch (error) {
|
|
436
|
+
debug('LVM probe failed for Linux-data partition, treating as regular partition', {
|
|
437
|
+
partition,
|
|
438
|
+
error,
|
|
439
|
+
})
|
|
440
|
+
}
|
|
441
|
+
if (lvResults.length > 0) {
|
|
442
|
+
results.push(...lvResults)
|
|
443
|
+
} else {
|
|
444
|
+
results.push(partition)
|
|
445
|
+
}
|
|
446
|
+
})
|
|
447
|
+
return results
|
|
448
|
+
})
|
|
449
|
+
},
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// Applied by RemoteAdapter.mjs via decorateMethodsWith(RemoteAdapter, fileRestoreDecorators).
|
|
453
|
+
// debounceResourceFactory/deduped reference `this._debounceResource` at call time.
|
|
454
|
+
export const fileRestoreDecorators = {
|
|
455
|
+
_getLvmLogicalVolumes: compose([
|
|
456
|
+
Disposable.factory,
|
|
457
|
+
[deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
|
|
458
|
+
debounceResourceFactory,
|
|
459
|
+
]),
|
|
460
|
+
|
|
461
|
+
_getLvmPhysicalVolume: compose([
|
|
462
|
+
Disposable.factory,
|
|
463
|
+
[deduped, (devicePath, partition) => [devicePath, partition?.id]],
|
|
464
|
+
debounceResourceFactory,
|
|
465
|
+
]),
|
|
466
|
+
|
|
467
|
+
_getPartition: compose([
|
|
468
|
+
Disposable.factory,
|
|
469
|
+
[deduped, (devicePath, partition) => [devicePath, partition?.id]],
|
|
470
|
+
debounceResourceFactory,
|
|
471
|
+
]),
|
|
472
|
+
|
|
473
|
+
getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
|
|
474
|
+
|
|
475
|
+
getPartition: Disposable.factory,
|
|
476
|
+
}
|
package/_listPartitions.mjs
CHANGED
|
@@ -29,6 +29,15 @@ export const LVM_PARTITION_TYPE_MBR = 0x8e
|
|
|
29
29
|
// GPT LVM type
|
|
30
30
|
export const LVM_PARTITION_TYPE_GPT = 'e6d6d379-f507-44c2-a23c-238f2a3df928'
|
|
31
31
|
|
|
32
|
+
// Generic "Linux filesystem data" types. These are the only non-LVM-typed
|
|
33
|
+
// partitions worth probing for a hidden LVM PV, because some installers (e.g.
|
|
34
|
+
// Ubuntu subiquity) place a PV on a partition left with this generic type.
|
|
35
|
+
// Other types (BIOS boot, EFI, swap, …) are never LVM PVs.
|
|
36
|
+
// MBR Linux native
|
|
37
|
+
export const LINUX_DATA_PARTITION_TYPE_MBR = 0x83
|
|
38
|
+
// GPT Linux filesystem data
|
|
39
|
+
export const LINUX_DATA_PARTITION_TYPE_GPT = '0fc63daf-8483-4772-8e79-3d69d8477de4'
|
|
40
|
+
|
|
32
41
|
const parsePartxLine = createParser({
|
|
33
42
|
keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
|
|
34
43
|
valueTransform: (value, key) => {
|
package/_runners/Metadata.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { asyncEach } from '@vates/async-each'
|
|
2
2
|
import { Task } from '@vates/task'
|
|
3
3
|
import Disposable from 'promise-toolbox/Disposable'
|
|
4
4
|
import ignoreErrors from 'promise-toolbox/ignoreErrors'
|
|
@@ -82,10 +82,11 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
|
|
|
82
82
|
// remove pools that failed (already handled)
|
|
83
83
|
pools = pools.filter(_ => _ !== undefined)
|
|
84
84
|
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
85
|
+
const tasks = []
|
|
86
|
+
|
|
87
|
+
if (settings.retentionPoolMetadata !== 0) {
|
|
88
|
+
for (const pool of pools) {
|
|
89
|
+
tasks.push(async () =>
|
|
89
90
|
Task.run(
|
|
90
91
|
{
|
|
91
92
|
properties: {
|
|
@@ -107,11 +108,11 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
|
|
|
107
108
|
}).run()
|
|
108
109
|
).catch(noop)
|
|
109
110
|
)
|
|
110
|
-
|
|
111
|
+
}
|
|
111
112
|
}
|
|
112
113
|
|
|
113
114
|
if (job.xoMetadata !== undefined && settings.retentionXoMetadata !== 0) {
|
|
114
|
-
|
|
115
|
+
tasks.push(() =>
|
|
115
116
|
Task.run(
|
|
116
117
|
{
|
|
117
118
|
properties: {
|
|
@@ -130,7 +131,16 @@ export const Metadata = class MetadataBackupRunner extends Abstract {
|
|
|
130
131
|
).catch(noop)
|
|
131
132
|
)
|
|
132
133
|
}
|
|
133
|
-
|
|
134
|
+
|
|
135
|
+
const total = tasks.length
|
|
136
|
+
let transferred = 0
|
|
137
|
+
|
|
138
|
+
await asyncEach(tasks, async task => {
|
|
139
|
+
await task()
|
|
140
|
+
transferred++
|
|
141
|
+
Task.set('progress', Math.round(transferred / total))
|
|
142
|
+
})
|
|
143
|
+
Task.set('progress', 1)
|
|
134
144
|
}
|
|
135
145
|
)
|
|
136
146
|
}
|
package/_runners/VmsRemote.mjs
CHANGED
|
@@ -63,6 +63,8 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
|
|
63
63
|
const vmsUuids = await sourceRemoteAdapter.listAllVms()
|
|
64
64
|
|
|
65
65
|
Task.info('vms', { vms: vmsUuids })
|
|
66
|
+
const nbVms = vmsUuids.length
|
|
67
|
+
let nbVmsDone = 0
|
|
66
68
|
|
|
67
69
|
remoteAdapters = getAdaptersByRemote(remoteAdapters)
|
|
68
70
|
const allSettings = this._job.settings
|
|
@@ -123,11 +125,14 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
|
|
123
125
|
)
|
|
124
126
|
.then(result => {
|
|
125
127
|
if (taskError === undefined) {
|
|
128
|
+
nbVmsDone++
|
|
126
129
|
return task.success(result)
|
|
127
130
|
}
|
|
128
131
|
if (isLastRun) {
|
|
132
|
+
nbVmsDone++
|
|
129
133
|
return task.failure(taskError)
|
|
130
134
|
}
|
|
135
|
+
Task.set('progress', Math.round((nbVmsDone * 100) / nbVms))
|
|
131
136
|
// don't end the task
|
|
132
137
|
task.warning(`Retry the VM mirror backup due to an error`, {
|
|
133
138
|
attempt: nTriesByVmId[vmUuid],
|
|
@@ -148,6 +153,7 @@ export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
|
|
|
148
153
|
|
|
149
154
|
await asyncMapSettled(vmIds, _handleVm)
|
|
150
155
|
}
|
|
156
|
+
Task.set('progress', 100)
|
|
151
157
|
}
|
|
152
158
|
)
|
|
153
159
|
}
|
package/_runners/VmsXapi.mjs
CHANGED
|
@@ -101,10 +101,14 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
|
|
101
101
|
|
|
102
102
|
const handleVm = vmUuid => {
|
|
103
103
|
const getVmTask = () => {
|
|
104
|
-
|
|
104
|
+
const started = taskByVmId[vmUuid] !== undefined
|
|
105
|
+
if (!started) {
|
|
105
106
|
taskByVmId[vmUuid] = new Task(taskStart)
|
|
106
107
|
}
|
|
107
|
-
return
|
|
108
|
+
return {
|
|
109
|
+
task: taskByVmId[vmUuid],
|
|
110
|
+
started,
|
|
111
|
+
}
|
|
108
112
|
}
|
|
109
113
|
const vmBackupFailed = async (error, task) => {
|
|
110
114
|
if (isLastRun) {
|
|
@@ -135,7 +139,7 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
|
|
135
139
|
taskStart.properties.name_label = vm.name_label
|
|
136
140
|
}
|
|
137
141
|
|
|
138
|
-
const task = getVmTask()
|
|
142
|
+
const { task } = getVmTask()
|
|
139
143
|
// error has to be caught in the task to prevent its failure, but handled outside the task to execute another task.run()
|
|
140
144
|
let taskError
|
|
141
145
|
return task
|
|
@@ -174,12 +178,19 @@ export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
|
|
|
174
178
|
task.success(result)
|
|
175
179
|
} else {
|
|
176
180
|
// ending the task with error or not ending the task
|
|
177
|
-
vmBackupFailed(taskError, task)
|
|
181
|
+
return vmBackupFailed(taskError, task)
|
|
178
182
|
}
|
|
179
183
|
})
|
|
180
184
|
.catch(noop) // errors are handled by logs
|
|
181
185
|
}),
|
|
182
|
-
error =>
|
|
186
|
+
error => {
|
|
187
|
+
const { task: vmTask, started } = getVmTask()
|
|
188
|
+
if (!started) {
|
|
189
|
+
// the task is not started (except if it's a retry), and an unstarted task can't be failed
|
|
190
|
+
vmTask.start()
|
|
191
|
+
}
|
|
192
|
+
return vmBackupFailed(error, vmTask)
|
|
193
|
+
}
|
|
183
194
|
)
|
|
184
195
|
}
|
|
185
196
|
const { concurrency } = settings
|
|
@@ -3,6 +3,7 @@ import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
|
|
|
3
3
|
import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
|
|
4
4
|
import { watchStreamSize } from '../../_watchStreamSize.mjs'
|
|
5
5
|
import { AggregatedFullRemoteWriter } from '../_writers/AggregatedFullRemoteWriter.mjs'
|
|
6
|
+
import { Task } from '@vates/task'
|
|
6
7
|
|
|
7
8
|
export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
|
|
8
9
|
_getRemoteWriters() {
|
|
@@ -15,7 +16,8 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
|
|
|
15
16
|
|
|
16
17
|
async _run() {
|
|
17
18
|
const transferList = await this._computeTransferList(({ mode }) => mode === 'full')
|
|
18
|
-
|
|
19
|
+
const nbTransferrableVms = transferList.length
|
|
20
|
+
let nbTransferredVms = 0
|
|
19
21
|
for (const metadata of transferList) {
|
|
20
22
|
const stream = this._throttleStream(await this._sourceRemoteAdapter.readFullVmBackup(metadata))
|
|
21
23
|
const sizeContainer = watchStreamSize(stream)
|
|
@@ -27,7 +29,7 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
|
|
|
27
29
|
stream: forkStreamUnpipe(stream),
|
|
28
30
|
// stream will be forked and transformed, it's not safe to attach additional properties to it
|
|
29
31
|
streamLength: stream.length,
|
|
30
|
-
maxStreamLength: stream.maxStreamLength, //
|
|
32
|
+
maxStreamLength: stream.maxStreamLength, // for encrypted destination/source without length
|
|
31
33
|
timestamp: metadata.timestamp,
|
|
32
34
|
vm: metadata.vm,
|
|
33
35
|
vmSnapshot: metadata.vmSnapshot,
|
|
@@ -37,7 +39,10 @@ export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote
|
|
|
37
39
|
)
|
|
38
40
|
// for healthcheck
|
|
39
41
|
this._tags = metadata.vm.tags
|
|
42
|
+
nbTransferredVms++
|
|
43
|
+
Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
|
|
40
44
|
}
|
|
45
|
+
Task.set('progress', 100)
|
|
41
46
|
this._hasTransferredData = transferList.length > 0
|
|
42
47
|
}
|
|
43
48
|
}
|
|
@@ -12,6 +12,8 @@ import { openVhd } from 'vhd-lib'
|
|
|
12
12
|
import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
|
|
13
13
|
import { SynchronizedDisk, ThrottledDisk } from '@xen-orchestra/disk-transform'
|
|
14
14
|
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
|
|
15
|
+
import { Task } from '@vates/task'
|
|
16
|
+
import { TaskProgressHandler } from './_TaskProgressHandler.mjs'
|
|
15
17
|
|
|
16
18
|
const { warn } = createLogger('xo:backups:Incrementalremote')
|
|
17
19
|
class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
|
@@ -60,7 +62,8 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
|
|
60
62
|
}
|
|
61
63
|
async _run() {
|
|
62
64
|
const transferList = await this._computeTransferList(({ mode }) => mode === 'delta')
|
|
63
|
-
|
|
65
|
+
const nbTransferrableVms = transferList.length
|
|
66
|
+
let nbTransferredVms = 0
|
|
64
67
|
for (const metadata of transferList) {
|
|
65
68
|
assert.strictEqual(metadata.mode, 'delta')
|
|
66
69
|
const incrementalExport = await this._sourceRemoteAdapter.readIncrementalVmBackup(metadata, undefined, {
|
|
@@ -74,6 +77,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
|
|
74
77
|
for (const key in incrementalExport.disks) {
|
|
75
78
|
let disk = incrementalExport.disks[key]
|
|
76
79
|
isVhdDifferencing[key] = disk.isDifferencing()
|
|
80
|
+
disk.addProgressHandler(new TaskProgressHandler())
|
|
77
81
|
disk = new ThrottledDisk(disk, this._throttleGenerator)
|
|
78
82
|
incrementalExport.disks[key] = new SynchronizedDisk(disk)
|
|
79
83
|
}
|
|
@@ -116,7 +120,10 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
|
|
|
116
120
|
await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
|
|
117
121
|
// for healthcheck
|
|
118
122
|
this._tags = metadata.vm.tags
|
|
123
|
+
nbTransferredVms++
|
|
124
|
+
Task.set('progress', Math.round((nbTransferredVms * 100) / nbTransferrableVms))
|
|
119
125
|
}
|
|
126
|
+
Task.set('progress', 100)
|
|
120
127
|
this._hasTransferredData = transferList.length > 0
|
|
121
128
|
}
|
|
122
129
|
}
|
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
import { ThrottledDisk, SynchronizedDisk } from '@xen-orchestra/disk-transform'
|
|
19
19
|
import { AggregatedIncrementalRemoteWriter } from '../_writers/AggregatedIncrementalRemoteWriter.mjs'
|
|
20
20
|
import { AggregatedIncrementalXapiWriter } from '../_writers/AggregatedIncrementalXapiWriter.mjs'
|
|
21
|
+
import { TaskProgressHandler } from './_TaskProgressHandler.mjs'
|
|
21
22
|
|
|
22
23
|
const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
|
|
23
24
|
|
|
@@ -52,6 +53,7 @@ export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends Abstr
|
|
|
52
53
|
let useNbd = false
|
|
53
54
|
for (const key in deltaExport.disks) {
|
|
54
55
|
let disk = deltaExport.disks[key]
|
|
56
|
+
disk.addProgressHandler(new TaskProgressHandler())
|
|
55
57
|
isVhdDifferencing[key] = disk.isDifferencing()
|
|
56
58
|
if (!isFull && !isVhdDifferencing[key] && key !== exportedVm.$suspend_VDI?.$ref) {
|
|
57
59
|
Task.warning('Backup fell back to a full')
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { Task } from '@vates/task'
|
|
2
|
+
|
|
3
|
+
const MAX_DURATION_BETWEEN_PROGRESS_EMIT = 5e3
|
|
4
|
+
const MIN_THRESHOLD_PERCENT_BETWEEN_PROGRESS_EMIT = 0.01
|
|
5
|
+
|
|
6
|
+
export class TaskProgressHandler {
|
|
7
|
+
#lastProgressDate
|
|
8
|
+
#lastProgressValue
|
|
9
|
+
constructor() {
|
|
10
|
+
Task.set('progress', 0)
|
|
11
|
+
}
|
|
12
|
+
async setProgress(progress) {
|
|
13
|
+
if (progress < 0 || progress > 1) {
|
|
14
|
+
return
|
|
15
|
+
}
|
|
16
|
+
if (
|
|
17
|
+
this.#lastProgressDate !== undefined &&
|
|
18
|
+
this.#lastProgressValue !== undefined &&
|
|
19
|
+
Date.now() - this.#lastProgressDate < MAX_DURATION_BETWEEN_PROGRESS_EMIT &&
|
|
20
|
+
progress - this.#lastProgressValue < MIN_THRESHOLD_PERCENT_BETWEEN_PROGRESS_EMIT
|
|
21
|
+
) {
|
|
22
|
+
return
|
|
23
|
+
}
|
|
24
|
+
this.#lastProgressDate = Date.now()
|
|
25
|
+
this.#lastProgressValue = progress
|
|
26
|
+
Task.set('progress', Math.round(progress * 100))
|
|
27
|
+
}
|
|
28
|
+
done() {}
|
|
29
|
+
}
|
package/_watchStreamSize.mjs
CHANGED
|
@@ -1,6 +1,16 @@
|
|
|
1
|
+
import { TaskProgressHandler } from './_runners/_vmRunners/_TaskProgressHandler.mjs'
|
|
1
2
|
export function watchStreamSize(stream, container = { size: 0 }) {
|
|
3
|
+
const progressHandler = container.progressHandler ?? new TaskProgressHandler()
|
|
2
4
|
stream.on('data', data => {
|
|
3
5
|
container.size += data.length
|
|
6
|
+
if (stream.length) {
|
|
7
|
+
// empty for xva exported by xapi
|
|
8
|
+
progressHandler.setProgress(container.size / stream.length)
|
|
9
|
+
}
|
|
10
|
+
})
|
|
11
|
+
|
|
12
|
+
stream.on('finish', () => {
|
|
13
|
+
progressHandler.setProgress(1)
|
|
4
14
|
})
|
|
5
15
|
stream.pause()
|
|
6
16
|
return container
|
package/package.json
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"type": "git",
|
|
9
9
|
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
|
10
10
|
},
|
|
11
|
-
"version": "0.73.
|
|
11
|
+
"version": "0.73.4",
|
|
12
12
|
"engines": {
|
|
13
13
|
"node": ">=14.18"
|
|
14
14
|
},
|
|
@@ -25,19 +25,19 @@
|
|
|
25
25
|
"@vates/compose": "^2.1.0",
|
|
26
26
|
"@vates/decorate-with": "^2.1.0",
|
|
27
27
|
"@vates/disposable": "^0.1.6",
|
|
28
|
-
"@vates/fuse-vhd": "^2.1.
|
|
29
|
-
"@vates/generator-toolbox": "^1.1.
|
|
28
|
+
"@vates/fuse-vhd": "^2.1.3",
|
|
29
|
+
"@vates/generator-toolbox": "^1.1.3",
|
|
30
30
|
"@vates/nbd-client": "^3.4.0",
|
|
31
31
|
"@vates/parse-duration": "^0.1.1",
|
|
32
32
|
"@vates/task": "^0.7.0",
|
|
33
|
-
"@vates/types": "^1.
|
|
33
|
+
"@vates/types": "^1.27.0",
|
|
34
34
|
"@xen-orchestra/async-map": "^0.1.3",
|
|
35
|
-
"@xen-orchestra/disk-transform": "^1.3.
|
|
36
|
-
"@xen-orchestra/fs": "^4.9.
|
|
35
|
+
"@xen-orchestra/disk-transform": "^1.3.1",
|
|
36
|
+
"@xen-orchestra/fs": "^4.9.1",
|
|
37
37
|
"@xen-orchestra/log": "^0.7.2",
|
|
38
|
-
"@xen-orchestra/qcow2": "^1.3.
|
|
38
|
+
"@xen-orchestra/qcow2": "^1.3.1",
|
|
39
39
|
"@xen-orchestra/template": "^0.1.1",
|
|
40
|
-
"@xen-orchestra/backup-archive": "^1.0.
|
|
40
|
+
"@xen-orchestra/backup-archive": "^1.0.2",
|
|
41
41
|
"app-conf": "^3.0.0",
|
|
42
42
|
"compare-versions": "^6.0.0",
|
|
43
43
|
"d3-time-format": "^4.1.0",
|
|
@@ -56,7 +56,7 @@
|
|
|
56
56
|
"uuid": "^9.0.0",
|
|
57
57
|
"value-matcher": "^0.2.0",
|
|
58
58
|
"vhd-lib": "^4.16.0",
|
|
59
|
-
"xen-api": "^4.7.
|
|
59
|
+
"xen-api": "^4.7.8",
|
|
60
60
|
"yazl": "^2.5.1"
|
|
61
61
|
},
|
|
62
62
|
"devDependencies": {
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
"typescript": "^5.9.3"
|
|
68
68
|
},
|
|
69
69
|
"peerDependencies": {
|
|
70
|
-
"@xen-orchestra/xapi": "^8.
|
|
70
|
+
"@xen-orchestra/xapi": "^8.9.0"
|
|
71
71
|
},
|
|
72
72
|
"license": "AGPL-3.0-or-later",
|
|
73
73
|
"author": {
|
package/runBackupWorker.mjs
CHANGED
|
@@ -4,22 +4,10 @@ import { fork } from 'child_process'
|
|
|
4
4
|
const { warn } = createLogger('xo:backups:backupWorker')
|
|
5
5
|
|
|
6
6
|
const PATH = new URL('_backupWorker.mjs', import.meta.url).pathname
|
|
7
|
-
const DEFAULT_INSPECTOR_PORT = 9229
|
|
8
7
|
|
|
9
8
|
export function runBackupWorker(params, onLog) {
|
|
10
9
|
return new Promise((resolve, reject) => {
|
|
11
|
-
|
|
12
|
-
const inspectArg = process.execArgv.find(arg => arg.startsWith('--inspect') || arg.startsWith('--inspect-brk'))
|
|
13
|
-
const execArgv = inspectArg
|
|
14
|
-
? [
|
|
15
|
-
inspectArg.replace(/^(--inspect(-brk)?)(=([^:]+:)?(\d+))?$/, (_, prefix, brk, _fullMatch, host, port) => {
|
|
16
|
-
const basePort = port ? parseInt(port) : DEFAULT_INSPECTOR_PORT
|
|
17
|
-
return `${prefix}=${host || ''}${basePort + 1}`
|
|
18
|
-
}),
|
|
19
|
-
]
|
|
20
|
-
: []
|
|
21
|
-
|
|
22
|
-
const worker = fork(PATH, [], { execArgv })
|
|
10
|
+
const worker = fork(PATH, [])
|
|
23
11
|
|
|
24
12
|
worker.on('exit', (code, signal) => reject(new Error(`worker exited with code ${code} and signal ${signal}`)))
|
|
25
13
|
worker.on('error', reject)
|