@datatruck/cli 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/BackupAction.d.ts +10 -3
- package/Action/BackupAction.js +41 -49
- package/Action/RestoreAction.d.ts +2 -2
- package/Action/RestoreAction.js +8 -16
- package/Command/BackupCommand.js +2 -1
- package/Command/BackupSessionsCommand.js +1 -0
- package/Command/RestoreCommand.js +1 -1
- package/Command/RestoreSessionsCommand.js +1 -0
- package/Entity/StateEntityAbstract.d.ts +2 -5
- package/Error/AppError.d.ts +1 -0
- package/Error/AppError.js +4 -0
- package/Repository/DatatruckRepository.d.ts +1 -0
- package/Repository/DatatruckRepository.js +162 -158
- package/Repository/RepositoryAbstract.d.ts +4 -10
- package/Repository/ResticRepository.js +34 -17
- package/SessionDriver/ConsoleSessionDriver.d.ts +2 -7
- package/SessionDriver/ConsoleSessionDriver.js +51 -24
- package/SessionDriver/SqliteSessionDriver.js +5 -0
- package/SessionManager/BackupSessionManager.d.ts +12 -11
- package/SessionManager/BackupSessionManager.js +20 -5
- package/SessionManager/RestoreSessionManager.d.ts +14 -11
- package/SessionManager/RestoreSessionManager.js +20 -5
- package/SessionManager/SessionManagerAbstract.d.ts +18 -0
- package/SessionManager/SessionManagerAbstract.js +32 -0
- package/Task/GitTask.js +22 -14
- package/Task/MariadbTask.js +9 -4
- package/Task/MysqlDumpTask.d.ts +3 -1
- package/Task/MysqlDumpTask.js +5 -2
- package/Task/PostgresqlDumpTask.d.ts +3 -1
- package/Task/PostgresqlDumpTask.js +2 -2
- package/Task/SqlDumpTaskAbstract.d.ts +3 -1
- package/Task/SqlDumpTaskAbstract.js +55 -13
- package/Task/TaskAbstract.d.ts +3 -9
- package/cli.js +1 -1
- package/migrations/001-initial.sql +6 -30
- package/package.json +1 -1
- package/util/cli-util.d.ts +1 -1
- package/util/cli-util.js +17 -2
- package/util/fs-util.d.ts +10 -1
- package/util/fs-util.js +10 -1
- package/util/process-util.d.ts +3 -0
- package/util/process-util.js +11 -0
- package/util/progress.d.ts +12 -0
- package/util/progress.js +2 -0
- package/util/zip-util.d.ts +23 -5
- package/util/zip-util.js +82 -25
|
@@ -19,6 +19,7 @@ const fs_1 = require("fs");
|
|
|
19
19
|
const promises_1 = require("fs/promises");
|
|
20
20
|
const micromatch_1 = require("micromatch");
|
|
21
21
|
const path_1 = require("path");
|
|
22
|
+
const perf_hooks_1 = require("perf_hooks");
|
|
22
23
|
const readline_1 = require("readline");
|
|
23
24
|
exports.datatruckRepositoryName = "datatruck";
|
|
24
25
|
exports.datatruckRepositoryDefinition = {
|
|
@@ -75,8 +76,6 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
75
76
|
return `${date}_${pkgName}_${snapshotShortId}`;
|
|
76
77
|
}
|
|
77
78
|
static parseSnapshotName(name) {
|
|
78
|
-
if (!name.endsWith(".json"))
|
|
79
|
-
return null;
|
|
80
79
|
name = name.replace(/\.json$/, "");
|
|
81
80
|
const nameParts = name.split("_");
|
|
82
81
|
if (nameParts.length !== 3)
|
|
@@ -103,6 +102,66 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
103
102
|
async onInit(data) {
|
|
104
103
|
await (0, fs_util_1.mkdirIfNotExists)(this.config.outPath);
|
|
105
104
|
}
|
|
105
|
+
async createFileScanner(options) {
|
|
106
|
+
const object = {
|
|
107
|
+
total: 0,
|
|
108
|
+
current: 0,
|
|
109
|
+
progress: async (description, data) => {
|
|
110
|
+
await options.onProgress({
|
|
111
|
+
relative: {
|
|
112
|
+
description,
|
|
113
|
+
payload: data.path,
|
|
114
|
+
percent: data.percent,
|
|
115
|
+
},
|
|
116
|
+
absolute: {
|
|
117
|
+
total: object.total,
|
|
118
|
+
current: object.current + data.current,
|
|
119
|
+
percent: (0, math_util_1.progressPercent)(object.total, object.current + data.current),
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
if (data.type === "end") {
|
|
123
|
+
object.current += data.current;
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
updateProgress: async (end) => {
|
|
127
|
+
const currentTime = perf_hooks_1.performance.now();
|
|
128
|
+
const diff = currentTime - lastTime;
|
|
129
|
+
if (end || diff > 1000) {
|
|
130
|
+
await options.onProgress({
|
|
131
|
+
relative: {
|
|
132
|
+
description: end ? "Scanned files" : "Scanning files",
|
|
133
|
+
payload: object.total.toString(),
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
lastTime = currentTime;
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
start: async (cb) => {
|
|
140
|
+
for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
|
|
141
|
+
if (!options.disableCounting)
|
|
142
|
+
object.total++;
|
|
143
|
+
await object.updateProgress();
|
|
144
|
+
if (cb)
|
|
145
|
+
await cb(entry);
|
|
146
|
+
}
|
|
147
|
+
if (!options.disableEndProgress)
|
|
148
|
+
await object.updateProgress(true);
|
|
149
|
+
},
|
|
150
|
+
};
|
|
151
|
+
await options.onProgress({
|
|
152
|
+
relative: {
|
|
153
|
+
description: "Scanning files",
|
|
154
|
+
},
|
|
155
|
+
});
|
|
156
|
+
const stream = fast_glob_1.default.stream(options.glob.include, {
|
|
157
|
+
dot: true,
|
|
158
|
+
markDirectories: true,
|
|
159
|
+
stats: true,
|
|
160
|
+
...options.glob,
|
|
161
|
+
});
|
|
162
|
+
let lastTime = perf_hooks_1.performance.now();
|
|
163
|
+
return object;
|
|
164
|
+
}
|
|
106
165
|
async onPrune(data) {
|
|
107
166
|
const snapshotName = DatatruckRepository.buildSnapshotName({
|
|
108
167
|
snapshotId: data.snapshot.id,
|
|
@@ -110,15 +169,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
110
169
|
packageName: data.snapshot.packageName,
|
|
111
170
|
});
|
|
112
171
|
const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
113
|
-
const metaPath = `${snapshotPath}.json`;
|
|
114
172
|
if (data.options.verbose)
|
|
115
173
|
(0, cli_util_1.logExec)(`Deleting ${snapshotPath}`);
|
|
116
174
|
if (await (0, fs_util_1.checkDir)(snapshotPath))
|
|
117
175
|
await (0, promises_1.rm)(snapshotPath, {
|
|
118
176
|
recursive: true,
|
|
119
177
|
});
|
|
120
|
-
if (await (0, fs_util_1.checkFile)(metaPath))
|
|
121
|
-
await (0, promises_1.rm)(metaPath);
|
|
122
178
|
}
|
|
123
179
|
async onSnapshots(data) {
|
|
124
180
|
if (!(await (0, fs_util_1.checkDir)(this.config.outPath)))
|
|
@@ -137,7 +193,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
137
193
|
if (data.options.ids &&
|
|
138
194
|
!data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
|
|
139
195
|
continue;
|
|
140
|
-
const metaPath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
196
|
+
const metaPath = (0, path_1.join)(this.config.outPath, snapshotName, "meta.json");
|
|
141
197
|
const meta = await DatatruckRepository.parseMetaData(metaPath);
|
|
142
198
|
if (taskPatterns && !(0, string_util_1.checkMatch)(meta.task, taskPatterns))
|
|
143
199
|
continue;
|
|
@@ -199,34 +255,32 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
199
255
|
verbose: data.options.verbose,
|
|
200
256
|
})
|
|
201
257
|
: undefined;
|
|
202
|
-
const stream = fast_glob_1.default.stream(include, {
|
|
203
|
-
cwd: sourcePath,
|
|
204
|
-
ignore: exclude,
|
|
205
|
-
dot: true,
|
|
206
|
-
onlyFiles: false,
|
|
207
|
-
markDirectories: true,
|
|
208
|
-
stats: true,
|
|
209
|
-
});
|
|
210
258
|
const packs = compress?.packs || [];
|
|
211
259
|
const tmpDir = await (0, fs_util_1.mkTmpDir)("path-lists");
|
|
212
|
-
const
|
|
260
|
+
const unpackedStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "unpacked.txt"));
|
|
213
261
|
const singlePackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "single-pack.txt"));
|
|
214
262
|
const packStreams = Array.from({ length: packs.length }).map((v, i) => (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, `pack-${i}.txt`)));
|
|
215
|
-
|
|
216
|
-
const streams = [nonPackStream, singlePackStream, ...packStreams];
|
|
263
|
+
const streams = [unpackedStream, singlePackStream, ...packStreams];
|
|
217
264
|
if (data.options.verbose)
|
|
218
265
|
(0, cli_util_1.logExec)(`Writing file lists in ${tmpDir}`);
|
|
219
|
-
await
|
|
220
|
-
|
|
266
|
+
const scanner = await this.createFileScanner({
|
|
267
|
+
glob: {
|
|
268
|
+
include,
|
|
269
|
+
cwd: sourcePath,
|
|
270
|
+
ignore: exclude,
|
|
271
|
+
onlyFiles: false,
|
|
272
|
+
},
|
|
273
|
+
onProgress: data.onProgress,
|
|
274
|
+
disableCounting: true,
|
|
221
275
|
});
|
|
222
276
|
await Promise.all([
|
|
223
277
|
...streams.map((p) => (0, fs_util_1.waitForClose)(p)),
|
|
224
278
|
(async () => {
|
|
225
|
-
|
|
279
|
+
await scanner.start(async (entry) => {
|
|
226
280
|
const pathSubject = entry.stats.isDirectory()
|
|
227
281
|
? entry.path.slice(0, -1)
|
|
228
282
|
: entry.path;
|
|
229
|
-
let stream =
|
|
283
|
+
let stream = unpackedStream;
|
|
230
284
|
let successPackIndex;
|
|
231
285
|
for (const [packIndex, pack] of packs.entries()) {
|
|
232
286
|
if ((0, string_util_1.checkPath)(pathSubject, pack.include, pack.exclude)) {
|
|
@@ -237,60 +291,48 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
237
291
|
break;
|
|
238
292
|
}
|
|
239
293
|
}
|
|
240
|
-
const
|
|
241
|
-
const isPackStream = stream !==
|
|
294
|
+
const isUnpackedStream = stream === unpackedStream;
|
|
295
|
+
const isPackStream = stream !== unpackedStream;
|
|
242
296
|
const isSinglePackStream = stream === singlePackStream;
|
|
243
297
|
const include = isPackStream
|
|
244
298
|
? entry.stats.isDirectory()
|
|
245
|
-
? await (0, fs_util_1.isEmptyDir)(entry.path)
|
|
299
|
+
? await (0, fs_util_1.isEmptyDir)((0, path_1.join)(sourcePath, entry.path))
|
|
246
300
|
: true
|
|
247
301
|
: true;
|
|
248
302
|
if (include) {
|
|
249
303
|
let value = entry.path;
|
|
250
|
-
if (
|
|
304
|
+
if (isUnpackedStream) {
|
|
251
305
|
value += `:${entry.stats.uid}:${entry.stats.gid}:${entry.stats.mode}`;
|
|
252
306
|
}
|
|
253
307
|
else if (isSinglePackStream) {
|
|
254
308
|
value += `:${successPackIndex}`;
|
|
255
309
|
}
|
|
256
310
|
if (!entry.stats.isDirectory())
|
|
257
|
-
|
|
311
|
+
scanner.total++;
|
|
258
312
|
stream.write(`${value}\n`);
|
|
259
313
|
}
|
|
260
|
-
}
|
|
314
|
+
});
|
|
261
315
|
for (const stream of streams) {
|
|
262
316
|
stream.end();
|
|
263
317
|
}
|
|
264
318
|
})(),
|
|
265
319
|
]);
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
await (0, promises_1.mkdir)(dttPath);
|
|
320
|
+
const unpackedPath = (0, path_1.join)(outPath, "unpacked");
|
|
321
|
+
await (0, promises_1.mkdir)(unpackedPath);
|
|
322
|
+
await (0, promises_1.copyFile)(unpackedStream.path, (0, path_1.join)(outPath, "permissions.txt"));
|
|
270
323
|
// Non pack
|
|
271
324
|
if (data.options.verbose)
|
|
272
|
-
(0, cli_util_1.logExec)(`Copying files to ${
|
|
273
|
-
await (0, promises_1.copyFile)(nonPackStream.path, (0, path_1.join)(dttPath, "permissions.txt"));
|
|
325
|
+
(0, cli_util_1.logExec)(`Copying files from ${unpackedStream.path.toString()} to ${unpackedPath}`);
|
|
274
326
|
await (0, fs_util_1.cpy)({
|
|
275
327
|
input: {
|
|
276
328
|
type: "pathList",
|
|
277
|
-
path:
|
|
329
|
+
path: unpackedStream.path.toString(),
|
|
278
330
|
basePath: sourcePath,
|
|
279
331
|
},
|
|
280
|
-
targetPath:
|
|
332
|
+
targetPath: unpackedPath,
|
|
281
333
|
skipNotFoundError: true,
|
|
282
334
|
concurrency: this.config.fileCopyConcurrency,
|
|
283
|
-
async
|
|
284
|
-
if (isDir)
|
|
285
|
-
return;
|
|
286
|
-
currentFiles++;
|
|
287
|
-
await data.onProgress({
|
|
288
|
-
total: totalFiles,
|
|
289
|
-
current: currentFiles,
|
|
290
|
-
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles),
|
|
291
|
-
step: entryPath,
|
|
292
|
-
});
|
|
293
|
-
},
|
|
335
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
|
|
294
336
|
});
|
|
295
337
|
// Single pack
|
|
296
338
|
const singleReader = (0, readline_1.createInterface)({
|
|
@@ -304,48 +346,33 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
304
346
|
const outBasename = (`pack${pack.name ? `-${encodeURIComponent(pack.name)}` : ""}` +
|
|
305
347
|
`-${encodeURIComponent(packPath.replace(/[\\/]/g, "-"))}` +
|
|
306
348
|
`.zip`).slice(0, 255);
|
|
307
|
-
const target = (0, path_1.join)(
|
|
308
|
-
|
|
349
|
+
const target = (0, path_1.join)(outPath, outBasename);
|
|
350
|
+
await (0, zip_util_1.zip)({
|
|
309
351
|
path: pkg.path,
|
|
310
352
|
output: target,
|
|
311
353
|
filter: [{ patterns: [packPath] }],
|
|
312
354
|
verbose: data.options.verbose,
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
total: totalFiles,
|
|
317
|
-
current: currentFiles + stream.data.files,
|
|
318
|
-
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
|
|
319
|
-
step: stream.data.path,
|
|
320
|
-
stepPercent: stream.data.progress,
|
|
321
|
-
});
|
|
322
|
-
},
|
|
355
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "start"
|
|
356
|
+
? "Starting compressing"
|
|
357
|
+
: "Compressing file", progress),
|
|
323
358
|
});
|
|
324
|
-
currentFiles += stats.files;
|
|
325
359
|
}
|
|
326
360
|
// Packs
|
|
327
361
|
for (const [packIndex, packStream] of packStreams.entries()) {
|
|
328
362
|
const pack = packs[packIndex];
|
|
329
|
-
const target = (0, path_1.join)(
|
|
330
|
-
|
|
363
|
+
const target = (0, path_1.join)(outPath, `pack-${packIndex}${pack.name ? `-${pack.name}` : ""}.zip`);
|
|
364
|
+
await (0, zip_util_1.zip)({
|
|
331
365
|
path: sourcePath,
|
|
332
366
|
output: target,
|
|
333
367
|
includeList: packStream.path.toString(),
|
|
334
368
|
verbose: data.options.verbose,
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
total: totalFiles,
|
|
339
|
-
current: currentFiles + stream.data.files,
|
|
340
|
-
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
|
|
341
|
-
step: stream.data.path,
|
|
342
|
-
});
|
|
343
|
-
},
|
|
369
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "start"
|
|
370
|
+
? "Starting compressing"
|
|
371
|
+
: "Compressing file", progress),
|
|
344
372
|
});
|
|
345
|
-
currentFiles += stats.files;
|
|
346
373
|
}
|
|
347
374
|
// Meta
|
|
348
|
-
const metaPath = `${outPath}.json`;
|
|
375
|
+
const metaPath = `${outPath}/meta.json`;
|
|
349
376
|
const nodePkg = (0, fs_util_1.parsePackageFile)();
|
|
350
377
|
const meta = {
|
|
351
378
|
id: data.snapshot.id,
|
|
@@ -354,8 +381,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
354
381
|
package: data.package.name,
|
|
355
382
|
task: data.package.task?.name,
|
|
356
383
|
version: nodePkg.version,
|
|
357
|
-
size:
|
|
358
|
-
(await (0, fs_util_1.fastFolderSizeAsync)(dttPath)),
|
|
384
|
+
size: await (0, fs_util_1.fastFolderSizeAsync)(outPath),
|
|
359
385
|
};
|
|
360
386
|
if (data.options.verbose)
|
|
361
387
|
(0, cli_util_1.logExec)(`Writing metadata into ${metaPath}`);
|
|
@@ -369,19 +395,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
369
395
|
});
|
|
370
396
|
const sourcePath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
|
|
371
397
|
const targetPath = (0, path_1.resolve)((0, path_1.join)(data.mirrorRepositoryConfig.outPath, snapshotName));
|
|
372
|
-
const sourceMetaPath = `${sourcePath}.json`;
|
|
373
|
-
const targetMetaPath = `${targetPath}.json`;
|
|
374
398
|
if (data.options.verbose)
|
|
375
399
|
(0, cli_util_1.logExec)(`Copying backup files to ${targetPath}`);
|
|
376
400
|
await (0, promises_1.mkdir)(targetPath);
|
|
401
|
+
const scanner = await this.createFileScanner({
|
|
402
|
+
glob: {
|
|
403
|
+
include: ["**/*"],
|
|
404
|
+
cwd: sourcePath,
|
|
405
|
+
},
|
|
406
|
+
onProgress: data.onProgress,
|
|
407
|
+
});
|
|
408
|
+
await scanner.start();
|
|
377
409
|
await (0, fs_util_1.cpy)({
|
|
378
410
|
input: {
|
|
379
411
|
type: "glob",
|
|
380
412
|
sourcePath,
|
|
381
413
|
},
|
|
382
414
|
targetPath,
|
|
415
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
|
|
383
416
|
});
|
|
384
|
-
await (0, promises_1.copyFile)(sourceMetaPath, targetMetaPath);
|
|
385
417
|
}
|
|
386
418
|
async onRestore(data) {
|
|
387
419
|
const relRestorePath = data.targetPath ?? data.package.restorePath;
|
|
@@ -400,99 +432,71 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
400
432
|
packageName: data.package.name,
|
|
401
433
|
});
|
|
402
434
|
const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
435
|
+
const scanner = await this.createFileScanner({
|
|
436
|
+
glob: {
|
|
437
|
+
include: ["unpacked/**/*"],
|
|
438
|
+
cwd: sourcePath,
|
|
439
|
+
},
|
|
440
|
+
onProgress: data.onProgress,
|
|
441
|
+
disableEndProgress: true,
|
|
407
442
|
});
|
|
408
|
-
|
|
409
|
-
const
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
});
|
|
428
|
-
}
|
|
443
|
+
await scanner.start();
|
|
444
|
+
const it = await (0, promises_1.opendir)(sourcePath);
|
|
445
|
+
for await (const dirent of it) {
|
|
446
|
+
const path = (0, path_1.join)(sourcePath, dirent.name);
|
|
447
|
+
if (dirent.name === "permissions.txt") {
|
|
448
|
+
scanner.total++;
|
|
449
|
+
await scanner.updateProgress();
|
|
450
|
+
}
|
|
451
|
+
else if (dirent.name.endsWith(".zip")) {
|
|
452
|
+
await (0, zip_util_1.listZip)({
|
|
453
|
+
path,
|
|
454
|
+
verbose: data.options.verbose,
|
|
455
|
+
onStream: async (item) => {
|
|
456
|
+
const isDir = item.Folder === "+";
|
|
457
|
+
if (!isDir)
|
|
458
|
+
scanner.total++;
|
|
459
|
+
await scanner.updateProgress();
|
|
460
|
+
},
|
|
461
|
+
});
|
|
429
462
|
}
|
|
430
463
|
}
|
|
431
|
-
|
|
432
|
-
cwd: sourcePath,
|
|
433
|
-
ignore: [dttFolder],
|
|
434
|
-
dot: true,
|
|
435
|
-
onlyFiles: true,
|
|
436
|
-
});
|
|
437
|
-
for await (const _file of allFiles) {
|
|
438
|
-
totalFiles++;
|
|
439
|
-
}
|
|
464
|
+
await scanner.updateProgress(true);
|
|
440
465
|
if (data.options.verbose)
|
|
441
466
|
(0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
|
|
442
467
|
await (0, fs_util_1.cpy)({
|
|
443
468
|
input: {
|
|
444
469
|
type: "glob",
|
|
445
|
-
sourcePath,
|
|
446
|
-
exclude: [dttFolder],
|
|
470
|
+
sourcePath: (0, path_1.join)(sourcePath, "unpacked"),
|
|
447
471
|
},
|
|
448
472
|
targetPath: restorePath,
|
|
449
473
|
concurrency: this.config.fileCopyConcurrency,
|
|
450
|
-
|
|
451
|
-
if (!isDir) {
|
|
452
|
-
currentFiles++;
|
|
453
|
-
await data.onProgress({
|
|
454
|
-
total: totalFiles,
|
|
455
|
-
current: Math.max(currentFiles, 0),
|
|
456
|
-
percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
|
|
457
|
-
step: entryPath,
|
|
458
|
-
});
|
|
459
|
-
}
|
|
460
|
-
},
|
|
474
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
|
|
461
475
|
});
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
if (
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
current: currentFiles + stream.data.files,
|
|
487
|
-
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
|
|
488
|
-
step: stream.type === "progress"
|
|
489
|
-
? `Extracting ${stream.data.path}`
|
|
490
|
-
: "",
|
|
491
|
-
stepPercent: stream.data.progress,
|
|
492
|
-
});
|
|
493
|
-
},
|
|
494
|
-
});
|
|
495
|
-
}
|
|
476
|
+
const it2 = await (0, promises_1.opendir)(sourcePath);
|
|
477
|
+
for await (const dirent of it2) {
|
|
478
|
+
const path = (0, path_1.join)(sourcePath, dirent.name);
|
|
479
|
+
if (dirent.name === "permissions.txt") {
|
|
480
|
+
if (data.options.verbose)
|
|
481
|
+
(0, cli_util_1.logExec)(`Applying permissions (${path})`);
|
|
482
|
+
await scanner.progress("Applying permissions", {
|
|
483
|
+
current: 0,
|
|
484
|
+
});
|
|
485
|
+
await (0, fs_util_1.applyPermissions)(restorePath, path);
|
|
486
|
+
await scanner.progress("Permissions applied", {
|
|
487
|
+
current: 1,
|
|
488
|
+
type: "end",
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
else if (dirent.name.endsWith(".zip")) {
|
|
492
|
+
await (0, zip_util_1.unzip)({
|
|
493
|
+
input: path,
|
|
494
|
+
output: restorePath,
|
|
495
|
+
verbose: data.options.verbose,
|
|
496
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "start"
|
|
497
|
+
? "Starting extracting"
|
|
498
|
+
: "Extracting file", progress),
|
|
499
|
+
});
|
|
496
500
|
}
|
|
497
501
|
}
|
|
498
502
|
}
|
|
@@ -4,6 +4,7 @@ import type { RestoreActionOptionsType } from "../Action/RestoreAction";
|
|
|
4
4
|
import type { SnapshotExtendedType, SnapshotsActionOptionsType } from "../Action/SnapshotsAction";
|
|
5
5
|
import type { PackageConfigType } from "../Config/PackageConfig";
|
|
6
6
|
import type { RepositoryConfigType } from "../Config/RepositoryConfig";
|
|
7
|
+
import { Progress } from "../util/progress";
|
|
7
8
|
export declare type SnapshotType = {
|
|
8
9
|
id: string;
|
|
9
10
|
date: string;
|
|
@@ -15,13 +16,6 @@ export declare type SnapshotResultType = SnapshotType & {
|
|
|
15
16
|
tags: string[];
|
|
16
17
|
size: number;
|
|
17
18
|
};
|
|
18
|
-
export declare type ProgressDataType = {
|
|
19
|
-
total?: number;
|
|
20
|
-
current?: number;
|
|
21
|
-
percent?: number;
|
|
22
|
-
step?: string;
|
|
23
|
-
stepPercent?: number | null;
|
|
24
|
-
};
|
|
25
19
|
export declare type InitDataType = {
|
|
26
20
|
options: InitActionOptionsType;
|
|
27
21
|
};
|
|
@@ -33,7 +27,7 @@ export declare type CopyBackupType<TRepositoryConfig> = {
|
|
|
33
27
|
snapshot: SnapshotType;
|
|
34
28
|
package: PackageConfigType;
|
|
35
29
|
mirrorRepositoryConfig: TRepositoryConfig;
|
|
36
|
-
onProgress: (data:
|
|
30
|
+
onProgress: (data: Progress) => Promise<void>;
|
|
37
31
|
};
|
|
38
32
|
export declare type BackupDataType<TPackageConfig> = {
|
|
39
33
|
options: BackupActionOptionsType;
|
|
@@ -41,7 +35,7 @@ export declare type BackupDataType<TPackageConfig> = {
|
|
|
41
35
|
package: PackageConfigType;
|
|
42
36
|
targetPath: string | undefined;
|
|
43
37
|
packageConfig: TPackageConfig | undefined;
|
|
44
|
-
onProgress: (data:
|
|
38
|
+
onProgress: (data: Progress) => Promise<void>;
|
|
45
39
|
};
|
|
46
40
|
export declare type RestoreDataType<TPackageConfig> = {
|
|
47
41
|
options: RestoreActionOptionsType;
|
|
@@ -49,7 +43,7 @@ export declare type RestoreDataType<TPackageConfig> = {
|
|
|
49
43
|
package: PackageConfigType;
|
|
50
44
|
targetPath: string | undefined;
|
|
51
45
|
packageConfig: TPackageConfig;
|
|
52
|
-
onProgress: (data:
|
|
46
|
+
onProgress: (data: Progress) => Promise<void>;
|
|
53
47
|
};
|
|
54
48
|
export declare type PruneDataType = {
|
|
55
49
|
snapshot: SnapshotExtendedType;
|
|
@@ -195,7 +195,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
195
195
|
verbose: data.options.verbose,
|
|
196
196
|
});
|
|
197
197
|
await data.onProgress({
|
|
198
|
-
|
|
198
|
+
relative: {
|
|
199
|
+
description: "Writing excluded paths list",
|
|
200
|
+
},
|
|
199
201
|
});
|
|
200
202
|
const tmpDir = await (0, fs_util_1.mkTmpDir)("restic-exclude");
|
|
201
203
|
const ignoredContents = (0, fs_util_1.fastglobToGitIgnore)(exclude, sourcePath).join("\n");
|
|
@@ -223,7 +225,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
223
225
|
if (data.options.verbose)
|
|
224
226
|
(0, cli_util_1.logExec)(`Writing paths lists`);
|
|
225
227
|
await data.onProgress({
|
|
226
|
-
|
|
228
|
+
relative: {
|
|
229
|
+
description: "Writing excluded paths list",
|
|
230
|
+
},
|
|
227
231
|
});
|
|
228
232
|
gitignorePath = await (0, fs_util_1.writeGitIgnoreList)({
|
|
229
233
|
paths: stream,
|
|
@@ -233,7 +237,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
233
237
|
throw new AppError_1.AppError(`Tag prefix is not allowed`);
|
|
234
238
|
const packageTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.PACKAGE, data.package.name);
|
|
235
239
|
await data.onProgress({
|
|
236
|
-
|
|
240
|
+
relative: {
|
|
241
|
+
description: "Fetching last snapshot",
|
|
242
|
+
},
|
|
237
243
|
});
|
|
238
244
|
const [lastSnapshot] = await restic.snapshots({
|
|
239
245
|
json: true,
|
|
@@ -245,7 +251,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
245
251
|
let totalFilesChanges = 0;
|
|
246
252
|
const totalFilesChangesLimit = 10;
|
|
247
253
|
await data.onProgress({
|
|
248
|
-
|
|
254
|
+
relative: {
|
|
255
|
+
description: "Executing backup action",
|
|
256
|
+
},
|
|
249
257
|
});
|
|
250
258
|
let resticSnapshotId;
|
|
251
259
|
let resticTotalBytes;
|
|
@@ -278,19 +286,24 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
278
286
|
if (totalFilesChanges > totalFilesChangesLimit) {
|
|
279
287
|
showProgressBar = true;
|
|
280
288
|
}
|
|
281
|
-
else if (lastProgress?.total !== streamData.total_files) {
|
|
289
|
+
else if (lastProgress?.absolute?.total !== streamData.total_files) {
|
|
282
290
|
totalFilesChanges = 0;
|
|
283
291
|
}
|
|
284
292
|
else {
|
|
285
293
|
totalFilesChanges++;
|
|
286
294
|
}
|
|
287
295
|
await data.onProgress((lastProgress = {
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
296
|
+
relative: {
|
|
297
|
+
description: "Copying file",
|
|
298
|
+
payload: streamData.current_files?.join(", ") ?? "-",
|
|
299
|
+
},
|
|
300
|
+
absolute: {
|
|
301
|
+
total: Math.max(lastProgress?.absolute?.total || 0, streamData.total_files || 0),
|
|
302
|
+
current: Math.max(lastProgress?.absolute?.current || 0, streamData.files_done ?? 0),
|
|
303
|
+
percent: showProgressBar
|
|
304
|
+
? Number((streamData.percent_done * 100).toFixed(2))
|
|
305
|
+
: 0,
|
|
306
|
+
},
|
|
294
307
|
}));
|
|
295
308
|
}
|
|
296
309
|
else if (streamData.message_type === "summary") {
|
|
@@ -306,9 +319,11 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
306
319
|
const sizeTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.SIZE, resticTotalBytes.toString());
|
|
307
320
|
await restic.exec(["tag", "--add", sizeTag, resticSnapshotId]);
|
|
308
321
|
await data.onProgress({
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
322
|
+
absolute: {
|
|
323
|
+
total: lastProgress?.absolute?.total || 0,
|
|
324
|
+
current: lastProgress?.absolute?.total || 0,
|
|
325
|
+
percent: 100,
|
|
326
|
+
},
|
|
312
327
|
});
|
|
313
328
|
}
|
|
314
329
|
async onCopyBackup(data) {
|
|
@@ -357,9 +372,11 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
357
372
|
if (streamData.message_type === "restore-status") {
|
|
358
373
|
const current = Math.min(streamData.total_bytes, snapshot.size);
|
|
359
374
|
await data.onProgress({
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
375
|
+
absolute: {
|
|
376
|
+
total: snapshot.size,
|
|
377
|
+
current,
|
|
378
|
+
percent: (0, math_util_1.progressPercent)(snapshot.size, current),
|
|
379
|
+
},
|
|
363
380
|
});
|
|
364
381
|
}
|
|
365
382
|
},
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
+
import { Progress } from "../util/progress";
|
|
2
3
|
import { WriteDataType, ReadResultType, SessionDriverAbstract, SessionDriverOptions } from "./SessionDriverAbstract";
|
|
3
4
|
declare type BadgeType = {
|
|
4
5
|
name: string;
|
|
@@ -12,20 +13,14 @@ declare type MessageType = {
|
|
|
12
13
|
text?: string;
|
|
13
14
|
badges: BadgeType[];
|
|
14
15
|
errorBadge?: BadgeType;
|
|
15
|
-
|
|
16
|
-
progressTotal?: number | null;
|
|
17
|
-
progressPercent?: number | null;
|
|
18
|
-
progressStep?: string | null;
|
|
19
|
-
progressStepPercent?: number | null;
|
|
16
|
+
progress?: Progress;
|
|
20
17
|
};
|
|
21
18
|
declare type ConsoleSessionDriverOptions = SessionDriverOptions & {
|
|
22
19
|
progress?: "auto" | "tty" | "plain";
|
|
23
|
-
progressInterval?: number;
|
|
24
20
|
};
|
|
25
21
|
export declare class ConsoleSessionDriver extends SessionDriverAbstract<ConsoleSessionDriverOptions> {
|
|
26
22
|
protected lastMessage: MessageType | undefined;
|
|
27
23
|
protected lastMessageText: string | undefined;
|
|
28
|
-
protected lastProgressDate: number | undefined;
|
|
29
24
|
protected prints: number;
|
|
30
25
|
protected renderInterval: NodeJS.Timeout;
|
|
31
26
|
protected rendering?: boolean;
|