@datatruck/cli 0.12.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Command/BackupCommand.js +2 -0
- package/Command/CommandAbstract.d.ts +2 -0
- package/Command/RestoreCommand.js +2 -0
- package/Command/SnapshotsCommand.js +6 -0
- package/Config/PackageRepositoryConfig.d.ts +3 -3
- package/Config/PackageRepositoryConfig.js +2 -2
- package/Config/RepositoryConfig.d.ts +3 -3
- package/Config/RepositoryConfig.js +2 -2
- package/Factory/RepositoryFactory.js +3 -3
- package/JsonSchema/DefinitionEnum.d.ts +2 -2
- package/JsonSchema/DefinitionEnum.js +2 -2
- package/JsonSchema/JsonSchema.js +3 -3
- package/Repository/{LocalRepository.d.ts → DatatruckRepository.d.ts} +11 -9
- package/Repository/{LocalRepository.js → DatatruckRepository.js} +215 -102
- package/Repository/GitRepository.js +3 -0
- package/Repository/RepositoryAbstract.d.ts +4 -1
- package/Repository/RepositoryAbstract.js +1 -0
- package/Repository/ResticRepository.js +25 -1
- package/SessionDriver/ConsoleSessionDriver.d.ts +8 -2
- package/SessionDriver/ConsoleSessionDriver.js +18 -10
- package/SessionDriver/SessionDriverAbstract.d.ts +6 -7
- package/Task/GitTask.js +1 -0
- package/Task/SqlDumpTaskAbstract.js +1 -0
- package/cli.js +2 -0
- package/config.schema.json +10 -6
- package/package.json +3 -1
- package/util/ResticUtil.d.ts +9 -2
- package/util/ResticUtil.js +47 -20
- package/util/datatruck/config-util.d.ts +6 -6
- package/util/fs-util.d.ts +18 -20
- package/util/fs-util.js +86 -101
- package/util/math-util.js +2 -0
- package/util/process-util.d.ts +1 -0
- package/util/process-util.js +23 -5
- package/util/string-util.d.ts +1 -0
- package/util/string-util.js +8 -1
- package/util/zip-util.d.ts +45 -18
- package/util/zip-util.js +98 -52
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.
|
|
6
|
+
exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exports.datatruckRepositoryDefinition = exports.datatruckRepositoryName = void 0;
|
|
7
7
|
const AppError_1 = require("../Error/AppError");
|
|
8
8
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
9
9
|
const cli_util_1 = require("../util/cli-util");
|
|
@@ -20,8 +20,8 @@ const promises_1 = require("fs/promises");
|
|
|
20
20
|
const micromatch_1 = require("micromatch");
|
|
21
21
|
const path_1 = require("path");
|
|
22
22
|
const readline_1 = require("readline");
|
|
23
|
-
exports.
|
|
24
|
-
exports.
|
|
23
|
+
exports.datatruckRepositoryName = "datatruck";
|
|
24
|
+
exports.datatruckRepositoryDefinition = {
|
|
25
25
|
type: "object",
|
|
26
26
|
required: ["outPath"],
|
|
27
27
|
additionalProperties: false,
|
|
@@ -30,7 +30,7 @@ exports.localRepositoryDefinition = {
|
|
|
30
30
|
compress: { type: "boolean" },
|
|
31
31
|
},
|
|
32
32
|
};
|
|
33
|
-
exports.
|
|
33
|
+
exports.datatruckPackageRepositoryDefinition = {
|
|
34
34
|
type: "object",
|
|
35
35
|
additionalProperties: false,
|
|
36
36
|
properties: {
|
|
@@ -50,6 +50,7 @@ exports.localPackageRepositoryDefinition = {
|
|
|
50
50
|
additionalProperties: false,
|
|
51
51
|
required: ["include"],
|
|
52
52
|
properties: {
|
|
53
|
+
name: { type: "string" },
|
|
53
54
|
include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
|
|
54
55
|
exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
|
|
55
56
|
onePackByResult: { type: "boolean" },
|
|
@@ -66,7 +67,7 @@ exports.localPackageRepositoryDefinition = {
|
|
|
66
67
|
},
|
|
67
68
|
},
|
|
68
69
|
};
|
|
69
|
-
class
|
|
70
|
+
class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
70
71
|
static buildSnapshotName(data) {
|
|
71
72
|
const date = data.snapshotDate.replace(/:/g, "-");
|
|
72
73
|
const pkgName = encodeURIComponent(data.packageName).replace(/%40/g, "@");
|
|
@@ -103,7 +104,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
103
104
|
await (0, fs_util_1.mkdirIfNotExists)(this.config.outPath);
|
|
104
105
|
}
|
|
105
106
|
async onPrune(data) {
|
|
106
|
-
const snapshotName =
|
|
107
|
+
const snapshotName = DatatruckRepository.buildSnapshotName({
|
|
107
108
|
snapshotId: data.snapshot.id,
|
|
108
109
|
snapshotDate: data.snapshot.date,
|
|
109
110
|
packageName: data.snapshot.packageName,
|
|
@@ -127,7 +128,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
127
128
|
const packagePatterns = (0, string_util_1.makePathPatterns)(data.options.packageNames);
|
|
128
129
|
const taskPatterns = (0, string_util_1.makePathPatterns)(data.options.packageTaskNames);
|
|
129
130
|
for (const snapshotName of snapshotNames) {
|
|
130
|
-
const snapshotNameData =
|
|
131
|
+
const snapshotNameData = DatatruckRepository.parseSnapshotName(snapshotName);
|
|
131
132
|
if (!snapshotNameData)
|
|
132
133
|
continue;
|
|
133
134
|
if (packagePatterns &&
|
|
@@ -137,7 +138,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
137
138
|
!data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
|
|
138
139
|
continue;
|
|
139
140
|
const metaPath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
140
|
-
const meta = await
|
|
141
|
+
const meta = await DatatruckRepository.parseMetaData(metaPath);
|
|
141
142
|
if (taskPatterns && !(0, string_util_1.checkMatch)(meta.task, taskPatterns))
|
|
142
143
|
continue;
|
|
143
144
|
if (data.options.ids &&
|
|
@@ -153,13 +154,14 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
153
154
|
packageName: meta.package,
|
|
154
155
|
packageTaskName: meta.task,
|
|
155
156
|
tags: meta.tags,
|
|
157
|
+
size: meta.size || 0,
|
|
156
158
|
});
|
|
157
159
|
}
|
|
158
160
|
return snapshots;
|
|
159
161
|
}
|
|
160
162
|
normalizeCompressConfig(packageConfig) {
|
|
161
163
|
let compress = packageConfig?.compress ?? this.config.compress;
|
|
162
|
-
if (compress === true) {
|
|
164
|
+
if (compress === true || (compress && !Array.isArray(compress.packs))) {
|
|
163
165
|
return {
|
|
164
166
|
packs: [
|
|
165
167
|
{
|
|
@@ -174,7 +176,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
174
176
|
return compress;
|
|
175
177
|
}
|
|
176
178
|
async onBackup(data) {
|
|
177
|
-
const snapshotName =
|
|
179
|
+
const snapshotName = DatatruckRepository.buildSnapshotName({
|
|
178
180
|
snapshotId: data.snapshot.id,
|
|
179
181
|
snapshotDate: data.snapshot.date,
|
|
180
182
|
packageName: data.package.name,
|
|
@@ -201,90 +203,148 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
201
203
|
cwd: sourcePath,
|
|
202
204
|
ignore: exclude,
|
|
203
205
|
dot: true,
|
|
204
|
-
onlyFiles:
|
|
206
|
+
onlyFiles: false,
|
|
205
207
|
markDirectories: true,
|
|
208
|
+
stats: true,
|
|
206
209
|
});
|
|
210
|
+
const packs = compress?.packs || [];
|
|
211
|
+
const tmpDir = await (0, fs_util_1.mkTmpDir)("path-lists");
|
|
212
|
+
const nonPackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "nonpack.txt"));
|
|
213
|
+
const singlePackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "single-pack.txt"));
|
|
214
|
+
const packStreams = Array.from({ length: packs.length }).map((v, i) => (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, `pack-${i}.txt`)));
|
|
215
|
+
let totalFiles = 0;
|
|
216
|
+
const streams = [nonPackStream, singlePackStream, ...packStreams];
|
|
207
217
|
if (data.options.verbose)
|
|
208
|
-
(0, cli_util_1.logExec)(`Writing
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
packs: compress?.packs,
|
|
218
|
+
(0, cli_util_1.logExec)(`Writing file lists in ${tmpDir}`);
|
|
219
|
+
await data.onProgress({
|
|
220
|
+
step: "Writing the file lists...",
|
|
212
221
|
});
|
|
213
|
-
|
|
214
|
-
(0,
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
222
|
+
await Promise.all([
|
|
223
|
+
...streams.map((p) => (0, fs_util_1.waitForClose)(p)),
|
|
224
|
+
(async () => {
|
|
225
|
+
for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
|
|
226
|
+
const pathSubject = entry.stats.isDirectory()
|
|
227
|
+
? entry.path.slice(0, -1)
|
|
228
|
+
: entry.path;
|
|
229
|
+
let stream = nonPackStream;
|
|
230
|
+
let successPackIndex;
|
|
231
|
+
for (const [packIndex, pack] of packs.entries()) {
|
|
232
|
+
if ((0, string_util_1.checkPath)(pathSubject, pack.include, pack.exclude)) {
|
|
233
|
+
stream = pack.onePackByResult
|
|
234
|
+
? singlePackStream
|
|
235
|
+
: packStreams[packIndex];
|
|
236
|
+
successPackIndex = packIndex;
|
|
237
|
+
break;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
const isNonPackStream = stream === nonPackStream;
|
|
241
|
+
const isPackStream = stream !== nonPackStream;
|
|
242
|
+
const isSinglePackStream = stream === singlePackStream;
|
|
243
|
+
const include = isPackStream
|
|
244
|
+
? entry.stats.isDirectory()
|
|
245
|
+
? await (0, fs_util_1.isEmptyDir)(entry.path)
|
|
246
|
+
: true
|
|
247
|
+
: true;
|
|
248
|
+
if (include) {
|
|
249
|
+
let value = entry.path;
|
|
250
|
+
if (isNonPackStream) {
|
|
251
|
+
value += `:${entry.stats.uid}:${entry.stats.gid}:${entry.stats.mode}`;
|
|
252
|
+
}
|
|
253
|
+
else if (isSinglePackStream) {
|
|
254
|
+
value += `:${successPackIndex}`;
|
|
255
|
+
}
|
|
256
|
+
if (!entry.stats.isDirectory())
|
|
257
|
+
totalFiles++;
|
|
258
|
+
stream.write(`${value}\n`);
|
|
244
259
|
}
|
|
245
260
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
const stats = await (0, zip_util_1.zip)({
|
|
249
|
-
path: sourcePath,
|
|
250
|
-
output: target,
|
|
251
|
-
includeList: packsPath,
|
|
252
|
-
excludeList: pathLists.excludedPackPaths[packIndex],
|
|
253
|
-
verbose: data.options.verbose,
|
|
254
|
-
onStream: async (stream) => await data.onProgress({
|
|
255
|
-
total: pathLists.total.all,
|
|
256
|
-
current: currentFiles + stream.data.files,
|
|
257
|
-
percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
|
|
258
|
-
step: stream.type === "progress" ? stream.data.path : "",
|
|
259
|
-
}),
|
|
260
|
-
});
|
|
261
|
-
currentFiles += stats.files;
|
|
261
|
+
for (const stream of streams) {
|
|
262
|
+
stream.end();
|
|
262
263
|
}
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
264
|
+
})(),
|
|
265
|
+
]);
|
|
266
|
+
let currentFiles = 0;
|
|
267
|
+
const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
|
|
268
|
+
const dttPath = (0, path_1.join)(outPath, dttFolder);
|
|
269
|
+
await (0, promises_1.mkdir)(dttPath);
|
|
270
|
+
// Non pack
|
|
266
271
|
if (data.options.verbose)
|
|
267
272
|
(0, cli_util_1.logExec)(`Copying files to ${outPath}`);
|
|
273
|
+
await (0, promises_1.copyFile)(nonPackStream.path, (0, path_1.join)(dttPath, "permissions.txt"));
|
|
268
274
|
await (0, fs_util_1.cpy)({
|
|
269
275
|
input: {
|
|
270
276
|
type: "pathList",
|
|
271
|
-
path:
|
|
277
|
+
path: nonPackStream.path.toString(),
|
|
272
278
|
basePath: sourcePath,
|
|
273
279
|
},
|
|
274
280
|
targetPath: outPath,
|
|
281
|
+
skipNotFoundError: true,
|
|
275
282
|
concurrency: this.config.fileCopyConcurrency,
|
|
276
283
|
async onPath({ isDir, entryPath }) {
|
|
277
284
|
if (isDir)
|
|
278
285
|
return;
|
|
279
286
|
currentFiles++;
|
|
280
287
|
await data.onProgress({
|
|
281
|
-
total:
|
|
288
|
+
total: totalFiles,
|
|
282
289
|
current: currentFiles,
|
|
283
|
-
percent: (0, math_util_1.progressPercent)(
|
|
290
|
+
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles),
|
|
284
291
|
step: entryPath,
|
|
285
292
|
});
|
|
286
293
|
},
|
|
287
294
|
});
|
|
295
|
+
// Single pack
|
|
296
|
+
const singleReader = (0, readline_1.createInterface)({
|
|
297
|
+
input: (0, fs_1.createReadStream)(singlePackStream.path),
|
|
298
|
+
});
|
|
299
|
+
for await (const line of singleReader) {
|
|
300
|
+
let [packPath, packIndex] = line.split(":");
|
|
301
|
+
const pack = packs[packIndex];
|
|
302
|
+
if (packPath.endsWith("/"))
|
|
303
|
+
packPath = packPath.slice(0, -1);
|
|
304
|
+
const outBasename = (`pack${pack.name ? `-${encodeURIComponent(pack.name)}` : ""}` +
|
|
305
|
+
`-${encodeURIComponent(packPath.replace(/[\\/]/g, "-"))}` +
|
|
306
|
+
`.zip`).slice(0, 255);
|
|
307
|
+
const target = (0, path_1.join)(dttPath, outBasename);
|
|
308
|
+
const stats = await (0, zip_util_1.zip)({
|
|
309
|
+
path: pkg.path,
|
|
310
|
+
output: target,
|
|
311
|
+
filter: [{ patterns: [packPath] }],
|
|
312
|
+
verbose: data.options.verbose,
|
|
313
|
+
onStream: async (stream) => {
|
|
314
|
+
if (stream.type === "progress")
|
|
315
|
+
await data.onProgress({
|
|
316
|
+
total: totalFiles,
|
|
317
|
+
current: currentFiles + stream.data.files,
|
|
318
|
+
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
|
|
319
|
+
step: stream.data.path,
|
|
320
|
+
stepPercent: stream.data.progress,
|
|
321
|
+
});
|
|
322
|
+
},
|
|
323
|
+
});
|
|
324
|
+
currentFiles += stats.files;
|
|
325
|
+
}
|
|
326
|
+
// Packs
|
|
327
|
+
for (const [packIndex, packStream] of packStreams.entries()) {
|
|
328
|
+
const pack = packs[packIndex];
|
|
329
|
+
const target = (0, path_1.join)(dttPath, `pack-${packIndex}${pack.name ? `-${pack.name}` : ""}.zip`);
|
|
330
|
+
const stats = await (0, zip_util_1.zip)({
|
|
331
|
+
path: sourcePath,
|
|
332
|
+
output: target,
|
|
333
|
+
includeList: packStream.path.toString(),
|
|
334
|
+
verbose: data.options.verbose,
|
|
335
|
+
onStream: async (stream) => {
|
|
336
|
+
if (stream.type === "progress")
|
|
337
|
+
await data.onProgress({
|
|
338
|
+
total: totalFiles,
|
|
339
|
+
current: currentFiles + stream.data.files,
|
|
340
|
+
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
|
|
341
|
+
step: stream.data.path,
|
|
342
|
+
});
|
|
343
|
+
},
|
|
344
|
+
});
|
|
345
|
+
currentFiles += stats.files;
|
|
346
|
+
}
|
|
347
|
+
// Meta
|
|
288
348
|
const metaPath = `${outPath}.json`;
|
|
289
349
|
const nodePkg = (0, fs_util_1.parsePackageFile)();
|
|
290
350
|
const meta = {
|
|
@@ -294,13 +354,15 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
294
354
|
package: data.package.name,
|
|
295
355
|
task: data.package.task?.name,
|
|
296
356
|
version: nodePkg.version,
|
|
357
|
+
size: (await (0, fs_util_1.fastFolderSizeAsync)(outPath)) -
|
|
358
|
+
(await (0, fs_util_1.fastFolderSizeAsync)(dttPath)),
|
|
297
359
|
};
|
|
298
360
|
if (data.options.verbose)
|
|
299
361
|
(0, cli_util_1.logExec)(`Writing metadata into ${metaPath}`);
|
|
300
|
-
await (0, promises_1.writeFile)(metaPath,
|
|
362
|
+
await (0, promises_1.writeFile)(metaPath, DatatruckRepository.stringifyMetaData(meta));
|
|
301
363
|
}
|
|
302
364
|
async onCopyBackup(data) {
|
|
303
|
-
const snapshotName =
|
|
365
|
+
const snapshotName = DatatruckRepository.buildSnapshotName({
|
|
304
366
|
snapshotId: data.snapshot.id,
|
|
305
367
|
snapshotDate: data.snapshot.date,
|
|
306
368
|
packageName: data.package.name,
|
|
@@ -310,7 +372,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
310
372
|
const sourceMetaPath = `${sourcePath}.json`;
|
|
311
373
|
const targetMetaPath = `${targetPath}.json`;
|
|
312
374
|
if (data.options.verbose)
|
|
313
|
-
(0, cli_util_1.logExec)(`Copying files to ${targetPath}`);
|
|
375
|
+
(0, cli_util_1.logExec)(`Copying backup files to ${targetPath}`);
|
|
314
376
|
await (0, promises_1.mkdir)(targetPath);
|
|
315
377
|
await (0, fs_util_1.cpy)({
|
|
316
378
|
input: {
|
|
@@ -332,57 +394,108 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
332
394
|
});
|
|
333
395
|
if (!snapshot)
|
|
334
396
|
throw new AppError_1.AppError("Snapshot not found");
|
|
335
|
-
const snapshotName =
|
|
397
|
+
const snapshotName = DatatruckRepository.buildSnapshotName({
|
|
336
398
|
snapshotId: data.snapshot.id,
|
|
337
399
|
snapshotDate: data.snapshot.date,
|
|
338
400
|
packageName: data.package.name,
|
|
339
401
|
});
|
|
340
402
|
const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
341
403
|
let totalFiles = 0;
|
|
342
|
-
let currentFiles =
|
|
343
|
-
await
|
|
404
|
+
let currentFiles = 0;
|
|
405
|
+
await data.onProgress({
|
|
406
|
+
step: "Counting files...",
|
|
407
|
+
});
|
|
408
|
+
const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
|
|
409
|
+
const dttPath = (0, path_1.join)(sourcePath, dttFolder);
|
|
410
|
+
const dttPathExists = await (0, fs_util_1.checkDir)(dttPath);
|
|
411
|
+
if (dttPathExists) {
|
|
412
|
+
const it = await (0, promises_1.opendir)(dttPath);
|
|
413
|
+
for await (const dirent of it) {
|
|
414
|
+
const path = (0, path_1.join)(dttPath, dirent.name);
|
|
415
|
+
if (dirent.name === "permissions.txt") {
|
|
416
|
+
totalFiles++;
|
|
417
|
+
}
|
|
418
|
+
else if (dirent.name.endsWith(".zip")) {
|
|
419
|
+
await (0, zip_util_1.listZip)({
|
|
420
|
+
path,
|
|
421
|
+
verbose: data.options.verbose,
|
|
422
|
+
onStream: (item) => {
|
|
423
|
+
const isDir = item.Folder === "+";
|
|
424
|
+
if (!isDir)
|
|
425
|
+
totalFiles++;
|
|
426
|
+
},
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
const allFiles = fast_glob_1.default.stream(["**"], {
|
|
432
|
+
cwd: sourcePath,
|
|
433
|
+
ignore: [dttFolder],
|
|
434
|
+
dot: true,
|
|
435
|
+
onlyFiles: true,
|
|
436
|
+
});
|
|
437
|
+
for await (const _file of allFiles) {
|
|
344
438
|
totalFiles++;
|
|
345
|
-
}
|
|
439
|
+
}
|
|
346
440
|
if (data.options.verbose)
|
|
347
441
|
(0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
|
|
348
442
|
await (0, fs_util_1.cpy)({
|
|
349
443
|
input: {
|
|
350
444
|
type: "glob",
|
|
351
445
|
sourcePath,
|
|
446
|
+
exclude: [dttFolder],
|
|
352
447
|
},
|
|
353
448
|
targetPath: restorePath,
|
|
354
449
|
concurrency: this.config.fileCopyConcurrency,
|
|
355
|
-
onPath: async ({ entryPath,
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
450
|
+
onPath: async ({ entryPath, isDir }) => {
|
|
451
|
+
if (!isDir) {
|
|
452
|
+
currentFiles++;
|
|
453
|
+
await data.onProgress({
|
|
454
|
+
total: totalFiles,
|
|
455
|
+
current: Math.max(currentFiles, 0),
|
|
456
|
+
percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
|
|
457
|
+
step: entryPath,
|
|
458
|
+
});
|
|
459
|
+
}
|
|
460
|
+
},
|
|
461
|
+
});
|
|
462
|
+
if (dttPathExists) {
|
|
463
|
+
const it = await (0, promises_1.opendir)(dttPath);
|
|
464
|
+
for await (const dirent of it) {
|
|
465
|
+
const path = (0, path_1.join)(dttPath, dirent.name);
|
|
466
|
+
if (dirent.name === "permissions.txt") {
|
|
467
|
+
if (data.options.verbose)
|
|
468
|
+
(0, cli_util_1.logExec)(`Applying permissions (${path})`);
|
|
469
|
+
currentFiles++;
|
|
470
|
+
await data.onProgress({
|
|
471
|
+
total: totalFiles,
|
|
472
|
+
current: currentFiles,
|
|
473
|
+
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles),
|
|
474
|
+
step: "Applying permissions",
|
|
475
|
+
});
|
|
476
|
+
await (0, fs_util_1.applyPermissions)(restorePath, path);
|
|
477
|
+
}
|
|
478
|
+
else if (dirent.name.endsWith(".zip")) {
|
|
367
479
|
await (0, zip_util_1.unzip)({
|
|
368
|
-
input:
|
|
480
|
+
input: path,
|
|
369
481
|
output: restorePath,
|
|
370
482
|
verbose: data.options.verbose,
|
|
371
|
-
onStream: async (stream) =>
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
483
|
+
onStream: async (stream) => {
|
|
484
|
+
await data.onProgress({
|
|
485
|
+
total: totalFiles,
|
|
486
|
+
current: currentFiles + stream.data.files,
|
|
487
|
+
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
|
|
488
|
+
step: stream.type === "progress"
|
|
489
|
+
? `Extracting ${stream.data.path}`
|
|
490
|
+
: "",
|
|
491
|
+
stepPercent: stream.data.progress,
|
|
492
|
+
});
|
|
493
|
+
},
|
|
379
494
|
});
|
|
380
495
|
}
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
},
|
|
384
|
-
});
|
|
496
|
+
}
|
|
497
|
+
}
|
|
385
498
|
}
|
|
386
499
|
}
|
|
387
|
-
exports.
|
|
388
|
-
|
|
500
|
+
exports.DatatruckRepository = DatatruckRepository;
|
|
501
|
+
DatatruckRepository.zipBasenameTpl = `.*.dd.zip`;
|
|
@@ -136,6 +136,7 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
136
136
|
packageName: parsedTag.package,
|
|
137
137
|
packageTaskName: parsedTag.task,
|
|
138
138
|
tags: parsedTag.tags,
|
|
139
|
+
size: Number(parsedTag.size) || 0,
|
|
139
140
|
});
|
|
140
141
|
return result;
|
|
141
142
|
}, [])
|
|
@@ -207,6 +208,8 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
207
208
|
package: data.package.name,
|
|
208
209
|
task: data.package.task?.name,
|
|
209
210
|
version: nodePkg.version,
|
|
211
|
+
size: ((await (0, fs_util_1.fastFolderSizeAsync)(tmpPath)) -
|
|
212
|
+
(await (0, fs_util_1.fastFolderSizeAsync)((0, path_1.join)(tmpPath, ".git")))).toString(),
|
|
210
213
|
});
|
|
211
214
|
await git.addTag(meta.name, meta.message);
|
|
212
215
|
await git.push({ branchName });
|
|
@@ -13,6 +13,7 @@ export declare type SnapshotResultType = SnapshotType & {
|
|
|
13
13
|
packageName: string;
|
|
14
14
|
packageTaskName: string | undefined;
|
|
15
15
|
tags: string[];
|
|
16
|
+
size: number;
|
|
16
17
|
};
|
|
17
18
|
export declare type ProgressDataType = {
|
|
18
19
|
total?: number;
|
|
@@ -63,7 +64,8 @@ export declare enum SnapshotTagEnum {
|
|
|
63
64
|
PACKAGE = "package",
|
|
64
65
|
TASK = "task",
|
|
65
66
|
TAGS = "tags",
|
|
66
|
-
VERSION = "version"
|
|
67
|
+
VERSION = "version",
|
|
68
|
+
SIZE = "size"
|
|
67
69
|
}
|
|
68
70
|
export declare type SnapshotTagObjectType = {
|
|
69
71
|
[SnapshotTagEnum.ID]: string;
|
|
@@ -73,6 +75,7 @@ export declare type SnapshotTagObjectType = {
|
|
|
73
75
|
[SnapshotTagEnum.TASK]: string | undefined;
|
|
74
76
|
[SnapshotTagEnum.TAGS]: string[];
|
|
75
77
|
[SnapshotTagEnum.VERSION]: string;
|
|
78
|
+
[SnapshotTagEnum.SIZE]: string;
|
|
76
79
|
};
|
|
77
80
|
export declare abstract class RepositoryAbstract<TConfig> {
|
|
78
81
|
readonly repository: RepositoryConfigType;
|
|
@@ -10,6 +10,7 @@ var SnapshotTagEnum;
|
|
|
10
10
|
SnapshotTagEnum["TASK"] = "task";
|
|
11
11
|
SnapshotTagEnum["TAGS"] = "tags";
|
|
12
12
|
SnapshotTagEnum["VERSION"] = "version";
|
|
13
|
+
SnapshotTagEnum["SIZE"] = "size";
|
|
13
14
|
})(SnapshotTagEnum = exports.SnapshotTagEnum || (exports.SnapshotTagEnum = {}));
|
|
14
15
|
class RepositoryAbstract {
|
|
15
16
|
constructor(repository) {
|
|
@@ -9,6 +9,7 @@ const ResticUtil_1 = require("../util/ResticUtil");
|
|
|
9
9
|
const cli_util_1 = require("../util/cli-util");
|
|
10
10
|
const paths_util_1 = require("../util/datatruck/paths-util");
|
|
11
11
|
const fs_util_1 = require("../util/fs-util");
|
|
12
|
+
const math_util_1 = require("../util/math-util");
|
|
12
13
|
const string_util_1 = require("../util/string-util");
|
|
13
14
|
const RepositoryAbstract_1 = require("./RepositoryAbstract");
|
|
14
15
|
const assert_1 = require("assert");
|
|
@@ -164,6 +165,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
164
165
|
date: tag.date,
|
|
165
166
|
id: tag.id,
|
|
166
167
|
tags: itemTags,
|
|
168
|
+
size: Number(tag.size) || 0,
|
|
167
169
|
});
|
|
168
170
|
return items;
|
|
169
171
|
}, []);
|
|
@@ -245,6 +247,8 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
245
247
|
await data.onProgress({
|
|
246
248
|
step: "Executing backup action...",
|
|
247
249
|
});
|
|
250
|
+
let resticSnapshotId;
|
|
251
|
+
let resticTotalBytes;
|
|
248
252
|
await restic.backup({
|
|
249
253
|
cwd: sourcePath,
|
|
250
254
|
paths: ["."],
|
|
@@ -281,7 +285,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
281
285
|
totalFilesChanges++;
|
|
282
286
|
}
|
|
283
287
|
await data.onProgress((lastProgress = {
|
|
284
|
-
total: Math.max(lastProgress?.total || 0, streamData.total_files),
|
|
288
|
+
total: Math.max(lastProgress?.total || 0, streamData.total_files || 0),
|
|
285
289
|
current: Math.max(lastProgress?.current || 0, streamData.files_done ?? 0),
|
|
286
290
|
percent: showProgressBar
|
|
287
291
|
? Number((streamData.percent_done * 100).toFixed(2))
|
|
@@ -289,8 +293,18 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
289
293
|
step: streamData.current_files?.join(", ") ?? "-",
|
|
290
294
|
}));
|
|
291
295
|
}
|
|
296
|
+
else if (streamData.message_type === "summary") {
|
|
297
|
+
resticSnapshotId = streamData.snapshot_id;
|
|
298
|
+
resticTotalBytes = streamData.total_bytes_processed;
|
|
299
|
+
}
|
|
292
300
|
},
|
|
293
301
|
});
|
|
302
|
+
if (typeof resticSnapshotId !== "string")
|
|
303
|
+
throw new AppError_1.AppError(`Restic snapshot id is is not defined`);
|
|
304
|
+
if (typeof resticTotalBytes !== "number")
|
|
305
|
+
throw new AppError_1.AppError(`Restic snapshot total bytes is not defined`);
|
|
306
|
+
const sizeTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.SIZE, resticTotalBytes.toString());
|
|
307
|
+
await restic.exec(["tag", "--add", sizeTag, resticSnapshotId]);
|
|
294
308
|
await data.onProgress({
|
|
295
309
|
total: lastProgress?.total || 0,
|
|
296
310
|
current: lastProgress?.total || 0,
|
|
@@ -339,6 +353,16 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
339
353
|
await restic.restore({
|
|
340
354
|
id: snapshot.originalId,
|
|
341
355
|
target: restorePath,
|
|
356
|
+
onStream: async (streamData) => {
|
|
357
|
+
if (streamData.message_type === "restore-status") {
|
|
358
|
+
const current = Math.min(streamData.total_bytes, snapshot.size);
|
|
359
|
+
await data.onProgress({
|
|
360
|
+
total: snapshot.size,
|
|
361
|
+
current,
|
|
362
|
+
percent: (0, math_util_1.progressPercent)(snapshot.size, current),
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
},
|
|
342
366
|
});
|
|
343
367
|
}
|
|
344
368
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { WriteDataType, ReadResultType, SessionDriverAbstract } from "./SessionDriverAbstract";
|
|
2
|
+
import { WriteDataType, ReadResultType, SessionDriverAbstract, SessionDriverOptions } from "./SessionDriverAbstract";
|
|
3
3
|
declare type BadgeType = {
|
|
4
4
|
name: string;
|
|
5
5
|
value: string;
|
|
@@ -18,9 +18,14 @@ declare type MessageType = {
|
|
|
18
18
|
progressStep?: string | null;
|
|
19
19
|
progressStepPercent?: number | null;
|
|
20
20
|
};
|
|
21
|
-
|
|
21
|
+
declare type ConsoleSessionDriverOptions = SessionDriverOptions & {
|
|
22
|
+
progress?: "auto" | "tty" | "plain";
|
|
23
|
+
progressInterval?: number;
|
|
24
|
+
};
|
|
25
|
+
export declare class ConsoleSessionDriver extends SessionDriverAbstract<ConsoleSessionDriverOptions> {
|
|
22
26
|
protected lastMessage: MessageType | undefined;
|
|
23
27
|
protected lastMessageText: string | undefined;
|
|
28
|
+
protected lastProgressDate: number | undefined;
|
|
24
29
|
protected prints: number;
|
|
25
30
|
protected renderInterval: NodeJS.Timeout;
|
|
26
31
|
protected rendering?: boolean;
|
|
@@ -30,6 +35,7 @@ export declare class ConsoleSessionDriver extends SessionDriverAbstract {
|
|
|
30
35
|
start: () => number;
|
|
31
36
|
elapsed: (formatted?: boolean | undefined) => string | number;
|
|
32
37
|
};
|
|
38
|
+
protected tty: boolean;
|
|
33
39
|
onInit(): Promise<void>;
|
|
34
40
|
onEnd(data?: Record<string, any>): Promise<void>;
|
|
35
41
|
onRead(): Promise<ReadResultType[]>;
|