@datatruck/cli 0.12.1 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/Command/BackupCommand.js +2 -0
  2. package/Command/CommandAbstract.d.ts +2 -0
  3. package/Command/RestoreCommand.js +2 -0
  4. package/Command/SnapshotsCommand.js +6 -0
  5. package/Config/PackageRepositoryConfig.d.ts +3 -3
  6. package/Config/PackageRepositoryConfig.js +2 -2
  7. package/Config/RepositoryConfig.d.ts +3 -3
  8. package/Config/RepositoryConfig.js +2 -2
  9. package/Factory/RepositoryFactory.js +3 -3
  10. package/JsonSchema/DefinitionEnum.d.ts +2 -2
  11. package/JsonSchema/DefinitionEnum.js +2 -2
  12. package/JsonSchema/JsonSchema.js +3 -3
  13. package/Repository/{LocalRepository.d.ts → DatatruckRepository.d.ts} +11 -9
  14. package/Repository/{LocalRepository.js → DatatruckRepository.js} +215 -102
  15. package/Repository/GitRepository.js +3 -0
  16. package/Repository/RepositoryAbstract.d.ts +4 -1
  17. package/Repository/RepositoryAbstract.js +1 -0
  18. package/Repository/ResticRepository.js +25 -1
  19. package/SessionDriver/ConsoleSessionDriver.d.ts +8 -2
  20. package/SessionDriver/ConsoleSessionDriver.js +18 -10
  21. package/SessionDriver/SessionDriverAbstract.d.ts +6 -7
  22. package/Task/GitTask.js +1 -0
  23. package/Task/SqlDumpTaskAbstract.js +1 -0
  24. package/cli.js +2 -0
  25. package/config.schema.json +10 -6
  26. package/package.json +3 -1
  27. package/util/ResticUtil.d.ts +9 -2
  28. package/util/ResticUtil.js +47 -20
  29. package/util/datatruck/config-util.d.ts +6 -6
  30. package/util/fs-util.d.ts +18 -20
  31. package/util/fs-util.js +86 -101
  32. package/util/math-util.js +2 -0
  33. package/util/process-util.d.ts +1 -0
  34. package/util/process-util.js +23 -5
  35. package/util/string-util.d.ts +1 -0
  36. package/util/string-util.js +8 -1
  37. package/util/zip-util.d.ts +45 -18
  38. package/util/zip-util.js +98 -52
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.LocalRepository = exports.localPackageRepositoryDefinition = exports.localRepositoryDefinition = exports.localRepositoryName = void 0;
6
+ exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exports.datatruckRepositoryDefinition = exports.datatruckRepositoryName = void 0;
7
7
  const AppError_1 = require("../Error/AppError");
8
8
  const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
9
9
  const cli_util_1 = require("../util/cli-util");
@@ -20,8 +20,8 @@ const promises_1 = require("fs/promises");
20
20
  const micromatch_1 = require("micromatch");
21
21
  const path_1 = require("path");
22
22
  const readline_1 = require("readline");
23
- exports.localRepositoryName = "local";
24
- exports.localRepositoryDefinition = {
23
+ exports.datatruckRepositoryName = "datatruck";
24
+ exports.datatruckRepositoryDefinition = {
25
25
  type: "object",
26
26
  required: ["outPath"],
27
27
  additionalProperties: false,
@@ -30,7 +30,7 @@ exports.localRepositoryDefinition = {
30
30
  compress: { type: "boolean" },
31
31
  },
32
32
  };
33
- exports.localPackageRepositoryDefinition = {
33
+ exports.datatruckPackageRepositoryDefinition = {
34
34
  type: "object",
35
35
  additionalProperties: false,
36
36
  properties: {
@@ -50,6 +50,7 @@ exports.localPackageRepositoryDefinition = {
50
50
  additionalProperties: false,
51
51
  required: ["include"],
52
52
  properties: {
53
+ name: { type: "string" },
53
54
  include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
54
55
  exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
55
56
  onePackByResult: { type: "boolean" },
@@ -66,7 +67,7 @@ exports.localPackageRepositoryDefinition = {
66
67
  },
67
68
  },
68
69
  };
69
- class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
70
+ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
70
71
  static buildSnapshotName(data) {
71
72
  const date = data.snapshotDate.replace(/:/g, "-");
72
73
  const pkgName = encodeURIComponent(data.packageName).replace(/%40/g, "@");
@@ -103,7 +104,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
103
104
  await (0, fs_util_1.mkdirIfNotExists)(this.config.outPath);
104
105
  }
105
106
  async onPrune(data) {
106
- const snapshotName = LocalRepository.buildSnapshotName({
107
+ const snapshotName = DatatruckRepository.buildSnapshotName({
107
108
  snapshotId: data.snapshot.id,
108
109
  snapshotDate: data.snapshot.date,
109
110
  packageName: data.snapshot.packageName,
@@ -127,7 +128,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
127
128
  const packagePatterns = (0, string_util_1.makePathPatterns)(data.options.packageNames);
128
129
  const taskPatterns = (0, string_util_1.makePathPatterns)(data.options.packageTaskNames);
129
130
  for (const snapshotName of snapshotNames) {
130
- const snapshotNameData = LocalRepository.parseSnapshotName(snapshotName);
131
+ const snapshotNameData = DatatruckRepository.parseSnapshotName(snapshotName);
131
132
  if (!snapshotNameData)
132
133
  continue;
133
134
  if (packagePatterns &&
@@ -137,7 +138,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
137
138
  !data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
138
139
  continue;
139
140
  const metaPath = (0, path_1.join)(this.config.outPath, snapshotName);
140
- const meta = await LocalRepository.parseMetaData(metaPath);
141
+ const meta = await DatatruckRepository.parseMetaData(metaPath);
141
142
  if (taskPatterns && !(0, string_util_1.checkMatch)(meta.task, taskPatterns))
142
143
  continue;
143
144
  if (data.options.ids &&
@@ -153,13 +154,14 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
153
154
  packageName: meta.package,
154
155
  packageTaskName: meta.task,
155
156
  tags: meta.tags,
157
+ size: meta.size || 0,
156
158
  });
157
159
  }
158
160
  return snapshots;
159
161
  }
160
162
  normalizeCompressConfig(packageConfig) {
161
163
  let compress = packageConfig?.compress ?? this.config.compress;
162
- if (compress === true) {
164
+ if (compress === true || (compress && !Array.isArray(compress.packs))) {
163
165
  return {
164
166
  packs: [
165
167
  {
@@ -174,7 +176,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
174
176
  return compress;
175
177
  }
176
178
  async onBackup(data) {
177
- const snapshotName = LocalRepository.buildSnapshotName({
179
+ const snapshotName = DatatruckRepository.buildSnapshotName({
178
180
  snapshotId: data.snapshot.id,
179
181
  snapshotDate: data.snapshot.date,
180
182
  packageName: data.package.name,
@@ -201,90 +203,148 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
201
203
  cwd: sourcePath,
202
204
  ignore: exclude,
203
205
  dot: true,
204
- onlyFiles: compress ? false : true,
206
+ onlyFiles: false,
205
207
  markDirectories: true,
208
+ stats: true,
206
209
  });
210
+ const packs = compress?.packs || [];
211
+ const tmpDir = await (0, fs_util_1.mkTmpDir)("path-lists");
212
+ const nonPackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "nonpack.txt"));
213
+ const singlePackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "single-pack.txt"));
214
+ const packStreams = Array.from({ length: packs.length }).map((v, i) => (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, `pack-${i}.txt`)));
215
+ let totalFiles = 0;
216
+ const streams = [nonPackStream, singlePackStream, ...packStreams];
207
217
  if (data.options.verbose)
208
- (0, cli_util_1.logExec)(`Writing paths lists`);
209
- const pathLists = await (0, fs_util_1.writePathLists)({
210
- paths: stream,
211
- packs: compress?.packs,
218
+ (0, cli_util_1.logExec)(`Writing file lists in ${tmpDir}`);
219
+ await data.onProgress({
220
+ step: "Writing the file lists...",
212
221
  });
213
- if (data.options.verbose)
214
- (0, cli_util_1.logExec)(`Path lists: ${pathLists.path}`);
215
- let currentFiles = 0;
216
- if (compress?.packs) {
217
- let packIndex = 0;
218
- for (const packsPath of pathLists.includedPackPaths) {
219
- const pack = compress.packs[packIndex];
220
- if (pack.onePackByResult) {
221
- const reader = (0, readline_1.createInterface)({
222
- input: (0, fs_1.createReadStream)(packsPath),
223
- });
224
- let multipleIndex = 0;
225
- for await (let packPath of reader) {
226
- if (packPath.endsWith("/"))
227
- packPath = packPath.slice(0, -1);
228
- const target = (0, path_1.join)(outPath, `.${packIndex}-${multipleIndex++}-${encodeURIComponent(packPath.replace(/[\\/]/g, "-")).slice(0, 255)}.dd.zip`);
229
- const stats = await (0, zip_util_1.zip)({
230
- path: pkg.path,
231
- output: target,
232
- filter: [{ patterns: [packPath] }],
233
- excludeList: pathLists.excludedPackPaths[packIndex],
234
- verbose: data.options.verbose,
235
- onStream: async (stream) => await data.onProgress({
236
- total: pathLists.total.all,
237
- current: currentFiles + stream.data.files,
238
- percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
239
- step: stream.type === "progress" ? stream.data.path : "",
240
- stepPercent: stream.type === "progress" ? stream.data.progress : null,
241
- }),
242
- });
243
- currentFiles += stats.files;
222
+ await Promise.all([
223
+ ...streams.map((p) => (0, fs_util_1.waitForClose)(p)),
224
+ (async () => {
225
+ for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
226
+ const pathSubject = entry.stats.isDirectory()
227
+ ? entry.path.slice(0, -1)
228
+ : entry.path;
229
+ let stream = nonPackStream;
230
+ let successPackIndex;
231
+ for (const [packIndex, pack] of packs.entries()) {
232
+ if ((0, string_util_1.checkPath)(pathSubject, pack.include, pack.exclude)) {
233
+ stream = pack.onePackByResult
234
+ ? singlePackStream
235
+ : packStreams[packIndex];
236
+ successPackIndex = packIndex;
237
+ break;
238
+ }
239
+ }
240
+ const isNonPackStream = stream === nonPackStream;
241
+ const isPackStream = stream !== nonPackStream;
242
+ const isSinglePackStream = stream === singlePackStream;
243
+ const include = isPackStream
244
+ ? entry.stats.isDirectory()
245
+ ? await (0, fs_util_1.isEmptyDir)(entry.path)
246
+ : true
247
+ : true;
248
+ if (include) {
249
+ let value = entry.path;
250
+ if (isNonPackStream) {
251
+ value += `:${entry.stats.uid}:${entry.stats.gid}:${entry.stats.mode}`;
252
+ }
253
+ else if (isSinglePackStream) {
254
+ value += `:${successPackIndex}`;
255
+ }
256
+ if (!entry.stats.isDirectory())
257
+ totalFiles++;
258
+ stream.write(`${value}\n`);
244
259
  }
245
260
  }
246
- else {
247
- const target = (0, path_1.join)(outPath, `.${packIndex}.dd.zip`);
248
- const stats = await (0, zip_util_1.zip)({
249
- path: sourcePath,
250
- output: target,
251
- includeList: packsPath,
252
- excludeList: pathLists.excludedPackPaths[packIndex],
253
- verbose: data.options.verbose,
254
- onStream: async (stream) => await data.onProgress({
255
- total: pathLists.total.all,
256
- current: currentFiles + stream.data.files,
257
- percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
258
- step: stream.type === "progress" ? stream.data.path : "",
259
- }),
260
- });
261
- currentFiles += stats.files;
261
+ for (const stream of streams) {
262
+ stream.end();
262
263
  }
263
- packIndex++;
264
- }
265
- }
264
+ })(),
265
+ ]);
266
+ let currentFiles = 0;
267
+ const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
268
+ const dttPath = (0, path_1.join)(outPath, dttFolder);
269
+ await (0, promises_1.mkdir)(dttPath);
270
+ // Non pack
266
271
  if (data.options.verbose)
267
272
  (0, cli_util_1.logExec)(`Copying files to ${outPath}`);
273
+ await (0, promises_1.copyFile)(nonPackStream.path, (0, path_1.join)(dttPath, "permissions.txt"));
268
274
  await (0, fs_util_1.cpy)({
269
275
  input: {
270
276
  type: "pathList",
271
- path: pathLists.path,
277
+ path: nonPackStream.path.toString(),
272
278
  basePath: sourcePath,
273
279
  },
274
280
  targetPath: outPath,
281
+ skipNotFoundError: true,
275
282
  concurrency: this.config.fileCopyConcurrency,
276
283
  async onPath({ isDir, entryPath }) {
277
284
  if (isDir)
278
285
  return;
279
286
  currentFiles++;
280
287
  await data.onProgress({
281
- total: pathLists.total.all,
288
+ total: totalFiles,
282
289
  current: currentFiles,
283
- percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles),
290
+ percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles),
284
291
  step: entryPath,
285
292
  });
286
293
  },
287
294
  });
295
+ // Single pack
296
+ const singleReader = (0, readline_1.createInterface)({
297
+ input: (0, fs_1.createReadStream)(singlePackStream.path),
298
+ });
299
+ for await (const line of singleReader) {
300
+ let [packPath, packIndex] = line.split(":");
301
+ const pack = packs[packIndex];
302
+ if (packPath.endsWith("/"))
303
+ packPath = packPath.slice(0, -1);
304
+ const outBasename = (`pack${pack.name ? `-${encodeURIComponent(pack.name)}` : ""}` +
305
+ `-${encodeURIComponent(packPath.replace(/[\\/]/g, "-"))}` +
306
+ `.zip`).slice(0, 255);
307
+ const target = (0, path_1.join)(dttPath, outBasename);
308
+ const stats = await (0, zip_util_1.zip)({
309
+ path: pkg.path,
310
+ output: target,
311
+ filter: [{ patterns: [packPath] }],
312
+ verbose: data.options.verbose,
313
+ onStream: async (stream) => {
314
+ if (stream.type === "progress")
315
+ await data.onProgress({
316
+ total: totalFiles,
317
+ current: currentFiles + stream.data.files,
318
+ percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
319
+ step: stream.data.path,
320
+ stepPercent: stream.data.progress,
321
+ });
322
+ },
323
+ });
324
+ currentFiles += stats.files;
325
+ }
326
+ // Packs
327
+ for (const [packIndex, packStream] of packStreams.entries()) {
328
+ const pack = packs[packIndex];
329
+ const target = (0, path_1.join)(dttPath, `pack-${packIndex}${pack.name ? `-${pack.name}` : ""}.zip`);
330
+ const stats = await (0, zip_util_1.zip)({
331
+ path: sourcePath,
332
+ output: target,
333
+ includeList: packStream.path.toString(),
334
+ verbose: data.options.verbose,
335
+ onStream: async (stream) => {
336
+ if (stream.type === "progress")
337
+ await data.onProgress({
338
+ total: totalFiles,
339
+ current: currentFiles + stream.data.files,
340
+ percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
341
+ step: stream.data.path,
342
+ });
343
+ },
344
+ });
345
+ currentFiles += stats.files;
346
+ }
347
+ // Meta
288
348
  const metaPath = `${outPath}.json`;
289
349
  const nodePkg = (0, fs_util_1.parsePackageFile)();
290
350
  const meta = {
@@ -294,13 +354,15 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
294
354
  package: data.package.name,
295
355
  task: data.package.task?.name,
296
356
  version: nodePkg.version,
357
+ size: (await (0, fs_util_1.fastFolderSizeAsync)(outPath)) -
358
+ (await (0, fs_util_1.fastFolderSizeAsync)(dttPath)),
297
359
  };
298
360
  if (data.options.verbose)
299
361
  (0, cli_util_1.logExec)(`Writing metadata into ${metaPath}`);
300
- await (0, promises_1.writeFile)(metaPath, LocalRepository.stringifyMetaData(meta));
362
+ await (0, promises_1.writeFile)(metaPath, DatatruckRepository.stringifyMetaData(meta));
301
363
  }
302
364
  async onCopyBackup(data) {
303
- const snapshotName = LocalRepository.buildSnapshotName({
365
+ const snapshotName = DatatruckRepository.buildSnapshotName({
304
366
  snapshotId: data.snapshot.id,
305
367
  snapshotDate: data.snapshot.date,
306
368
  packageName: data.package.name,
@@ -310,7 +372,7 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
310
372
  const sourceMetaPath = `${sourcePath}.json`;
311
373
  const targetMetaPath = `${targetPath}.json`;
312
374
  if (data.options.verbose)
313
- (0, cli_util_1.logExec)(`Copying files to ${targetPath}`);
375
+ (0, cli_util_1.logExec)(`Copying backup files to ${targetPath}`);
314
376
  await (0, promises_1.mkdir)(targetPath);
315
377
  await (0, fs_util_1.cpy)({
316
378
  input: {
@@ -332,57 +394,108 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
332
394
  });
333
395
  if (!snapshot)
334
396
  throw new AppError_1.AppError("Snapshot not found");
335
- const snapshotName = LocalRepository.buildSnapshotName({
397
+ const snapshotName = DatatruckRepository.buildSnapshotName({
336
398
  snapshotId: data.snapshot.id,
337
399
  snapshotDate: data.snapshot.date,
338
400
  packageName: data.package.name,
339
401
  });
340
402
  const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
341
403
  let totalFiles = 0;
342
- let currentFiles = -1;
343
- await (0, fs_util_1.forEachFile)(sourcePath, () => {
404
+ let currentFiles = 0;
405
+ await data.onProgress({
406
+ step: "Counting files...",
407
+ });
408
+ const dttFolder = `.dtt-${data.snapshot.id.slice(0, 8)}`;
409
+ const dttPath = (0, path_1.join)(sourcePath, dttFolder);
410
+ const dttPathExists = await (0, fs_util_1.checkDir)(dttPath);
411
+ if (dttPathExists) {
412
+ const it = await (0, promises_1.opendir)(dttPath);
413
+ for await (const dirent of it) {
414
+ const path = (0, path_1.join)(dttPath, dirent.name);
415
+ if (dirent.name === "permissions.txt") {
416
+ totalFiles++;
417
+ }
418
+ else if (dirent.name.endsWith(".zip")) {
419
+ await (0, zip_util_1.listZip)({
420
+ path,
421
+ verbose: data.options.verbose,
422
+ onStream: (item) => {
423
+ const isDir = item.Folder === "+";
424
+ if (!isDir)
425
+ totalFiles++;
426
+ },
427
+ });
428
+ }
429
+ }
430
+ }
431
+ const allFiles = fast_glob_1.default.stream(["**"], {
432
+ cwd: sourcePath,
433
+ ignore: [dttFolder],
434
+ dot: true,
435
+ onlyFiles: true,
436
+ });
437
+ for await (const _file of allFiles) {
344
438
  totalFiles++;
345
- }, true);
439
+ }
346
440
  if (data.options.verbose)
347
441
  (0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
348
442
  await (0, fs_util_1.cpy)({
349
443
  input: {
350
444
  type: "glob",
351
445
  sourcePath,
446
+ exclude: [dttFolder],
352
447
  },
353
448
  targetPath: restorePath,
354
449
  concurrency: this.config.fileCopyConcurrency,
355
- onPath: async ({ entryPath, entrySourcePath }) => {
356
- const isRootFile = (0, path_1.basename)(entryPath) === entryPath;
357
- const isZipFile = isRootFile &&
358
- entryPath.startsWith(".") &&
359
- entryPath.endsWith(".dd.zip");
360
- await data.onProgress({
361
- total: totalFiles,
362
- current: Math.max(currentFiles, 0),
363
- percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
364
- step: entryPath,
365
- });
366
- if (isZipFile) {
450
+ onPath: async ({ entryPath, isDir }) => {
451
+ if (!isDir) {
452
+ currentFiles++;
453
+ await data.onProgress({
454
+ total: totalFiles,
455
+ current: Math.max(currentFiles, 0),
456
+ percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
457
+ step: entryPath,
458
+ });
459
+ }
460
+ },
461
+ });
462
+ if (dttPathExists) {
463
+ const it = await (0, promises_1.opendir)(dttPath);
464
+ for await (const dirent of it) {
465
+ const path = (0, path_1.join)(dttPath, dirent.name);
466
+ if (dirent.name === "permissions.txt") {
467
+ if (data.options.verbose)
468
+ (0, cli_util_1.logExec)(`Applying permissions (${path})`);
469
+ currentFiles++;
470
+ await data.onProgress({
471
+ total: totalFiles,
472
+ current: currentFiles,
473
+ percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles),
474
+ step: "Applying permissions",
475
+ });
476
+ await (0, fs_util_1.applyPermissions)(restorePath, path);
477
+ }
478
+ else if (dirent.name.endsWith(".zip")) {
367
479
  await (0, zip_util_1.unzip)({
368
- input: entrySourcePath,
480
+ input: path,
369
481
  output: restorePath,
370
482
  verbose: data.options.verbose,
371
- onStream: async (stream) => await data.onProgress({
372
- total: totalFiles,
373
- current: currentFiles + 1,
374
- percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + 1),
375
- step: stream.type === "progress"
376
- ? `Extracting ${stream.data.path}`
377
- : "",
378
- }),
483
+ onStream: async (stream) => {
484
+ await data.onProgress({
485
+ total: totalFiles,
486
+ current: currentFiles + stream.data.files,
487
+ percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + stream.data.files),
488
+ step: stream.type === "progress"
489
+ ? `Extracting ${stream.data.path}`
490
+ : "",
491
+ stepPercent: stream.data.progress,
492
+ });
493
+ },
379
494
  });
380
495
  }
381
- currentFiles++;
382
- return isZipFile ? false : true;
383
- },
384
- });
496
+ }
497
+ }
385
498
  }
386
499
  }
387
- exports.LocalRepository = LocalRepository;
388
- LocalRepository.zipBasenameTpl = `.*.dd.zip`;
500
+ exports.DatatruckRepository = DatatruckRepository;
501
+ DatatruckRepository.zipBasenameTpl = `.*.dd.zip`;
@@ -136,6 +136,7 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
136
136
  packageName: parsedTag.package,
137
137
  packageTaskName: parsedTag.task,
138
138
  tags: parsedTag.tags,
139
+ size: Number(parsedTag.size) || 0,
139
140
  });
140
141
  return result;
141
142
  }, [])
@@ -207,6 +208,8 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
207
208
  package: data.package.name,
208
209
  task: data.package.task?.name,
209
210
  version: nodePkg.version,
211
+ size: ((await (0, fs_util_1.fastFolderSizeAsync)(tmpPath)) -
212
+ (await (0, fs_util_1.fastFolderSizeAsync)((0, path_1.join)(tmpPath, ".git")))).toString(),
210
213
  });
211
214
  await git.addTag(meta.name, meta.message);
212
215
  await git.push({ branchName });
@@ -13,6 +13,7 @@ export declare type SnapshotResultType = SnapshotType & {
13
13
  packageName: string;
14
14
  packageTaskName: string | undefined;
15
15
  tags: string[];
16
+ size: number;
16
17
  };
17
18
  export declare type ProgressDataType = {
18
19
  total?: number;
@@ -63,7 +64,8 @@ export declare enum SnapshotTagEnum {
63
64
  PACKAGE = "package",
64
65
  TASK = "task",
65
66
  TAGS = "tags",
66
- VERSION = "version"
67
+ VERSION = "version",
68
+ SIZE = "size"
67
69
  }
68
70
  export declare type SnapshotTagObjectType = {
69
71
  [SnapshotTagEnum.ID]: string;
@@ -73,6 +75,7 @@ export declare type SnapshotTagObjectType = {
73
75
  [SnapshotTagEnum.TASK]: string | undefined;
74
76
  [SnapshotTagEnum.TAGS]: string[];
75
77
  [SnapshotTagEnum.VERSION]: string;
78
+ [SnapshotTagEnum.SIZE]: string;
76
79
  };
77
80
  export declare abstract class RepositoryAbstract<TConfig> {
78
81
  readonly repository: RepositoryConfigType;
@@ -10,6 +10,7 @@ var SnapshotTagEnum;
10
10
  SnapshotTagEnum["TASK"] = "task";
11
11
  SnapshotTagEnum["TAGS"] = "tags";
12
12
  SnapshotTagEnum["VERSION"] = "version";
13
+ SnapshotTagEnum["SIZE"] = "size";
13
14
  })(SnapshotTagEnum = exports.SnapshotTagEnum || (exports.SnapshotTagEnum = {}));
14
15
  class RepositoryAbstract {
15
16
  constructor(repository) {
@@ -9,6 +9,7 @@ const ResticUtil_1 = require("../util/ResticUtil");
9
9
  const cli_util_1 = require("../util/cli-util");
10
10
  const paths_util_1 = require("../util/datatruck/paths-util");
11
11
  const fs_util_1 = require("../util/fs-util");
12
+ const math_util_1 = require("../util/math-util");
12
13
  const string_util_1 = require("../util/string-util");
13
14
  const RepositoryAbstract_1 = require("./RepositoryAbstract");
14
15
  const assert_1 = require("assert");
@@ -164,6 +165,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
164
165
  date: tag.date,
165
166
  id: tag.id,
166
167
  tags: itemTags,
168
+ size: Number(tag.size) || 0,
167
169
  });
168
170
  return items;
169
171
  }, []);
@@ -245,6 +247,8 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
245
247
  await data.onProgress({
246
248
  step: "Executing backup action...",
247
249
  });
250
+ let resticSnapshotId;
251
+ let resticTotalBytes;
248
252
  await restic.backup({
249
253
  cwd: sourcePath,
250
254
  paths: ["."],
@@ -281,7 +285,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
281
285
  totalFilesChanges++;
282
286
  }
283
287
  await data.onProgress((lastProgress = {
284
- total: Math.max(lastProgress?.total || 0, streamData.total_files),
288
+ total: Math.max(lastProgress?.total || 0, streamData.total_files || 0),
285
289
  current: Math.max(lastProgress?.current || 0, streamData.files_done ?? 0),
286
290
  percent: showProgressBar
287
291
  ? Number((streamData.percent_done * 100).toFixed(2))
@@ -289,8 +293,18 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
289
293
  step: streamData.current_files?.join(", ") ?? "-",
290
294
  }));
291
295
  }
296
+ else if (streamData.message_type === "summary") {
297
+ resticSnapshotId = streamData.snapshot_id;
298
+ resticTotalBytes = streamData.total_bytes_processed;
299
+ }
292
300
  },
293
301
  });
302
+ if (typeof resticSnapshotId !== "string")
303
+ throw new AppError_1.AppError(`Restic snapshot id is is not defined`);
304
+ if (typeof resticTotalBytes !== "number")
305
+ throw new AppError_1.AppError(`Restic snapshot total bytes is not defined`);
306
+ const sizeTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.SIZE, resticTotalBytes.toString());
307
+ await restic.exec(["tag", "--add", sizeTag, resticSnapshotId]);
294
308
  await data.onProgress({
295
309
  total: lastProgress?.total || 0,
296
310
  current: lastProgress?.total || 0,
@@ -339,6 +353,16 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
339
353
  await restic.restore({
340
354
  id: snapshot.originalId,
341
355
  target: restorePath,
356
+ onStream: async (streamData) => {
357
+ if (streamData.message_type === "restore-status") {
358
+ const current = Math.min(streamData.total_bytes, snapshot.size);
359
+ await data.onProgress({
360
+ total: snapshot.size,
361
+ current,
362
+ percent: (0, math_util_1.progressPercent)(snapshot.size, current),
363
+ });
364
+ }
365
+ },
342
366
  });
343
367
  }
344
368
  }
@@ -1,5 +1,5 @@
1
1
  /// <reference types="node" />
2
- import { WriteDataType, ReadResultType, SessionDriverAbstract } from "./SessionDriverAbstract";
2
+ import { WriteDataType, ReadResultType, SessionDriverAbstract, SessionDriverOptions } from "./SessionDriverAbstract";
3
3
  declare type BadgeType = {
4
4
  name: string;
5
5
  value: string;
@@ -18,9 +18,14 @@ declare type MessageType = {
18
18
  progressStep?: string | null;
19
19
  progressStepPercent?: number | null;
20
20
  };
21
- export declare class ConsoleSessionDriver extends SessionDriverAbstract {
21
+ declare type ConsoleSessionDriverOptions = SessionDriverOptions & {
22
+ progress?: "auto" | "tty" | "plain";
23
+ progressInterval?: number;
24
+ };
25
+ export declare class ConsoleSessionDriver extends SessionDriverAbstract<ConsoleSessionDriverOptions> {
22
26
  protected lastMessage: MessageType | undefined;
23
27
  protected lastMessageText: string | undefined;
28
+ protected lastProgressDate: number | undefined;
24
29
  protected prints: number;
25
30
  protected renderInterval: NodeJS.Timeout;
26
31
  protected rendering?: boolean;
@@ -30,6 +35,7 @@ export declare class ConsoleSessionDriver extends SessionDriverAbstract {
30
35
  start: () => number;
31
36
  elapsed: (formatted?: boolean | undefined) => string | number;
32
37
  };
38
+ protected tty: boolean;
33
39
  onInit(): Promise<void>;
34
40
  onEnd(data?: Record<string, any>): Promise<void>;
35
41
  onRead(): Promise<ReadResultType[]>;