@datatruck/cli 0.13.1 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/Action/BackupAction.js +7 -19
  2. package/Action/RestoreAction.js +4 -12
  3. package/Command/BackupCommand.js +2 -1
  4. package/Command/BackupSessionsCommand.js +1 -0
  5. package/Command/RestoreCommand.js +1 -1
  6. package/Command/RestoreSessionsCommand.js +1 -0
  7. package/Entity/StateEntityAbstract.d.ts +2 -5
  8. package/Error/AppError.d.ts +1 -0
  9. package/Error/AppError.js +4 -0
  10. package/Repository/DatatruckRepository.d.ts +2 -0
  11. package/Repository/DatatruckRepository.js +234 -123
  12. package/Repository/RepositoryAbstract.d.ts +4 -10
  13. package/Repository/ResticRepository.js +34 -17
  14. package/SessionDriver/ConsoleSessionDriver.d.ts +2 -7
  15. package/SessionDriver/ConsoleSessionDriver.js +51 -24
  16. package/SessionDriver/SqliteSessionDriver.js +5 -0
  17. package/SessionManager/BackupSessionManager.d.ts +12 -11
  18. package/SessionManager/BackupSessionManager.js +20 -5
  19. package/SessionManager/RestoreSessionManager.d.ts +14 -11
  20. package/SessionManager/RestoreSessionManager.js +20 -5
  21. package/SessionManager/SessionManagerAbstract.d.ts +18 -0
  22. package/SessionManager/SessionManagerAbstract.js +32 -0
  23. package/Task/GitTask.js +22 -14
  24. package/Task/MariadbTask.js +9 -4
  25. package/Task/MysqlDumpTask.d.ts +3 -1
  26. package/Task/MysqlDumpTask.js +5 -2
  27. package/Task/PostgresqlDumpTask.d.ts +3 -1
  28. package/Task/PostgresqlDumpTask.js +2 -2
  29. package/Task/SqlDumpTaskAbstract.d.ts +3 -1
  30. package/Task/SqlDumpTaskAbstract.js +55 -13
  31. package/Task/TaskAbstract.d.ts +3 -9
  32. package/cli.js +1 -1
  33. package/config.schema.json +3 -0
  34. package/migrations/001-initial.sql +6 -30
  35. package/package.json +1 -1
  36. package/util/cli-util.d.ts +1 -1
  37. package/util/cli-util.js +17 -2
  38. package/util/fs-util.d.ts +25 -21
  39. package/util/fs-util.js +60 -93
  40. package/util/math-util.js +2 -0
  41. package/util/process-util.d.ts +4 -0
  42. package/util/process-util.js +31 -4
  43. package/util/progress.d.ts +12 -0
  44. package/util/progress.js +2 -0
  45. package/util/string-util.d.ts +1 -0
  46. package/util/string-util.js +8 -1
  47. package/util/zip-util.d.ts +64 -20
  48. package/util/zip-util.js +153 -59
@@ -15,7 +15,7 @@ class BackupAction {
15
15
  }
16
16
  async init(session) {
17
17
  const snapshot = {
18
- id: (0, crypto_1.randomBytes)(20).toString("hex"),
18
+ id: (0, crypto_1.randomUUID)().replaceAll("-", ""),
19
19
  date: this.options.date ?? new Date().toISOString(),
20
20
  };
21
21
  await session.initDrivers();
@@ -74,14 +74,10 @@ class BackupAction {
74
74
  options: this.options,
75
75
  snapshot,
76
76
  targetPath,
77
- onProgress: async (data) => {
77
+ onProgress: async (progress) => {
78
78
  await session.progressTask({
79
79
  id: taskId,
80
- progressCurrent: data.current,
81
- progressPercent: data.percent,
82
- progressStep: data.step,
83
- progressStepPercent: data.stepPercent,
84
- progressTotal: data.total,
80
+ progress,
85
81
  });
86
82
  },
87
83
  });
@@ -120,14 +116,10 @@ class BackupAction {
120
116
  (!config.names || config.names.includes(repo.name)))?.config,
121
117
  options: this.options,
122
118
  snapshot: snapshot,
123
- onProgress: async (data) => {
119
+ onProgress: async (progress) => {
124
120
  await session.progressRepository({
125
121
  id: repositoryId,
126
- progressCurrent: data.current,
127
- progressPercent: data.percent,
128
- progressStep: data.step,
129
- progressStepPercent: data.stepPercent,
130
- progressTotal: data.total,
122
+ progress,
131
123
  });
132
124
  },
133
125
  });
@@ -164,14 +156,10 @@ class BackupAction {
164
156
  package: pkg,
165
157
  snapshot,
166
158
  mirrorRepositoryConfig: mirrorRepo.config,
167
- onProgress: async (data) => {
159
+ onProgress: async (progress) => {
168
160
  await session.progressRepository({
169
161
  id: repositoryId,
170
- progressCurrent: data.current,
171
- progressPercent: data.percent,
172
- progressStep: data.step,
173
- progressStepPercent: data.stepPercent,
174
- progressTotal: data.total,
162
+ progress,
175
163
  });
176
164
  },
177
165
  });
@@ -111,14 +111,10 @@ class RestoreAction {
111
111
  options: this.options,
112
112
  snapshot,
113
113
  targetPath,
114
- onProgress: async (data) => {
114
+ onProgress: async (progress) => {
115
115
  await session.progressTask({
116
116
  id: taskId,
117
- progressCurrent: data.current,
118
- progressPercent: data.percent,
119
- progressStep: data.step,
120
- progressStepPercent: data.stepPercent,
121
- progressTotal: data.total,
117
+ progress,
122
118
  });
123
119
  },
124
120
  });
@@ -160,14 +156,10 @@ class RestoreAction {
160
156
  (!config.names || config.names.includes(repo.name)))?.config,
161
157
  options: this.options,
162
158
  snapshot: snapshot,
163
- onProgress: async (data) => {
159
+ onProgress: async (progress) => {
164
160
  await session.progressRepository({
165
161
  id: repositoryId,
166
- progressCurrent: data.current,
167
- progressPercent: data.percent,
168
- progressStep: data.step,
169
- progressStepPercent: data.stepPercent,
170
- progressTotal: data.total,
162
+ progress,
171
163
  });
172
164
  },
173
165
  });
@@ -68,9 +68,10 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
68
68
  new ConsoleSessionDriver_1.ConsoleSessionDriver({
69
69
  verbose: verbose > 0,
70
70
  progress: this.globalOptions.progress,
71
- progressInterval: this.globalOptions.progressInterval,
72
71
  }),
73
72
  ],
73
+ verbose: verbose > 1,
74
+ progressInterval: this.globalOptions.progressInterval,
74
75
  });
75
76
  const result = await backup.exec(sessionManager);
76
77
  if (result.errors) {
@@ -50,6 +50,7 @@ class BackupSessionsCommand extends CommandAbstract_1.CommandAbstract {
50
50
  verbose: verbose > 1,
51
51
  }),
52
52
  verbose: verbose > 1,
53
+ progressInterval: this.globalOptions.progressInterval,
53
54
  });
54
55
  const items = await action.exec(manager);
55
56
  const dataFormat = new DataFormat_1.DataFormat({
@@ -68,10 +68,10 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
68
68
  new ConsoleSessionDriver_1.ConsoleSessionDriver({
69
69
  verbose: verbose > 0,
70
70
  progress: this.globalOptions.progress,
71
- progressInterval: this.globalOptions.progressInterval,
72
71
  }),
73
72
  ],
74
73
  verbose: verbose > 1,
74
+ progressInterval: this.globalOptions.progressInterval,
75
75
  });
76
76
  const result = await restore.exec(sessionManager);
77
77
  return result ? 0 : 1;
@@ -49,6 +49,7 @@ class RestoreSessionsCommand extends CommandAbstract_1.CommandAbstract {
49
49
  driver: new SqliteSessionDriver_1.SqliteSessionDriver({
50
50
  verbose: verbose > 1,
51
51
  }),
52
+ progressInterval: this.globalOptions.progressInterval,
52
53
  });
53
54
  const items = await action.exec(manager);
54
55
  const dataFormat = new DataFormat_1.DataFormat({
@@ -1,12 +1,9 @@
1
+ import { Progress } from "../util/progress";
1
2
  import { CrudEntityAbstract } from "./CrudEntityAbstract";
2
3
  export declare abstract class StateEntityAbstract extends CrudEntityAbstract {
3
4
  state: "started" | "ended" | null;
4
5
  error?: string | null;
5
6
  startDate?: string | null;
6
7
  endDate?: string | null;
7
- progressTotal?: number | null;
8
- progressCurrent?: number | null;
9
- progressPercent?: number | null;
10
- progressStep?: string | null;
11
- progressStepPercent?: number | null;
8
+ progress?: Progress;
12
9
  }
@@ -1,2 +1,3 @@
1
1
  export declare class AppError extends Error {
2
+ constructor(message: string);
2
3
  }
package/Error/AppError.js CHANGED
@@ -2,5 +2,9 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.AppError = void 0;
4
4
  class AppError extends Error {
5
+ constructor(message) {
6
+ super(message);
7
+ this.name = AppError.name;
8
+ }
5
9
  }
6
10
  exports.AppError = AppError;
@@ -19,6 +19,7 @@ export declare type DatatruckRepositoryConfigType = {
19
19
  };
20
20
  declare type CompressObjectType = {
21
21
  packs?: {
22
+ name?: string;
22
23
  include: string[];
23
24
  exclude?: string[];
24
25
  onePackByResult?: boolean;
@@ -48,6 +49,7 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
48
49
  static stringifyMetaData(data: MetaDataType): string;
49
50
  onGetSource(): string;
50
51
  onInit(data: InitDataType): Promise<void>;
52
+ private createFileScanner;
51
53
  onPrune(data: PruneDataType): Promise<void>;
52
54
  onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
53
55
  private normalizeCompressConfig;
@@ -19,6 +19,7 @@ const fs_1 = require("fs");
19
19
  const promises_1 = require("fs/promises");
20
20
  const micromatch_1 = require("micromatch");
21
21
  const path_1 = require("path");
22
+ const perf_hooks_1 = require("perf_hooks");
22
23
  const readline_1 = require("readline");
23
24
  exports.datatruckRepositoryName = "datatruck";
24
25
  exports.datatruckRepositoryDefinition = {
@@ -50,6 +51,7 @@ exports.datatruckPackageRepositoryDefinition = {
50
51
  additionalProperties: false,
51
52
  required: ["include"],
52
53
  properties: {
54
+ name: { type: "string" },
53
55
  include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
54
56
  exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
55
57
  onePackByResult: { type: "boolean" },
@@ -74,8 +76,6 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
74
76
  return `${date}_${pkgName}_${snapshotShortId}`;
75
77
  }
76
78
  static parseSnapshotName(name) {
77
- if (!name.endsWith(".json"))
78
- return null;
79
79
  name = name.replace(/\.json$/, "");
80
80
  const nameParts = name.split("_");
81
81
  if (nameParts.length !== 3)
@@ -102,6 +102,67 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
102
102
  async onInit(data) {
103
103
  await (0, fs_util_1.mkdirIfNotExists)(this.config.outPath);
104
104
  }
105
+ async createFileScanner(options) {
106
+ const object = {
107
+ total: 0,
108
+ current: 0,
109
+ progress: async (description, data) => {
110
+ await options.onProgress({
111
+ relative: {
112
+ description,
113
+ payload: data.path,
114
+ percent: data.percent,
115
+ },
116
+ absolute: {
117
+ total: object.total,
118
+ current: object.current + data.current,
119
+ percent: (0, math_util_1.progressPercent)(object.total, object.current + data.current),
120
+ },
121
+ });
122
+ if (data.type === "end") {
123
+ object.current += data.current;
124
+ }
125
+ },
126
+ start: async (cb) => {
127
+ for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
128
+ if (!options.disableCounting)
129
+ object.total++;
130
+ const currentTime = perf_hooks_1.performance.now();
131
+ const diff = currentTime - lastTime;
132
+ if (diff > 1000) {
133
+ await options.onProgress({
134
+ relative: {
135
+ description: "Scanning files",
136
+ payload: object.total.toString(),
137
+ },
138
+ });
139
+ lastTime = currentTime;
140
+ }
141
+ if (cb)
142
+ await cb(entry);
143
+ }
144
+ await options.onProgress({
145
+ relative: {
146
+ description: "Scanned files",
147
+ payload: object.total.toString(),
148
+ },
149
+ });
150
+ },
151
+ };
152
+ await options.onProgress({
153
+ relative: {
154
+ description: "Scanning files",
155
+ },
156
+ });
157
+ const stream = fast_glob_1.default.stream(options.glob.include, {
158
+ dot: true,
159
+ markDirectories: true,
160
+ stats: true,
161
+ ...options.glob,
162
+ });
163
+ let lastTime = perf_hooks_1.performance.now();
164
+ return object;
165
+ }
105
166
  async onPrune(data) {
106
167
  const snapshotName = DatatruckRepository.buildSnapshotName({
107
168
  snapshotId: data.snapshot.id,
@@ -109,15 +170,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
109
170
  packageName: data.snapshot.packageName,
110
171
  });
111
172
  const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
112
- const metaPath = `${snapshotPath}.json`;
113
173
  if (data.options.verbose)
114
174
  (0, cli_util_1.logExec)(`Deleting ${snapshotPath}`);
115
175
  if (await (0, fs_util_1.checkDir)(snapshotPath))
116
176
  await (0, promises_1.rm)(snapshotPath, {
117
177
  recursive: true,
118
178
  });
119
- if (await (0, fs_util_1.checkFile)(metaPath))
120
- await (0, promises_1.rm)(metaPath);
121
179
  }
122
180
  async onSnapshots(data) {
123
181
  if (!(await (0, fs_util_1.checkDir)(this.config.outPath)))
@@ -136,7 +194,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
136
194
  if (data.options.ids &&
137
195
  !data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
138
196
  continue;
139
- const metaPath = (0, path_1.join)(this.config.outPath, snapshotName);
197
+ const metaPath = (0, path_1.join)(this.config.outPath, snapshotName, "meta.json");
140
198
  const meta = await DatatruckRepository.parseMetaData(metaPath);
141
199
  if (taskPatterns && !(0, string_util_1.checkMatch)(meta.task, taskPatterns))
142
200
  continue;
@@ -160,7 +218,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
160
218
  }
161
219
  normalizeCompressConfig(packageConfig) {
162
220
  let compress = packageConfig?.compress ?? this.config.compress;
163
- if (compress === true) {
221
+ if (compress === true || (compress && !Array.isArray(compress.packs))) {
164
222
  return {
165
223
  packs: [
166
224
  {
@@ -198,96 +256,124 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
198
256
  verbose: data.options.verbose,
199
257
  })
200
258
  : undefined;
201
- const stream = fast_glob_1.default.stream(include, {
202
- cwd: sourcePath,
203
- ignore: exclude,
204
- dot: true,
205
- onlyFiles: compress ? false : true,
206
- markDirectories: true,
207
- });
259
+ const packs = compress?.packs || [];
260
+ const tmpDir = await (0, fs_util_1.mkTmpDir)("path-lists");
261
+ const unpackedStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "unpacked.txt"));
262
+ const singlePackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "single-pack.txt"));
263
+ const packStreams = Array.from({ length: packs.length }).map((v, i) => (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, `pack-${i}.txt`)));
264
+ const streams = [unpackedStream, singlePackStream, ...packStreams];
208
265
  if (data.options.verbose)
209
- (0, cli_util_1.logExec)(`Writing paths lists`);
210
- const pathLists = await (0, fs_util_1.writePathLists)({
211
- paths: stream,
212
- packs: compress?.packs,
266
+ (0, cli_util_1.logExec)(`Writing file lists in ${tmpDir}`);
267
+ const scanner = await this.createFileScanner({
268
+ glob: {
269
+ include,
270
+ cwd: sourcePath,
271
+ ignore: exclude,
272
+ onlyFiles: false,
273
+ },
274
+ onProgress: data.onProgress,
275
+ disableCounting: true,
213
276
  });
214
- if (data.options.verbose)
215
- (0, cli_util_1.logExec)(`Path lists: ${pathLists.path}`);
216
- let currentFiles = 0;
217
- if (compress?.packs) {
218
- let packIndex = 0;
219
- for (const packsPath of pathLists.includedPackPaths) {
220
- const pack = compress.packs[packIndex];
221
- if (pack.onePackByResult) {
222
- const reader = (0, readline_1.createInterface)({
223
- input: (0, fs_1.createReadStream)(packsPath),
224
- });
225
- let multipleIndex = 0;
226
- for await (let packPath of reader) {
227
- if (packPath.endsWith("/"))
228
- packPath = packPath.slice(0, -1);
229
- const target = (0, path_1.join)(outPath, `.${packIndex}-${multipleIndex++}-${encodeURIComponent(packPath.replace(/[\\/]/g, "-")).slice(0, 255)}.dd.zip`);
230
- const stats = await (0, zip_util_1.zip)({
231
- path: pkg.path,
232
- output: target,
233
- filter: [{ patterns: [packPath] }],
234
- excludeList: pathLists.excludedPackPaths[packIndex],
235
- verbose: data.options.verbose,
236
- onStream: async (stream) => await data.onProgress({
237
- total: pathLists.total.all,
238
- current: currentFiles + stream.data.files,
239
- percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
240
- step: stream.type === "progress" ? stream.data.path : "",
241
- stepPercent: stream.type === "progress" ? stream.data.progress : null,
242
- }),
243
- });
244
- currentFiles += stats.files;
277
+ await Promise.all([
278
+ ...streams.map((p) => (0, fs_util_1.waitForClose)(p)),
279
+ (async () => {
280
+ await scanner.start(async (entry) => {
281
+ const pathSubject = entry.stats.isDirectory()
282
+ ? entry.path.slice(0, -1)
283
+ : entry.path;
284
+ let stream = unpackedStream;
285
+ let successPackIndex;
286
+ for (const [packIndex, pack] of packs.entries()) {
287
+ if ((0, string_util_1.checkPath)(pathSubject, pack.include, pack.exclude)) {
288
+ stream = pack.onePackByResult
289
+ ? singlePackStream
290
+ : packStreams[packIndex];
291
+ successPackIndex = packIndex;
292
+ break;
293
+ }
245
294
  }
295
+ const isUnpackedStream = stream === unpackedStream;
296
+ const isPackStream = stream !== unpackedStream;
297
+ const isSinglePackStream = stream === singlePackStream;
298
+ const include = isPackStream
299
+ ? entry.stats.isDirectory()
300
+ ? await (0, fs_util_1.isEmptyDir)((0, path_1.join)(sourcePath, entry.path))
301
+ : true
302
+ : true;
303
+ if (include) {
304
+ let value = entry.path;
305
+ if (isUnpackedStream) {
306
+ value += `:${entry.stats.uid}:${entry.stats.gid}:${entry.stats.mode}`;
307
+ }
308
+ else if (isSinglePackStream) {
309
+ value += `:${successPackIndex}`;
310
+ }
311
+ if (!entry.stats.isDirectory())
312
+ scanner.total++;
313
+ stream.write(`${value}\n`);
314
+ }
315
+ });
316
+ for (const stream of streams) {
317
+ stream.end();
246
318
  }
247
- else {
248
- const target = (0, path_1.join)(outPath, `.${packIndex}.dd.zip`);
249
- const stats = await (0, zip_util_1.zip)({
250
- path: sourcePath,
251
- output: target,
252
- includeList: packsPath,
253
- excludeList: pathLists.excludedPackPaths[packIndex],
254
- verbose: data.options.verbose,
255
- onStream: async (stream) => await data.onProgress({
256
- total: pathLists.total.all,
257
- current: currentFiles + stream.data.files,
258
- percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
259
- step: stream.type === "progress" ? stream.data.path : "",
260
- }),
261
- });
262
- currentFiles += stats.files;
263
- }
264
- packIndex++;
265
- }
266
- }
319
+ })(),
320
+ ]);
321
+ const unpackedPath = (0, path_1.join)(outPath, "unpacked");
322
+ await (0, promises_1.mkdir)(unpackedPath);
323
+ await (0, promises_1.copyFile)(unpackedStream.path, (0, path_1.join)(outPath, "permissions.txt"));
324
+ // Non pack
267
325
  if (data.options.verbose)
268
- (0, cli_util_1.logExec)(`Copying files to ${outPath}`);
326
+ (0, cli_util_1.logExec)(`Copying files from ${unpackedStream.path.toString()} to ${unpackedPath}`);
269
327
  await (0, fs_util_1.cpy)({
270
328
  input: {
271
329
  type: "pathList",
272
- path: pathLists.path,
330
+ path: unpackedStream.path.toString(),
273
331
  basePath: sourcePath,
274
332
  },
275
- targetPath: outPath,
333
+ targetPath: unpackedPath,
276
334
  skipNotFoundError: true,
277
335
  concurrency: this.config.fileCopyConcurrency,
278
- async onPath({ isDir, entryPath }) {
279
- if (isDir)
280
- return;
281
- currentFiles++;
282
- await data.onProgress({
283
- total: pathLists.total.all,
284
- current: currentFiles,
285
- percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles),
286
- step: entryPath,
287
- });
288
- },
336
+ onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
337
+ });
338
+ // Single pack
339
+ const singleReader = (0, readline_1.createInterface)({
340
+ input: (0, fs_1.createReadStream)(singlePackStream.path),
289
341
  });
290
- const metaPath = `${outPath}.json`;
342
+ for await (const line of singleReader) {
343
+ let [packPath, packIndex] = line.split(":");
344
+ const pack = packs[packIndex];
345
+ if (packPath.endsWith("/"))
346
+ packPath = packPath.slice(0, -1);
347
+ const outBasename = (`pack${pack.name ? `-${encodeURIComponent(pack.name)}` : ""}` +
348
+ `-${encodeURIComponent(packPath.replace(/[\\/]/g, "-"))}` +
349
+ `.zip`).slice(0, 255);
350
+ const target = (0, path_1.join)(outPath, outBasename);
351
+ await (0, zip_util_1.zip)({
352
+ path: pkg.path,
353
+ output: target,
354
+ filter: [{ patterns: [packPath] }],
355
+ verbose: data.options.verbose,
356
+ onProgress: async (progress) => await scanner.progress(progress.type === "start"
357
+ ? "Starting compressing"
358
+ : "Compressing file", progress),
359
+ });
360
+ }
361
+ // Packs
362
+ for (const [packIndex, packStream] of packStreams.entries()) {
363
+ const pack = packs[packIndex];
364
+ const target = (0, path_1.join)(outPath, `pack-${packIndex}${pack.name ? `-${pack.name}` : ""}.zip`);
365
+ await (0, zip_util_1.zip)({
366
+ path: sourcePath,
367
+ output: target,
368
+ includeList: packStream.path.toString(),
369
+ verbose: data.options.verbose,
370
+ onProgress: async (progress) => await scanner.progress(progress.type === "start"
371
+ ? "Starting compressing"
372
+ : "Compressing file", progress),
373
+ });
374
+ }
375
+ // Meta
376
+ const metaPath = `${outPath}/meta.json`;
291
377
  const nodePkg = (0, fs_util_1.parsePackageFile)();
292
378
  const meta = {
293
379
  id: data.snapshot.id,
@@ -310,19 +396,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
310
396
  });
311
397
  const sourcePath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
312
398
  const targetPath = (0, path_1.resolve)((0, path_1.join)(data.mirrorRepositoryConfig.outPath, snapshotName));
313
- const sourceMetaPath = `${sourcePath}.json`;
314
- const targetMetaPath = `${targetPath}.json`;
315
399
  if (data.options.verbose)
316
- (0, cli_util_1.logExec)(`Copying files to ${targetPath}`);
400
+ (0, cli_util_1.logExec)(`Copying backup files to ${targetPath}`);
317
401
  await (0, promises_1.mkdir)(targetPath);
402
+ const scanner = await this.createFileScanner({
403
+ glob: {
404
+ include: ["**/*"],
405
+ cwd: sourcePath,
406
+ },
407
+ onProgress: data.onProgress,
408
+ });
409
+ await scanner.start();
318
410
  await (0, fs_util_1.cpy)({
319
411
  input: {
320
412
  type: "glob",
321
413
  sourcePath,
322
414
  },
323
415
  targetPath,
416
+ onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
324
417
  });
325
- await (0, promises_1.copyFile)(sourceMetaPath, targetMetaPath);
326
418
  }
327
419
  async onRestore(data) {
328
420
  const relRestorePath = data.targetPath ?? data.package.restorePath;
@@ -341,50 +433,69 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
341
433
  packageName: data.package.name,
342
434
  });
343
435
  const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
344
- let totalFiles = 0;
345
- let currentFiles = -1;
346
- await (0, fs_util_1.forEachFile)(sourcePath, () => {
347
- totalFiles++;
348
- }, true);
436
+ const scanner = await this.createFileScanner({
437
+ glob: {
438
+ include: ["unpacked/**/*"],
439
+ cwd: sourcePath,
440
+ },
441
+ onProgress: data.onProgress,
442
+ });
443
+ await scanner.start();
444
+ const it = await (0, promises_1.opendir)(sourcePath);
445
+ for await (const dirent of it) {
446
+ const path = (0, path_1.join)(sourcePath, dirent.name);
447
+ if (dirent.name === "permissions.txt") {
448
+ scanner.total++;
449
+ }
450
+ else if (dirent.name.endsWith(".zip")) {
451
+ await (0, zip_util_1.listZip)({
452
+ path,
453
+ verbose: data.options.verbose,
454
+ onStream: async (item) => {
455
+ const isDir = item.Folder === "+";
456
+ if (!isDir)
457
+ scanner.total++;
458
+ },
459
+ });
460
+ }
461
+ }
349
462
  if (data.options.verbose)
350
463
  (0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
351
464
  await (0, fs_util_1.cpy)({
352
465
  input: {
353
466
  type: "glob",
354
- sourcePath,
467
+ sourcePath: (0, path_1.join)(sourcePath, "unpacked"),
355
468
  },
356
469
  targetPath: restorePath,
357
470
  concurrency: this.config.fileCopyConcurrency,
358
- onPath: async ({ entryPath, entrySourcePath }) => {
359
- const isRootFile = (0, path_1.basename)(entryPath) === entryPath;
360
- const isZipFile = isRootFile &&
361
- entryPath.startsWith(".") &&
362
- entryPath.endsWith(".dd.zip");
363
- await data.onProgress({
364
- total: totalFiles,
365
- current: Math.max(currentFiles, 0),
366
- percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
367
- step: entryPath,
368
- });
369
- if (isZipFile) {
370
- await (0, zip_util_1.unzip)({
371
- input: entrySourcePath,
372
- output: restorePath,
373
- verbose: data.options.verbose,
374
- onStream: async (stream) => await data.onProgress({
375
- total: totalFiles,
376
- current: currentFiles + 1,
377
- percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + 1),
378
- step: stream.type === "progress"
379
- ? `Extracting ${stream.data.path}`
380
- : "",
381
- }),
382
- });
383
- }
384
- currentFiles++;
385
- return isZipFile ? false : true;
386
- },
471
+ onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
387
472
  });
473
+ const it2 = await (0, promises_1.opendir)(sourcePath);
474
+ for await (const dirent of it2) {
475
+ const path = (0, path_1.join)(sourcePath, dirent.name);
476
+ if (dirent.name === "permissions.txt") {
477
+ if (data.options.verbose)
478
+ (0, cli_util_1.logExec)(`Applying permissions (${path})`);
479
+ await scanner.progress("Applying permissions", {
480
+ current: 0,
481
+ });
482
+ await (0, fs_util_1.applyPermissions)(restorePath, path);
483
+ await scanner.progress("Permissions applied", {
484
+ current: 1,
485
+ type: "end",
486
+ });
487
+ }
488
+ else if (dirent.name.endsWith(".zip")) {
489
+ await (0, zip_util_1.unzip)({
490
+ input: path,
491
+ output: restorePath,
492
+ verbose: data.options.verbose,
493
+ onProgress: async (progress) => await scanner.progress(progress.type === "start"
494
+ ? "Starting extracting"
495
+ : "Extracting file", progress),
496
+ });
497
+ }
498
+ }
388
499
  }
389
500
  }
390
501
  exports.DatatruckRepository = DatatruckRepository;