@datatruck/cli 0.26.2 → 0.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/Action/BackupAction.d.ts +69 -34
  2. package/Action/BackupAction.js +284 -244
  3. package/Action/CleanCacheAction.d.ts +8 -4
  4. package/Action/CleanCacheAction.js +8 -5
  5. package/Action/ConfigAction.d.ts +12 -5
  6. package/Action/ConfigAction.js +14 -18
  7. package/Action/CopyAction.d.ts +49 -0
  8. package/Action/CopyAction.js +144 -0
  9. package/Action/InitAction.d.ts +3 -3
  10. package/Action/InitAction.js +9 -9
  11. package/Action/PruneAction.d.ts +9 -9
  12. package/Action/PruneAction.js +39 -23
  13. package/Action/RestoreAction.d.ts +48 -23
  14. package/Action/RestoreAction.js +158 -195
  15. package/Action/SnapshotsAction.d.ts +8 -8
  16. package/Action/SnapshotsAction.js +8 -8
  17. package/CHANGELOG.md +495 -0
  18. package/Command/BackupCommand.d.ts +6 -4
  19. package/Command/BackupCommand.js +9 -26
  20. package/Command/CleanCacheCommand.d.ts +4 -4
  21. package/Command/CleanCacheCommand.js +26 -5
  22. package/Command/CommandAbstract.d.ts +10 -7
  23. package/Command/CommandAbstract.js +4 -1
  24. package/Command/ConfigCommand.d.ts +6 -9
  25. package/Command/ConfigCommand.js +13 -8
  26. package/Command/CopyCommand.d.ts +15 -0
  27. package/Command/CopyCommand.js +61 -0
  28. package/Command/InitCommand.d.ts +4 -4
  29. package/Command/InitCommand.js +11 -15
  30. package/Command/PruneCommand.d.ts +3 -3
  31. package/Command/PruneCommand.js +13 -12
  32. package/Command/RestoreCommand.js +9 -17
  33. package/Command/SnapshotsCommand.d.ts +4 -4
  34. package/Command/SnapshotsCommand.js +16 -15
  35. package/Command/StartServerCommand.d.ts +6 -0
  36. package/Command/StartServerCommand.js +24 -0
  37. package/Config/Config.d.ts +11 -0
  38. package/Config/Config.js +43 -0
  39. package/Config/PrunePolicyConfig.d.ts +2 -2
  40. package/Factory/CommandFactory.d.ts +29 -33
  41. package/Factory/CommandFactory.js +32 -53
  42. package/Factory/RepositoryFactory.d.ts +1 -1
  43. package/Factory/RepositoryFactory.js +3 -3
  44. package/Factory/TaskFactory.d.ts +1 -1
  45. package/Factory/TaskFactory.js +3 -3
  46. package/Repository/DatatruckRepository.d.ts +17 -16
  47. package/Repository/DatatruckRepository.js +131 -149
  48. package/Repository/GitRepository.d.ts +9 -8
  49. package/Repository/GitRepository.js +22 -25
  50. package/Repository/RepositoryAbstract.d.ts +39 -37
  51. package/Repository/RepositoryAbstract.js +4 -5
  52. package/Repository/ResticRepository.d.ts +9 -8
  53. package/Repository/ResticRepository.js +30 -28
  54. package/Task/GitTask.d.ts +6 -7
  55. package/Task/GitTask.js +24 -30
  56. package/Task/MariadbTask.d.ts +4 -5
  57. package/Task/MariadbTask.js +26 -32
  58. package/Task/MssqlTask.d.ts +5 -3
  59. package/Task/MssqlTask.js +11 -12
  60. package/Task/MysqlDumpTask.d.ts +10 -3
  61. package/Task/MysqlDumpTask.js +107 -31
  62. package/Task/ScriptTask.d.ts +23 -18
  63. package/Task/ScriptTask.js +34 -24
  64. package/Task/SqlDumpTaskAbstract.d.ts +8 -3
  65. package/Task/SqlDumpTaskAbstract.js +31 -19
  66. package/Task/TaskAbstract.d.ts +24 -25
  67. package/Task/TaskAbstract.js +6 -10
  68. package/cli.js +14 -5
  69. package/config.schema.json +124 -3
  70. package/package.json +4 -5
  71. package/utils/DataFormat.d.ts +23 -12
  72. package/utils/DataFormat.js +36 -14
  73. package/utils/cli.d.ts +2 -9
  74. package/utils/cli.js +9 -52
  75. package/utils/datatruck/client.d.ts +24 -0
  76. package/utils/datatruck/client.js +99 -0
  77. package/utils/datatruck/config.d.ts +8 -6
  78. package/utils/datatruck/config.js +18 -3
  79. package/utils/datatruck/paths.d.ts +5 -9
  80. package/utils/datatruck/paths.js +2 -2
  81. package/utils/datatruck/server.d.ts +21 -0
  82. package/utils/datatruck/server.js +96 -0
  83. package/utils/datatruck/snapshot.d.ts +2 -2
  84. package/utils/date.d.ts +7 -3
  85. package/utils/date.js +22 -14
  86. package/utils/fs.d.ts +27 -15
  87. package/utils/fs.js +110 -62
  88. package/utils/http.d.ts +21 -0
  89. package/utils/http.js +154 -0
  90. package/utils/list.d.ts +64 -0
  91. package/utils/list.js +145 -0
  92. package/utils/mysql.d.ts +2 -0
  93. package/utils/mysql.js +21 -2
  94. package/utils/process.d.ts +1 -0
  95. package/utils/process.js +24 -31
  96. package/utils/progress.d.ts +33 -0
  97. package/utils/progress.js +113 -0
  98. package/utils/steps.d.ts +11 -0
  99. package/utils/steps.js +22 -10
  100. package/utils/stream.d.ts +7 -0
  101. package/utils/stream.js +10 -0
  102. package/utils/string.d.ts +0 -1
  103. package/utils/string.js +1 -13
  104. package/utils/tar.d.ts +10 -3
  105. package/utils/tar.js +70 -44
  106. package/utils/temp.d.ts +26 -0
  107. package/utils/temp.js +133 -0
  108. package/utils/virtual-fs.d.ts +37 -0
  109. package/utils/virtual-fs.js +65 -0
  110. package/Action/BackupSessionsAction.d.ts +0 -13
  111. package/Action/BackupSessionsAction.js +0 -18
  112. package/Action/RestoreSessionsAction.d.ts +0 -13
  113. package/Action/RestoreSessionsAction.js +0 -18
  114. package/Command/BackupSessionsCommand.d.ts +0 -12
  115. package/Command/BackupSessionsCommand.js +0 -92
  116. package/Command/RestoreSessionsCommand.d.ts +0 -12
  117. package/Command/RestoreSessionsCommand.js +0 -91
  118. package/Decorator/EntityDecorator.d.ts +0 -11
  119. package/Decorator/EntityDecorator.js +0 -17
  120. package/Entity/BackupSessionEntity.d.ts +0 -6
  121. package/Entity/BackupSessionEntity.js +0 -25
  122. package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
  123. package/Entity/BackupSessionRepositoryEntity.js +0 -25
  124. package/Entity/BackupSessionTaskEntity.d.ts +0 -5
  125. package/Entity/BackupSessionTaskEntity.js +0 -24
  126. package/Entity/CrudEntityAbstract.d.ts +0 -5
  127. package/Entity/CrudEntityAbstract.js +0 -9
  128. package/Entity/RestoreSessionEntity.d.ts +0 -5
  129. package/Entity/RestoreSessionEntity.js +0 -24
  130. package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
  131. package/Entity/RestoreSessionRepositoryEntity.js +0 -25
  132. package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
  133. package/Entity/RestoreSessionTaskEntity.js +0 -24
  134. package/Entity/StateEntityAbstract.d.ts +0 -9
  135. package/Entity/StateEntityAbstract.js +0 -12
  136. package/Factory/EntityFactory.d.ts +0 -6
  137. package/Factory/EntityFactory.js +0 -40
  138. package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
  139. package/SessionDriver/ConsoleSessionDriver.js +0 -208
  140. package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
  141. package/SessionDriver/SessionDriverAbstract.js +0 -28
  142. package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
  143. package/SessionDriver/SqliteSessionDriver.js +0 -173
  144. package/SessionManager/BackupSessionManager.d.ts +0 -45
  145. package/SessionManager/BackupSessionManager.js +0 -218
  146. package/SessionManager/RestoreSessionManager.d.ts +0 -47
  147. package/SessionManager/RestoreSessionManager.js +0 -218
  148. package/SessionManager/SessionManagerAbstract.d.ts +0 -18
  149. package/SessionManager/SessionManagerAbstract.js +0 -36
  150. package/migrations/001-initial.sql +0 -98
  151. package/utils/entity.d.ts +0 -4
  152. package/utils/entity.js +0 -10
@@ -4,10 +4,13 @@ exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exp
4
4
  const AppError_1 = require("../Error/AppError");
5
5
  const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
6
6
  const cli_1 = require("../utils/cli");
7
+ const client_1 = require("../utils/datatruck/client");
7
8
  const paths_1 = require("../utils/datatruck/paths");
8
9
  const fs_1 = require("../utils/fs");
10
+ const math_1 = require("../utils/math");
9
11
  const string_1 = require("../utils/string");
10
12
  const tar_1 = require("../utils/tar");
13
+ const temp_1 = require("../utils/temp");
11
14
  const RepositoryAbstract_1 = require("./RepositoryAbstract");
12
15
  const assert_1 = require("assert");
13
16
  const promises_1 = require("fs/promises");
@@ -16,10 +19,10 @@ const path_1 = require("path");
16
19
  exports.datatruckRepositoryName = "datatruck";
17
20
  exports.datatruckRepositoryDefinition = {
18
21
  type: "object",
19
- required: ["outPath"],
22
+ required: ["backend"],
20
23
  additionalProperties: false,
21
24
  properties: {
22
- outPath: { type: "string" },
25
+ backend: { type: "string" },
23
26
  compress: {
24
27
  anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
25
28
  },
@@ -52,10 +55,10 @@ exports.datatruckPackageRepositoryDefinition = {
52
55
  };
53
56
  class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
54
57
  static zipBasenameTpl = `.*.dd.tar.gz`;
55
- static buildSnapshotName(data) {
56
- const date = data.snapshotDate.replace(/:/g, "-");
57
- const pkgName = encodeURIComponent(data.packageName).replace(/%40/g, "@");
58
- const snapshotShortId = data.snapshotId.slice(0, 8);
58
+ static buildSnapshotName(snapshot, pkg) {
59
+ const date = snapshot.date.replace(/:/g, "-");
60
+ const pkgName = encodeURIComponent(pkg.name).replace(/%40/g, "@");
61
+ const snapshotShortId = snapshot.id.slice(0, 8);
59
62
  return `${date}_${pkgName}_${snapshotShortId}`;
60
63
  }
61
64
  static parseSnapshotName(name) {
@@ -69,49 +72,36 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
69
72
  packageName = decodeURIComponent(packageName);
70
73
  return { snapshotDate, packageName, snapshotShortId, sourcePath: name };
71
74
  }
72
- buildMetaPath(snapshotName, packageName) {
73
- return (0, path_1.join)(this.config.outPath, snapshotName, packageName) + ".meta.json";
75
+ static async parseMetaData(data) {
76
+ return JSON.parse(data.toString());
74
77
  }
75
- static async parseMetaData(path) {
76
- let contents;
77
- try {
78
- contents = await (0, promises_1.readFile)(path);
79
- }
80
- catch (error) {
81
- if ((0, fs_1.isNotFoundError)(error))
82
- return;
83
- throw error;
84
- }
85
- return JSON.parse(contents.toString());
86
- }
87
- static stringifyMetaData(data) {
88
- return JSON.stringify(data);
78
+ getSource() {
79
+ return this.config.backend;
89
80
  }
90
- onGetSource() {
91
- return this.config.outPath;
81
+ fetchDiskStats(config) {
82
+ const fs = (0, client_1.createFs)(config.backend);
83
+ return fs.fetchDiskStats(".");
92
84
  }
93
- async onInit(data) {
94
- await (0, fs_1.mkdirIfNotExists)(this.config.outPath);
85
+ async init(data) {
86
+ const fs = (0, client_1.createFs)(this.config.backend);
87
+ await fs.mkdir(".");
95
88
  }
96
- async onPrune(data) {
97
- const snapshotName = DatatruckRepository.buildSnapshotName({
98
- snapshotId: data.snapshot.id,
99
- snapshotDate: data.snapshot.date,
100
- packageName: data.snapshot.packageName,
89
+ async prune(data) {
90
+ const fs = (0, client_1.createFs)(this.config.backend);
91
+ const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, {
92
+ name: data.snapshot.packageName,
101
93
  });
102
- const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
103
94
  if (data.options.verbose)
104
- (0, cli_1.logExec)(`Deleting ${snapshotPath}`);
105
- if (await (0, fs_1.existsDir)(snapshotPath))
106
- await (0, promises_1.rm)(snapshotPath, {
107
- recursive: true,
108
- });
95
+ (0, cli_1.logExec)(`Deleting ${fs.resolvePath(snapshotName)}`);
96
+ if (await fs.existsDir(snapshotName))
97
+ await fs.rmAll(snapshotName);
109
98
  }
110
- async onSnapshots(data) {
111
- if (!(await (0, fs_1.existsDir)(this.config.outPath)))
112
- throw new Error(`Repository (${this.repository.name}) out path does not exist: ${this.config.outPath}`);
113
- const snapshotNames = await (0, fs_1.readDir)(this.config.outPath);
99
+ async fetchSnapshots(data) {
100
+ const fs = (0, client_1.createFs)(this.config.backend);
101
+ if (!(await fs.existsDir(".")))
102
+ throw new Error(`Repository (${this.repository.name}) out path does not exist: ${fs.resolvePath(".")}`);
114
103
  const snapshots = [];
104
+ const snapshotNames = await fs.readdir(".");
115
105
  const packagePatterns = (0, string_1.makePathPatterns)(data.options.packageNames);
116
106
  const taskPatterns = (0, string_1.makePathPatterns)(data.options.packageTaskNames);
117
107
  for (const snapshotName of snapshotNames) {
@@ -124,8 +114,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
124
114
  if (data.options.ids &&
125
115
  !data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
126
116
  continue;
127
- const metaPath = (0, path_1.join)(this.config.outPath, snapshotName, "meta.json");
128
- const meta = await DatatruckRepository.parseMetaData(metaPath);
117
+ const metaData = await fs.readFileIfExists(`${snapshotName}/meta.json`);
118
+ const meta = !!metaData && (await DatatruckRepository.parseMetaData(metaData));
129
119
  if (!meta)
130
120
  continue;
131
121
  if (taskPatterns && !(0, string_1.checkMatch)(meta.task, taskPatterns))
@@ -148,27 +138,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
148
138
  }
149
139
  return snapshots;
150
140
  }
151
- async onBackup(data) {
152
- const snapshotName = DatatruckRepository.buildSnapshotName({
153
- snapshotId: data.snapshot.id,
154
- snapshotDate: data.snapshot.date,
155
- packageName: data.package.name,
156
- });
157
- const outPath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
141
+ async backup(data) {
142
+ const fs = (0, client_1.createFs)(this.config.backend);
143
+ const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
144
+ const outPath = fs.isLocal()
145
+ ? fs.resolvePath(snapshotName)
146
+ : await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "fs-remote");
158
147
  const pkg = data.package;
159
- const sourcePath = data.targetPath ?? pkg.path;
160
- (0, assert_1.ok)(sourcePath);
161
- await (0, promises_1.mkdir)(outPath, { recursive: true });
148
+ const path = pkg.path;
149
+ await fs.mkdir(snapshotName);
162
150
  const backupPathsOptions = {
163
151
  package: data.package,
164
152
  snapshot: data.snapshot,
165
- targetPath: sourcePath,
153
+ path: path,
166
154
  verbose: data.options.verbose,
167
155
  };
168
156
  const scanner = await (0, fs_1.createFileScanner)({
169
157
  onProgress: data.onProgress,
170
158
  glob: {
171
- cwd: sourcePath,
159
+ cwd: path,
172
160
  onlyFiles: false,
173
161
  include: await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions),
174
162
  ignore: pkg.exclude
@@ -187,7 +175,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
187
175
  const packs = [defaultsPack, ...configPacks];
188
176
  const defaultsPackIndex = packs.findIndex((p) => p === defaultsPack);
189
177
  const stream = (0, fs_1.createWriteStreamPool)({
190
- path: await this.mkTmpDir("files"),
178
+ path: await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "stream-pool"),
191
179
  onStreamPath: (key) => `files-${key}.txt`,
192
180
  });
193
181
  scanner.total++;
@@ -209,7 +197,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
209
197
  scanner.total++;
210
198
  stream.writeLine(packIndex, ".");
211
199
  }
212
- stream.writeLine(packIndex, entry.path);
200
+ stream.writeLine(packIndex, (0, tar_1.normalizeTarPath)(entry.path));
213
201
  return true;
214
202
  });
215
203
  await stream.end();
@@ -224,21 +212,28 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
224
212
  if (includeList) {
225
213
  tarStats[packBasename] = {
226
214
  files: stream.lines(packIndex),
215
+ size: 0,
227
216
  };
217
+ const tarPath = (0, path_1.join)(outPath, packBasename);
228
218
  await (0, tar_1.createTar)({
229
219
  compress: pack.compress,
230
220
  verbose: data.options.verbose,
231
221
  includeList,
232
- path: sourcePath,
233
- output: (0, path_1.join)(outPath, packBasename),
234
- onEntry: async (data) => await scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
222
+ path: path,
223
+ output: tarPath,
224
+ onEntry: async (data) => scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
235
225
  });
226
+ tarStats[packBasename].size = (await (0, promises_1.stat)(tarPath)).size;
227
+ if (!fs.isLocal()) {
228
+ await fs.upload(tarPath, `${snapshotName}/${packBasename}`);
229
+ await (0, promises_1.rm)(tarPath);
230
+ }
236
231
  }
237
232
  packIndex++;
238
233
  }
239
- await scanner.end();
234
+ scanner.end();
240
235
  // Meta
241
- const metaPath = `${outPath}/meta.json`;
236
+ const metaPath = `${snapshotName}/meta.json`;
242
237
  const nodePkg = (0, fs_1.parsePackageFile)();
243
238
  const meta = {
244
239
  id: data.snapshot.id,
@@ -247,112 +242,99 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
247
242
  package: data.package.name,
248
243
  task: data.package.task?.name,
249
244
  version: nodePkg.version,
250
- size: await (0, fs_1.fastFolderSizeAsync)(outPath),
245
+ size: Object.values(tarStats).reduce((total, { size }) => total + size, 0),
251
246
  tarStats,
252
247
  };
253
248
  if (data.options.verbose)
254
- (0, cli_1.logExec)(`Writing metadata into ${metaPath}`);
255
- await (0, promises_1.writeFile)(metaPath, DatatruckRepository.stringifyMetaData(meta));
249
+ (0, cli_1.logExec)(`Writing metadata into ${fs.resolvePath(metaPath)}`);
250
+ await fs.writeFile(`${snapshotName}/meta.json`, JSON.stringify(meta));
256
251
  }
257
- async onCopyBackup(data) {
258
- const snapshotName = DatatruckRepository.buildSnapshotName({
259
- snapshotId: data.snapshot.id,
260
- snapshotDate: data.snapshot.date,
261
- packageName: data.package.name,
262
- });
263
- const sourcePath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
264
- const targetPath = (0, path_1.resolve)((0, path_1.join)(data.mirrorRepositoryConfig.outPath, snapshotName));
252
+ async copy(data) {
253
+ const sourceFs = (0, client_1.createFs)(this.config.backend);
254
+ const targetFs = (0, client_1.createFs)(data.mirrorRepositoryConfig.backend);
255
+ const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
265
256
  if (data.options.verbose)
266
- (0, cli_1.logExec)(`Copying backup files to ${targetPath}`);
267
- await (0, promises_1.mkdir)(targetPath, { recursive: true });
268
- await (0, fs_1.ensureEmptyDir)(targetPath);
269
- const scanner = await (0, fs_1.createFileScanner)({
270
- onProgress: data.onProgress,
271
- glob: {
272
- include: ["**/*"],
273
- cwd: sourcePath,
274
- },
275
- });
276
- const entryPaths = [];
277
- await scanner.start(async (entry) => {
278
- entryPaths.push(entry.path);
279
- return true;
280
- });
281
- for (const entryPath of entryPaths) {
282
- const sourceFile = (0, path_1.join)(sourcePath, entryPath);
283
- const targetFile = (0, path_1.join)(targetPath, entryPath);
284
- await scanner.progress("Copying", entryPath);
285
- await (0, promises_1.mkdir)((0, path_1.dirname)(targetFile), { recursive: true });
286
- await (0, promises_1.cp)(sourceFile, targetFile);
257
+ (0, cli_1.logExec)(`Copying backup files to ${data.mirrorRepositoryConfig.backend}`);
258
+ await targetFs.mkdir(snapshotName);
259
+ await targetFs.ensureEmptyDir(snapshotName);
260
+ const entries = await sourceFs.readdir(snapshotName);
261
+ const total = entries.length;
262
+ let current = 0;
263
+ for (const entry of entries) {
264
+ data.onProgress({
265
+ absolute: {
266
+ current,
267
+ description: "Copying",
268
+ payload: entry,
269
+ total,
270
+ percent: (0, math_1.progressPercent)(total, current),
271
+ },
272
+ });
273
+ current++;
274
+ const sourceEntry = `${snapshotName}/${entry}`;
275
+ if (targetFs.isLocal()) {
276
+ await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
277
+ }
278
+ else {
279
+ const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "remote-copy", entry);
280
+ const tempFile = (0, path_1.join)(tempDir, entry);
281
+ try {
282
+ await sourceFs.download(sourceEntry, tempFile);
283
+ await targetFs.upload(tempFile, sourceEntry);
284
+ }
285
+ finally {
286
+ await (0, fs_1.tryRm)(tempFile);
287
+ }
288
+ }
287
289
  }
288
- await scanner.end();
289
290
  }
290
- async onRestore(data) {
291
- const relRestorePath = data.targetPath ?? data.package.restorePath;
291
+ async restore(data) {
292
+ const fs = (0, client_1.createFs)(this.config.backend);
293
+ const relRestorePath = data.snapshotPath;
292
294
  (0, assert_1.ok)(relRestorePath);
293
295
  const restorePath = (0, path_1.resolve)(relRestorePath);
294
- const [snapshot] = await this.onSnapshots({
296
+ const [snapshot] = await this.fetchSnapshots({
295
297
  options: {
296
298
  ids: [data.options.snapshotId],
297
299
  },
298
300
  });
299
301
  if (!snapshot)
300
302
  throw new AppError_1.AppError("Snapshot not found");
301
- const snapshotName = DatatruckRepository.buildSnapshotName({
302
- snapshotId: data.snapshot.id,
303
- snapshotDate: data.snapshot.date,
304
- packageName: data.package.name,
305
- });
306
- const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
307
- const metaPath = (0, path_1.join)(sourcePath, "meta.json");
308
- const meta = await DatatruckRepository.parseMetaData(metaPath);
309
- const scanner = await (0, fs_1.createFileScanner)({
310
- onProgress: data.onProgress,
311
- glob: {
312
- cwd: sourcePath,
313
- include: ["**/*"],
314
- },
315
- });
316
- const tarFiles = [];
303
+ const snapshotName = DatatruckRepository.buildSnapshotName(snapshot, data.package);
304
+ const meta = await DatatruckRepository.parseMetaData(await fs.readFile(`${snapshotName}/meta.json`));
305
+ const progress = (0, fs_1.createProgress)({ onProgress: data.onProgress });
306
+ progress.update("Scanning files");
307
+ const entries = (await fs.readdir(snapshotName)).filter((v) => v.endsWith(".tar") || v.endsWith(".tar.gz"));
317
308
  const tarStats = meta?.tarStats || {};
318
- await scanner.start(async (entry) => {
319
- const path = (0, path_1.join)(sourcePath, entry.name);
320
- const isTar = entry.name.endsWith(".tar");
321
- const isTarGz = entry.name.endsWith(".tar.gz");
322
- if (isTar || isTarGz) {
323
- tarFiles.push(path);
324
- if (typeof tarStats[entry.name]?.files === "number") {
325
- scanner.total += tarStats[entry.name].files;
326
- }
327
- else {
328
- scanner.progress("Scanning", entry.name, false);
329
- const selfTarStats = (tarStats[entry.name] = { files: 0 });
330
- await (0, tar_1.listTar)({
331
- input: path,
332
- verbose: data.options.verbose,
333
- onEntry: () => {
334
- scanner.total++;
335
- selfTarStats.files++;
336
- },
337
- });
338
- }
339
- }
340
- return false;
341
- });
309
+ for (const file in tarStats)
310
+ progress.total += tarStats[file].files;
311
+ progress.update(`Scanned files: ${progress.total}`);
342
312
  if (data.options.verbose)
343
313
  (0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
344
- for (const tarFile of tarFiles) {
345
- const entryName = (0, path_1.basename)(tarFile);
346
- await (0, tar_1.extractTar)({
347
- total: tarStats[entryName].files,
348
- input: tarFile,
349
- output: restorePath,
350
- decompress: tarFile.endsWith(".tar.gz"),
351
- verbose: data.options.verbose,
352
- onEntry: async (data) => await scanner.progress(tarFile.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
353
- });
314
+ for (const entry of entries) {
315
+ let tempEntry;
316
+ try {
317
+ const sourceEntry = `${snapshotName}/${entry}`;
318
+ if (!fs.isLocal()) {
319
+ const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "restore", "remote-fs", entry);
320
+ tempEntry = `${tempDir}/${entry}`;
321
+ await fs.download(sourceEntry, tempEntry);
322
+ }
323
+ await (0, tar_1.extractTar)({
324
+ total: tarStats[entry].files,
325
+ input: tempEntry ?? fs.resolvePath(sourceEntry),
326
+ output: restorePath,
327
+ decompress: entry.endsWith(".tar.gz"),
328
+ verbose: data.options.verbose,
329
+ onEntry: (data) => progress.update(entry.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
330
+ });
331
+ }
332
+ finally {
333
+ if (tempEntry)
334
+ await (0, fs_1.tryRm)(tempEntry);
335
+ }
354
336
  }
355
- await scanner.end();
337
+ progress.update("Finished");
356
338
  }
357
339
  }
358
340
  exports.DatatruckRepository = DatatruckRepository;
@@ -1,4 +1,4 @@
1
- import { RepositoryAbstract, BackupDataType, InitDataType, RestoreDataType, SnapshotsDataType, SnapshotResultType, SnapshotTagEnum, SnapshotTagObjectType, PruneDataType, CopyBackupType } from "./RepositoryAbstract";
1
+ import { RepositoryAbstract, RepoBackupData, RepoInitData, RepoRestoreData, RepoFetchSnapshotsData, Snapshot, SnapshotTagEnum, SnapshotTagObjectType, RepoPruneData, RepoCopyData } from "./RepositoryAbstract";
2
2
  import { JSONSchema7 } from "json-schema";
3
3
  export type GitRepositoryConfigType = {
4
4
  repo: string;
@@ -10,7 +10,8 @@ export declare const gitRepositoryDefinition: JSONSchema7;
10
10
  export declare const gitPackageRepositoryDefinition: JSONSchema7;
11
11
  export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfigType> {
12
12
  static refPrefix: string;
13
- onGetSource(): string;
13
+ getSource(): string;
14
+ fetchDiskStats(config: GitRepositoryConfigType): Promise<import("../utils/fs").DiskStats | undefined>;
14
15
  static buildSnapshotTagName(tag: Pick<SnapshotTagObjectType, SnapshotTagEnum.PACKAGE | SnapshotTagEnum.ID>): string;
15
16
  static buildSnapshotTag(tag: SnapshotTagObjectType): {
16
17
  name: string;
@@ -21,10 +22,10 @@ export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfi
21
22
  tags: string[];
22
23
  }) | null;
23
24
  static buildBranchName(packageName: string): string;
24
- onInit(data: InitDataType): Promise<void>;
25
- onPrune(data: PruneDataType): Promise<void>;
26
- onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
27
- onBackup(data: BackupDataType<GitPackageRepositoryConfigType>): Promise<void>;
28
- onCopyBackup(data: CopyBackupType<GitRepositoryConfigType>): Promise<void>;
29
- onRestore(data: RestoreDataType<GitPackageRepositoryConfigType>): Promise<void>;
25
+ init(data: RepoInitData): Promise<void>;
26
+ prune(data: RepoPruneData): Promise<void>;
27
+ fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
28
+ backup(data: RepoBackupData<GitPackageRepositoryConfigType>): Promise<void>;
29
+ copy(data: RepoCopyData<GitRepositoryConfigType>): Promise<void>;
30
+ restore(data: RepoRestoreData<GitPackageRepositoryConfigType>): Promise<void>;
30
31
  }
@@ -4,14 +4,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.GitRepository = exports.gitPackageRepositoryDefinition = exports.gitRepositoryDefinition = exports.gitRepositoryName = void 0;
7
- const AppError_1 = require("../Error/AppError");
8
7
  const Git_1 = require("../utils/Git");
9
8
  const cli_1 = require("../utils/cli");
10
9
  const paths_1 = require("../utils/datatruck/paths");
11
10
  const fs_1 = require("../utils/fs");
12
11
  const string_1 = require("../utils/string");
12
+ const temp_1 = require("../utils/temp");
13
13
  const RepositoryAbstract_1 = require("./RepositoryAbstract");
14
- const assert_1 = require("assert");
15
14
  const fast_glob_1 = __importDefault(require("fast-glob"));
16
15
  const promises_1 = require("fs/promises");
17
16
  const micromatch_1 = require("micromatch");
@@ -33,9 +32,13 @@ exports.gitPackageRepositoryDefinition = {
33
32
  };
34
33
  class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
35
34
  static refPrefix = "dt";
36
- onGetSource() {
35
+ getSource() {
37
36
  return this.config.repo;
38
37
  }
38
+ async fetchDiskStats(config) {
39
+ if ((0, fs_1.isLocalDir)(config.repo))
40
+ return await (0, fs_1.fetchDiskStats)(config.repo);
41
+ }
39
42
  static buildSnapshotTagName(tag) {
40
43
  return `${GitRepository.refPrefix}/${tag.package}/${tag.id}`;
41
44
  }
@@ -56,9 +59,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
56
59
  static buildBranchName(packageName) {
57
60
  return `${GitRepository.refPrefix}/${packageName}`;
58
61
  }
59
- async onInit(data) {
62
+ async init(data) {
60
63
  const git = new Git_1.Git({
61
- dir: (0, fs_1.tmpDir)(GitRepository.name + "-snapshot"),
64
+ dir: (0, temp_1.tmpDir)(exports.gitRepositoryName, "repository", "init"),
62
65
  log: data.options.verbose,
63
66
  });
64
67
  if (await git.canBeInit(this.config.repo)) {
@@ -83,9 +86,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
83
86
  await git.push({ branchName });
84
87
  }
85
88
  }
86
- async onPrune(data) {
89
+ async prune(data) {
87
90
  const git = new Git_1.Git({
88
- dir: await this.mkTmpDir(GitRepository.name + "-snapshot"),
91
+ dir: await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "prune"),
89
92
  log: data.options.verbose,
90
93
  });
91
94
  const branchName = GitRepository.buildBranchName(data.snapshot.packageName);
@@ -106,9 +109,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
106
109
  await git.exec(["push", "origin", branchName, "--force-with-lease"]);
107
110
  await git.exec(["push", "--delete", "origin", data.snapshot.originalId]);
108
111
  }
109
- async onSnapshots(data) {
112
+ async fetchSnapshots(data) {
110
113
  const git = new Git_1.Git({
111
- dir: await this.mkTmpDir(GitRepository.name + "-snapshot"),
114
+ dir: await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "snapshots"),
112
115
  log: data.options.verbose,
113
116
  });
114
117
  const pkgPatterns = (0, string_1.makePathPatterns)(data.options.packageNames);
@@ -143,13 +146,10 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
143
146
  }, [])
144
147
  .sort((a, b) => a.date.localeCompare(b.date));
145
148
  }
146
- async onBackup(data) {
149
+ async backup(data) {
147
150
  const pkg = data.package;
148
- const sourcePath = data.targetPath ?? pkg.path;
149
- (0, assert_1.ok)(typeof sourcePath === "string");
150
- if (!(await (0, fs_1.existsDir)(sourcePath)))
151
- throw new AppError_1.AppError(`Package path not exists: ${sourcePath}`);
152
- const tmpPath = await this.mkTmpDir(GitRepository.name + "-backup");
151
+ const path = pkg.path;
152
+ const tmpPath = await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "backup");
153
153
  const branchName = GitRepository.buildBranchName(data.package.name);
154
154
  const git = new Git_1.Git({
155
155
  dir: tmpPath,
@@ -169,7 +169,7 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
169
169
  const backupPathsOptions = {
170
170
  package: data.package,
171
171
  snapshot: data.snapshot,
172
- targetPath: sourcePath,
172
+ path: path,
173
173
  verbose: data.options.verbose,
174
174
  };
175
175
  const include = await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions);
@@ -177,13 +177,13 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
177
177
  ? await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions)
178
178
  : undefined;
179
179
  const stream = await (0, fast_glob_1.default)(include, {
180
- cwd: sourcePath,
180
+ cwd: path,
181
181
  ignore: exclude,
182
182
  dot: true,
183
183
  });
184
184
  let files = 0;
185
185
  for await (const entry of stream) {
186
- const source = (0, path_1.join)(sourcePath, entry);
186
+ const source = (0, path_1.join)(path, entry);
187
187
  const target = (0, path_1.join)(tmpPath, entry);
188
188
  const dir = (0, path_1.dirname)(target);
189
189
  if (!createdPaths.includes(dir)) {
@@ -215,16 +215,13 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
215
215
  await git.addTag(meta.name, meta.message);
216
216
  await git.push({ branchName });
217
217
  await git.pushTags();
218
- await (0, promises_1.rm)(tmpPath, {
219
- recursive: true,
220
- });
218
+ await (0, promises_1.rm)(tmpPath, { recursive: true });
221
219
  }
222
- onCopyBackup(data) {
220
+ copy(data) {
223
221
  throw new Error("Method not implemented.");
224
222
  }
225
- async onRestore(data) {
226
- const restorePath = data.targetPath ?? data.package.restorePath;
227
- (0, assert_1.ok)(restorePath);
223
+ async restore(data) {
224
+ const restorePath = data.snapshotPath;
228
225
  const tagName = GitRepository.buildSnapshotTagName({
229
226
  id: data.snapshot.id,
230
227
  package: data.package.name,