@datatruck/cli 0.26.2 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/BackupAction.d.ts +69 -34
- package/Action/BackupAction.js +284 -244
- package/Action/CleanCacheAction.d.ts +8 -4
- package/Action/CleanCacheAction.js +8 -5
- package/Action/ConfigAction.d.ts +12 -5
- package/Action/ConfigAction.js +14 -18
- package/Action/CopyAction.d.ts +49 -0
- package/Action/CopyAction.js +144 -0
- package/Action/InitAction.d.ts +3 -3
- package/Action/InitAction.js +9 -9
- package/Action/PruneAction.d.ts +9 -9
- package/Action/PruneAction.js +39 -23
- package/Action/RestoreAction.d.ts +48 -23
- package/Action/RestoreAction.js +158 -195
- package/Action/SnapshotsAction.d.ts +8 -8
- package/Action/SnapshotsAction.js +8 -8
- package/CHANGELOG.md +495 -0
- package/Command/BackupCommand.d.ts +6 -4
- package/Command/BackupCommand.js +9 -26
- package/Command/CleanCacheCommand.d.ts +4 -4
- package/Command/CleanCacheCommand.js +26 -5
- package/Command/CommandAbstract.d.ts +10 -7
- package/Command/CommandAbstract.js +4 -1
- package/Command/ConfigCommand.d.ts +6 -9
- package/Command/ConfigCommand.js +13 -8
- package/Command/CopyCommand.d.ts +15 -0
- package/Command/CopyCommand.js +61 -0
- package/Command/InitCommand.d.ts +4 -4
- package/Command/InitCommand.js +11 -15
- package/Command/PruneCommand.d.ts +3 -3
- package/Command/PruneCommand.js +13 -12
- package/Command/RestoreCommand.js +9 -17
- package/Command/SnapshotsCommand.d.ts +4 -4
- package/Command/SnapshotsCommand.js +16 -15
- package/Command/StartServerCommand.d.ts +6 -0
- package/Command/StartServerCommand.js +24 -0
- package/Config/Config.d.ts +11 -0
- package/Config/Config.js +43 -0
- package/Config/PrunePolicyConfig.d.ts +2 -2
- package/Factory/CommandFactory.d.ts +29 -33
- package/Factory/CommandFactory.js +32 -53
- package/Factory/RepositoryFactory.d.ts +1 -1
- package/Factory/RepositoryFactory.js +3 -3
- package/Factory/TaskFactory.d.ts +1 -1
- package/Factory/TaskFactory.js +3 -3
- package/Repository/DatatruckRepository.d.ts +17 -16
- package/Repository/DatatruckRepository.js +131 -149
- package/Repository/GitRepository.d.ts +9 -8
- package/Repository/GitRepository.js +22 -25
- package/Repository/RepositoryAbstract.d.ts +39 -37
- package/Repository/RepositoryAbstract.js +4 -5
- package/Repository/ResticRepository.d.ts +9 -8
- package/Repository/ResticRepository.js +30 -28
- package/Task/GitTask.d.ts +6 -7
- package/Task/GitTask.js +24 -30
- package/Task/MariadbTask.d.ts +4 -5
- package/Task/MariadbTask.js +26 -32
- package/Task/MssqlTask.d.ts +5 -3
- package/Task/MssqlTask.js +11 -12
- package/Task/MysqlDumpTask.d.ts +10 -3
- package/Task/MysqlDumpTask.js +107 -31
- package/Task/ScriptTask.d.ts +23 -18
- package/Task/ScriptTask.js +34 -24
- package/Task/SqlDumpTaskAbstract.d.ts +8 -3
- package/Task/SqlDumpTaskAbstract.js +31 -19
- package/Task/TaskAbstract.d.ts +24 -25
- package/Task/TaskAbstract.js +6 -10
- package/cli.js +14 -5
- package/config.schema.json +124 -3
- package/package.json +4 -5
- package/utils/DataFormat.d.ts +23 -12
- package/utils/DataFormat.js +36 -14
- package/utils/cli.d.ts +2 -9
- package/utils/cli.js +9 -52
- package/utils/datatruck/client.d.ts +24 -0
- package/utils/datatruck/client.js +99 -0
- package/utils/datatruck/config.d.ts +8 -6
- package/utils/datatruck/config.js +18 -3
- package/utils/datatruck/paths.d.ts +5 -9
- package/utils/datatruck/paths.js +2 -2
- package/utils/datatruck/server.d.ts +21 -0
- package/utils/datatruck/server.js +96 -0
- package/utils/datatruck/snapshot.d.ts +2 -2
- package/utils/date.d.ts +7 -3
- package/utils/date.js +22 -14
- package/utils/fs.d.ts +27 -15
- package/utils/fs.js +110 -62
- package/utils/http.d.ts +21 -0
- package/utils/http.js +154 -0
- package/utils/list.d.ts +64 -0
- package/utils/list.js +145 -0
- package/utils/mysql.d.ts +2 -0
- package/utils/mysql.js +21 -2
- package/utils/process.d.ts +1 -0
- package/utils/process.js +24 -31
- package/utils/progress.d.ts +33 -0
- package/utils/progress.js +113 -0
- package/utils/steps.d.ts +11 -0
- package/utils/steps.js +22 -10
- package/utils/stream.d.ts +7 -0
- package/utils/stream.js +10 -0
- package/utils/string.d.ts +0 -1
- package/utils/string.js +1 -13
- package/utils/tar.d.ts +10 -3
- package/utils/tar.js +70 -44
- package/utils/temp.d.ts +26 -0
- package/utils/temp.js +133 -0
- package/utils/virtual-fs.d.ts +37 -0
- package/utils/virtual-fs.js +65 -0
- package/Action/BackupSessionsAction.d.ts +0 -13
- package/Action/BackupSessionsAction.js +0 -18
- package/Action/RestoreSessionsAction.d.ts +0 -13
- package/Action/RestoreSessionsAction.js +0 -18
- package/Command/BackupSessionsCommand.d.ts +0 -12
- package/Command/BackupSessionsCommand.js +0 -92
- package/Command/RestoreSessionsCommand.d.ts +0 -12
- package/Command/RestoreSessionsCommand.js +0 -91
- package/Decorator/EntityDecorator.d.ts +0 -11
- package/Decorator/EntityDecorator.js +0 -17
- package/Entity/BackupSessionEntity.d.ts +0 -6
- package/Entity/BackupSessionEntity.js +0 -25
- package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
- package/Entity/BackupSessionRepositoryEntity.js +0 -25
- package/Entity/BackupSessionTaskEntity.d.ts +0 -5
- package/Entity/BackupSessionTaskEntity.js +0 -24
- package/Entity/CrudEntityAbstract.d.ts +0 -5
- package/Entity/CrudEntityAbstract.js +0 -9
- package/Entity/RestoreSessionEntity.d.ts +0 -5
- package/Entity/RestoreSessionEntity.js +0 -24
- package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
- package/Entity/RestoreSessionRepositoryEntity.js +0 -25
- package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
- package/Entity/RestoreSessionTaskEntity.js +0 -24
- package/Entity/StateEntityAbstract.d.ts +0 -9
- package/Entity/StateEntityAbstract.js +0 -12
- package/Factory/EntityFactory.d.ts +0 -6
- package/Factory/EntityFactory.js +0 -40
- package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
- package/SessionDriver/ConsoleSessionDriver.js +0 -208
- package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
- package/SessionDriver/SessionDriverAbstract.js +0 -28
- package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
- package/SessionDriver/SqliteSessionDriver.js +0 -173
- package/SessionManager/BackupSessionManager.d.ts +0 -45
- package/SessionManager/BackupSessionManager.js +0 -218
- package/SessionManager/RestoreSessionManager.d.ts +0 -47
- package/SessionManager/RestoreSessionManager.js +0 -218
- package/SessionManager/SessionManagerAbstract.d.ts +0 -18
- package/SessionManager/SessionManagerAbstract.js +0 -36
- package/migrations/001-initial.sql +0 -98
- package/utils/entity.d.ts +0 -4
- package/utils/entity.js +0 -10
|
@@ -4,10 +4,13 @@ exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exp
|
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
6
6
|
const cli_1 = require("../utils/cli");
|
|
7
|
+
const client_1 = require("../utils/datatruck/client");
|
|
7
8
|
const paths_1 = require("../utils/datatruck/paths");
|
|
8
9
|
const fs_1 = require("../utils/fs");
|
|
10
|
+
const math_1 = require("../utils/math");
|
|
9
11
|
const string_1 = require("../utils/string");
|
|
10
12
|
const tar_1 = require("../utils/tar");
|
|
13
|
+
const temp_1 = require("../utils/temp");
|
|
11
14
|
const RepositoryAbstract_1 = require("./RepositoryAbstract");
|
|
12
15
|
const assert_1 = require("assert");
|
|
13
16
|
const promises_1 = require("fs/promises");
|
|
@@ -16,10 +19,10 @@ const path_1 = require("path");
|
|
|
16
19
|
exports.datatruckRepositoryName = "datatruck";
|
|
17
20
|
exports.datatruckRepositoryDefinition = {
|
|
18
21
|
type: "object",
|
|
19
|
-
required: ["
|
|
22
|
+
required: ["backend"],
|
|
20
23
|
additionalProperties: false,
|
|
21
24
|
properties: {
|
|
22
|
-
|
|
25
|
+
backend: { type: "string" },
|
|
23
26
|
compress: {
|
|
24
27
|
anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
|
|
25
28
|
},
|
|
@@ -52,10 +55,10 @@ exports.datatruckPackageRepositoryDefinition = {
|
|
|
52
55
|
};
|
|
53
56
|
class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
54
57
|
static zipBasenameTpl = `.*.dd.tar.gz`;
|
|
55
|
-
static buildSnapshotName(
|
|
56
|
-
const date =
|
|
57
|
-
const pkgName = encodeURIComponent(
|
|
58
|
-
const snapshotShortId =
|
|
58
|
+
static buildSnapshotName(snapshot, pkg) {
|
|
59
|
+
const date = snapshot.date.replace(/:/g, "-");
|
|
60
|
+
const pkgName = encodeURIComponent(pkg.name).replace(/%40/g, "@");
|
|
61
|
+
const snapshotShortId = snapshot.id.slice(0, 8);
|
|
59
62
|
return `${date}_${pkgName}_${snapshotShortId}`;
|
|
60
63
|
}
|
|
61
64
|
static parseSnapshotName(name) {
|
|
@@ -69,49 +72,36 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
69
72
|
packageName = decodeURIComponent(packageName);
|
|
70
73
|
return { snapshotDate, packageName, snapshotShortId, sourcePath: name };
|
|
71
74
|
}
|
|
72
|
-
|
|
73
|
-
return (
|
|
75
|
+
static async parseMetaData(data) {
|
|
76
|
+
return JSON.parse(data.toString());
|
|
74
77
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
try {
|
|
78
|
-
contents = await (0, promises_1.readFile)(path);
|
|
79
|
-
}
|
|
80
|
-
catch (error) {
|
|
81
|
-
if ((0, fs_1.isNotFoundError)(error))
|
|
82
|
-
return;
|
|
83
|
-
throw error;
|
|
84
|
-
}
|
|
85
|
-
return JSON.parse(contents.toString());
|
|
86
|
-
}
|
|
87
|
-
static stringifyMetaData(data) {
|
|
88
|
-
return JSON.stringify(data);
|
|
78
|
+
getSource() {
|
|
79
|
+
return this.config.backend;
|
|
89
80
|
}
|
|
90
|
-
|
|
91
|
-
|
|
81
|
+
fetchDiskStats(config) {
|
|
82
|
+
const fs = (0, client_1.createFs)(config.backend);
|
|
83
|
+
return fs.fetchDiskStats(".");
|
|
92
84
|
}
|
|
93
|
-
async
|
|
94
|
-
|
|
85
|
+
async init(data) {
|
|
86
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
87
|
+
await fs.mkdir(".");
|
|
95
88
|
}
|
|
96
|
-
async
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
packageName: data.snapshot.packageName,
|
|
89
|
+
async prune(data) {
|
|
90
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
91
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, {
|
|
92
|
+
name: data.snapshot.packageName,
|
|
101
93
|
});
|
|
102
|
-
const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
103
94
|
if (data.options.verbose)
|
|
104
|
-
(0, cli_1.logExec)(`Deleting ${
|
|
105
|
-
if (await
|
|
106
|
-
await (
|
|
107
|
-
recursive: true,
|
|
108
|
-
});
|
|
95
|
+
(0, cli_1.logExec)(`Deleting ${fs.resolvePath(snapshotName)}`);
|
|
96
|
+
if (await fs.existsDir(snapshotName))
|
|
97
|
+
await fs.rmAll(snapshotName);
|
|
109
98
|
}
|
|
110
|
-
async
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
99
|
+
async fetchSnapshots(data) {
|
|
100
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
101
|
+
if (!(await fs.existsDir(".")))
|
|
102
|
+
throw new Error(`Repository (${this.repository.name}) out path does not exist: ${fs.resolvePath(".")}`);
|
|
114
103
|
const snapshots = [];
|
|
104
|
+
const snapshotNames = await fs.readdir(".");
|
|
115
105
|
const packagePatterns = (0, string_1.makePathPatterns)(data.options.packageNames);
|
|
116
106
|
const taskPatterns = (0, string_1.makePathPatterns)(data.options.packageTaskNames);
|
|
117
107
|
for (const snapshotName of snapshotNames) {
|
|
@@ -124,8 +114,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
124
114
|
if (data.options.ids &&
|
|
125
115
|
!data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
|
|
126
116
|
continue;
|
|
127
|
-
const
|
|
128
|
-
const meta = await DatatruckRepository.parseMetaData(
|
|
117
|
+
const metaData = await fs.readFileIfExists(`${snapshotName}/meta.json`);
|
|
118
|
+
const meta = !!metaData && (await DatatruckRepository.parseMetaData(metaData));
|
|
129
119
|
if (!meta)
|
|
130
120
|
continue;
|
|
131
121
|
if (taskPatterns && !(0, string_1.checkMatch)(meta.task, taskPatterns))
|
|
@@ -148,27 +138,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
148
138
|
}
|
|
149
139
|
return snapshots;
|
|
150
140
|
}
|
|
151
|
-
async
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
const outPath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
|
|
141
|
+
async backup(data) {
|
|
142
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
143
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
144
|
+
const outPath = fs.isLocal()
|
|
145
|
+
? fs.resolvePath(snapshotName)
|
|
146
|
+
: await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "fs-remote");
|
|
158
147
|
const pkg = data.package;
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
await (0, promises_1.mkdir)(outPath, { recursive: true });
|
|
148
|
+
const path = pkg.path;
|
|
149
|
+
await fs.mkdir(snapshotName);
|
|
162
150
|
const backupPathsOptions = {
|
|
163
151
|
package: data.package,
|
|
164
152
|
snapshot: data.snapshot,
|
|
165
|
-
|
|
153
|
+
path: path,
|
|
166
154
|
verbose: data.options.verbose,
|
|
167
155
|
};
|
|
168
156
|
const scanner = await (0, fs_1.createFileScanner)({
|
|
169
157
|
onProgress: data.onProgress,
|
|
170
158
|
glob: {
|
|
171
|
-
cwd:
|
|
159
|
+
cwd: path,
|
|
172
160
|
onlyFiles: false,
|
|
173
161
|
include: await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions),
|
|
174
162
|
ignore: pkg.exclude
|
|
@@ -187,7 +175,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
187
175
|
const packs = [defaultsPack, ...configPacks];
|
|
188
176
|
const defaultsPackIndex = packs.findIndex((p) => p === defaultsPack);
|
|
189
177
|
const stream = (0, fs_1.createWriteStreamPool)({
|
|
190
|
-
path: await
|
|
178
|
+
path: await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "stream-pool"),
|
|
191
179
|
onStreamPath: (key) => `files-${key}.txt`,
|
|
192
180
|
});
|
|
193
181
|
scanner.total++;
|
|
@@ -209,7 +197,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
209
197
|
scanner.total++;
|
|
210
198
|
stream.writeLine(packIndex, ".");
|
|
211
199
|
}
|
|
212
|
-
stream.writeLine(packIndex, entry.path);
|
|
200
|
+
stream.writeLine(packIndex, (0, tar_1.normalizeTarPath)(entry.path));
|
|
213
201
|
return true;
|
|
214
202
|
});
|
|
215
203
|
await stream.end();
|
|
@@ -224,21 +212,28 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
224
212
|
if (includeList) {
|
|
225
213
|
tarStats[packBasename] = {
|
|
226
214
|
files: stream.lines(packIndex),
|
|
215
|
+
size: 0,
|
|
227
216
|
};
|
|
217
|
+
const tarPath = (0, path_1.join)(outPath, packBasename);
|
|
228
218
|
await (0, tar_1.createTar)({
|
|
229
219
|
compress: pack.compress,
|
|
230
220
|
verbose: data.options.verbose,
|
|
231
221
|
includeList,
|
|
232
|
-
path:
|
|
233
|
-
output:
|
|
234
|
-
onEntry: async (data) =>
|
|
222
|
+
path: path,
|
|
223
|
+
output: tarPath,
|
|
224
|
+
onEntry: async (data) => scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
|
|
235
225
|
});
|
|
226
|
+
tarStats[packBasename].size = (await (0, promises_1.stat)(tarPath)).size;
|
|
227
|
+
if (!fs.isLocal()) {
|
|
228
|
+
await fs.upload(tarPath, `${snapshotName}/${packBasename}`);
|
|
229
|
+
await (0, promises_1.rm)(tarPath);
|
|
230
|
+
}
|
|
236
231
|
}
|
|
237
232
|
packIndex++;
|
|
238
233
|
}
|
|
239
|
-
|
|
234
|
+
scanner.end();
|
|
240
235
|
// Meta
|
|
241
|
-
const metaPath = `${
|
|
236
|
+
const metaPath = `${snapshotName}/meta.json`;
|
|
242
237
|
const nodePkg = (0, fs_1.parsePackageFile)();
|
|
243
238
|
const meta = {
|
|
244
239
|
id: data.snapshot.id,
|
|
@@ -247,112 +242,99 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
247
242
|
package: data.package.name,
|
|
248
243
|
task: data.package.task?.name,
|
|
249
244
|
version: nodePkg.version,
|
|
250
|
-
size:
|
|
245
|
+
size: Object.values(tarStats).reduce((total, { size }) => total + size, 0),
|
|
251
246
|
tarStats,
|
|
252
247
|
};
|
|
253
248
|
if (data.options.verbose)
|
|
254
|
-
(0, cli_1.logExec)(`Writing metadata into ${metaPath}`);
|
|
255
|
-
await
|
|
249
|
+
(0, cli_1.logExec)(`Writing metadata into ${fs.resolvePath(metaPath)}`);
|
|
250
|
+
await fs.writeFile(`${snapshotName}/meta.json`, JSON.stringify(meta));
|
|
256
251
|
}
|
|
257
|
-
async
|
|
258
|
-
const
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
packageName: data.package.name,
|
|
262
|
-
});
|
|
263
|
-
const sourcePath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
|
|
264
|
-
const targetPath = (0, path_1.resolve)((0, path_1.join)(data.mirrorRepositoryConfig.outPath, snapshotName));
|
|
252
|
+
async copy(data) {
|
|
253
|
+
const sourceFs = (0, client_1.createFs)(this.config.backend);
|
|
254
|
+
const targetFs = (0, client_1.createFs)(data.mirrorRepositoryConfig.backend);
|
|
255
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
265
256
|
if (data.options.verbose)
|
|
266
|
-
(0, cli_1.logExec)(`Copying backup files to ${
|
|
267
|
-
await
|
|
268
|
-
await
|
|
269
|
-
const
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
const
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
257
|
+
(0, cli_1.logExec)(`Copying backup files to ${data.mirrorRepositoryConfig.backend}`);
|
|
258
|
+
await targetFs.mkdir(snapshotName);
|
|
259
|
+
await targetFs.ensureEmptyDir(snapshotName);
|
|
260
|
+
const entries = await sourceFs.readdir(snapshotName);
|
|
261
|
+
const total = entries.length;
|
|
262
|
+
let current = 0;
|
|
263
|
+
for (const entry of entries) {
|
|
264
|
+
data.onProgress({
|
|
265
|
+
absolute: {
|
|
266
|
+
current,
|
|
267
|
+
description: "Copying",
|
|
268
|
+
payload: entry,
|
|
269
|
+
total,
|
|
270
|
+
percent: (0, math_1.progressPercent)(total, current),
|
|
271
|
+
},
|
|
272
|
+
});
|
|
273
|
+
current++;
|
|
274
|
+
const sourceEntry = `${snapshotName}/${entry}`;
|
|
275
|
+
if (targetFs.isLocal()) {
|
|
276
|
+
await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "remote-copy", entry);
|
|
280
|
+
const tempFile = (0, path_1.join)(tempDir, entry);
|
|
281
|
+
try {
|
|
282
|
+
await sourceFs.download(sourceEntry, tempFile);
|
|
283
|
+
await targetFs.upload(tempFile, sourceEntry);
|
|
284
|
+
}
|
|
285
|
+
finally {
|
|
286
|
+
await (0, fs_1.tryRm)(tempFile);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
287
289
|
}
|
|
288
|
-
await scanner.end();
|
|
289
290
|
}
|
|
290
|
-
async
|
|
291
|
-
const
|
|
291
|
+
async restore(data) {
|
|
292
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
293
|
+
const relRestorePath = data.snapshotPath;
|
|
292
294
|
(0, assert_1.ok)(relRestorePath);
|
|
293
295
|
const restorePath = (0, path_1.resolve)(relRestorePath);
|
|
294
|
-
const [snapshot] = await this.
|
|
296
|
+
const [snapshot] = await this.fetchSnapshots({
|
|
295
297
|
options: {
|
|
296
298
|
ids: [data.options.snapshotId],
|
|
297
299
|
},
|
|
298
300
|
});
|
|
299
301
|
if (!snapshot)
|
|
300
302
|
throw new AppError_1.AppError("Snapshot not found");
|
|
301
|
-
const snapshotName = DatatruckRepository.buildSnapshotName(
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
307
|
-
const metaPath = (0, path_1.join)(sourcePath, "meta.json");
|
|
308
|
-
const meta = await DatatruckRepository.parseMetaData(metaPath);
|
|
309
|
-
const scanner = await (0, fs_1.createFileScanner)({
|
|
310
|
-
onProgress: data.onProgress,
|
|
311
|
-
glob: {
|
|
312
|
-
cwd: sourcePath,
|
|
313
|
-
include: ["**/*"],
|
|
314
|
-
},
|
|
315
|
-
});
|
|
316
|
-
const tarFiles = [];
|
|
303
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(snapshot, data.package);
|
|
304
|
+
const meta = await DatatruckRepository.parseMetaData(await fs.readFile(`${snapshotName}/meta.json`));
|
|
305
|
+
const progress = (0, fs_1.createProgress)({ onProgress: data.onProgress });
|
|
306
|
+
progress.update("Scanning files");
|
|
307
|
+
const entries = (await fs.readdir(snapshotName)).filter((v) => v.endsWith(".tar") || v.endsWith(".tar.gz"));
|
|
317
308
|
const tarStats = meta?.tarStats || {};
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
const isTarGz = entry.name.endsWith(".tar.gz");
|
|
322
|
-
if (isTar || isTarGz) {
|
|
323
|
-
tarFiles.push(path);
|
|
324
|
-
if (typeof tarStats[entry.name]?.files === "number") {
|
|
325
|
-
scanner.total += tarStats[entry.name].files;
|
|
326
|
-
}
|
|
327
|
-
else {
|
|
328
|
-
scanner.progress("Scanning", entry.name, false);
|
|
329
|
-
const selfTarStats = (tarStats[entry.name] = { files: 0 });
|
|
330
|
-
await (0, tar_1.listTar)({
|
|
331
|
-
input: path,
|
|
332
|
-
verbose: data.options.verbose,
|
|
333
|
-
onEntry: () => {
|
|
334
|
-
scanner.total++;
|
|
335
|
-
selfTarStats.files++;
|
|
336
|
-
},
|
|
337
|
-
});
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
return false;
|
|
341
|
-
});
|
|
309
|
+
for (const file in tarStats)
|
|
310
|
+
progress.total += tarStats[file].files;
|
|
311
|
+
progress.update(`Scanned files: ${progress.total}`);
|
|
342
312
|
if (data.options.verbose)
|
|
343
313
|
(0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
|
|
344
|
-
for (const
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
314
|
+
for (const entry of entries) {
|
|
315
|
+
let tempEntry;
|
|
316
|
+
try {
|
|
317
|
+
const sourceEntry = `${snapshotName}/${entry}`;
|
|
318
|
+
if (!fs.isLocal()) {
|
|
319
|
+
const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "restore", "remote-fs", entry);
|
|
320
|
+
tempEntry = `${tempDir}/${entry}`;
|
|
321
|
+
await fs.download(sourceEntry, tempEntry);
|
|
322
|
+
}
|
|
323
|
+
await (0, tar_1.extractTar)({
|
|
324
|
+
total: tarStats[entry].files,
|
|
325
|
+
input: tempEntry ?? fs.resolvePath(sourceEntry),
|
|
326
|
+
output: restorePath,
|
|
327
|
+
decompress: entry.endsWith(".tar.gz"),
|
|
328
|
+
verbose: data.options.verbose,
|
|
329
|
+
onEntry: (data) => progress.update(entry.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
finally {
|
|
333
|
+
if (tempEntry)
|
|
334
|
+
await (0, fs_1.tryRm)(tempEntry);
|
|
335
|
+
}
|
|
354
336
|
}
|
|
355
|
-
|
|
337
|
+
progress.update("Finished");
|
|
356
338
|
}
|
|
357
339
|
}
|
|
358
340
|
exports.DatatruckRepository = DatatruckRepository;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RepositoryAbstract,
|
|
1
|
+
import { RepositoryAbstract, RepoBackupData, RepoInitData, RepoRestoreData, RepoFetchSnapshotsData, Snapshot, SnapshotTagEnum, SnapshotTagObjectType, RepoPruneData, RepoCopyData } from "./RepositoryAbstract";
|
|
2
2
|
import { JSONSchema7 } from "json-schema";
|
|
3
3
|
export type GitRepositoryConfigType = {
|
|
4
4
|
repo: string;
|
|
@@ -10,7 +10,8 @@ export declare const gitRepositoryDefinition: JSONSchema7;
|
|
|
10
10
|
export declare const gitPackageRepositoryDefinition: JSONSchema7;
|
|
11
11
|
export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfigType> {
|
|
12
12
|
static refPrefix: string;
|
|
13
|
-
|
|
13
|
+
getSource(): string;
|
|
14
|
+
fetchDiskStats(config: GitRepositoryConfigType): Promise<import("../utils/fs").DiskStats | undefined>;
|
|
14
15
|
static buildSnapshotTagName(tag: Pick<SnapshotTagObjectType, SnapshotTagEnum.PACKAGE | SnapshotTagEnum.ID>): string;
|
|
15
16
|
static buildSnapshotTag(tag: SnapshotTagObjectType): {
|
|
16
17
|
name: string;
|
|
@@ -21,10 +22,10 @@ export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfi
|
|
|
21
22
|
tags: string[];
|
|
22
23
|
}) | null;
|
|
23
24
|
static buildBranchName(packageName: string): string;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
init(data: RepoInitData): Promise<void>;
|
|
26
|
+
prune(data: RepoPruneData): Promise<void>;
|
|
27
|
+
fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
28
|
+
backup(data: RepoBackupData<GitPackageRepositoryConfigType>): Promise<void>;
|
|
29
|
+
copy(data: RepoCopyData<GitRepositoryConfigType>): Promise<void>;
|
|
30
|
+
restore(data: RepoRestoreData<GitPackageRepositoryConfigType>): Promise<void>;
|
|
30
31
|
}
|
|
@@ -4,14 +4,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.GitRepository = exports.gitPackageRepositoryDefinition = exports.gitRepositoryDefinition = exports.gitRepositoryName = void 0;
|
|
7
|
-
const AppError_1 = require("../Error/AppError");
|
|
8
7
|
const Git_1 = require("../utils/Git");
|
|
9
8
|
const cli_1 = require("../utils/cli");
|
|
10
9
|
const paths_1 = require("../utils/datatruck/paths");
|
|
11
10
|
const fs_1 = require("../utils/fs");
|
|
12
11
|
const string_1 = require("../utils/string");
|
|
12
|
+
const temp_1 = require("../utils/temp");
|
|
13
13
|
const RepositoryAbstract_1 = require("./RepositoryAbstract");
|
|
14
|
-
const assert_1 = require("assert");
|
|
15
14
|
const fast_glob_1 = __importDefault(require("fast-glob"));
|
|
16
15
|
const promises_1 = require("fs/promises");
|
|
17
16
|
const micromatch_1 = require("micromatch");
|
|
@@ -33,9 +32,13 @@ exports.gitPackageRepositoryDefinition = {
|
|
|
33
32
|
};
|
|
34
33
|
class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
35
34
|
static refPrefix = "dt";
|
|
36
|
-
|
|
35
|
+
getSource() {
|
|
37
36
|
return this.config.repo;
|
|
38
37
|
}
|
|
38
|
+
async fetchDiskStats(config) {
|
|
39
|
+
if ((0, fs_1.isLocalDir)(config.repo))
|
|
40
|
+
return await (0, fs_1.fetchDiskStats)(config.repo);
|
|
41
|
+
}
|
|
39
42
|
static buildSnapshotTagName(tag) {
|
|
40
43
|
return `${GitRepository.refPrefix}/${tag.package}/${tag.id}`;
|
|
41
44
|
}
|
|
@@ -56,9 +59,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
56
59
|
static buildBranchName(packageName) {
|
|
57
60
|
return `${GitRepository.refPrefix}/${packageName}`;
|
|
58
61
|
}
|
|
59
|
-
async
|
|
62
|
+
async init(data) {
|
|
60
63
|
const git = new Git_1.Git({
|
|
61
|
-
dir: (0,
|
|
64
|
+
dir: (0, temp_1.tmpDir)(exports.gitRepositoryName, "repository", "init"),
|
|
62
65
|
log: data.options.verbose,
|
|
63
66
|
});
|
|
64
67
|
if (await git.canBeInit(this.config.repo)) {
|
|
@@ -83,9 +86,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
83
86
|
await git.push({ branchName });
|
|
84
87
|
}
|
|
85
88
|
}
|
|
86
|
-
async
|
|
89
|
+
async prune(data) {
|
|
87
90
|
const git = new Git_1.Git({
|
|
88
|
-
dir: await
|
|
91
|
+
dir: await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "prune"),
|
|
89
92
|
log: data.options.verbose,
|
|
90
93
|
});
|
|
91
94
|
const branchName = GitRepository.buildBranchName(data.snapshot.packageName);
|
|
@@ -106,9 +109,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
106
109
|
await git.exec(["push", "origin", branchName, "--force-with-lease"]);
|
|
107
110
|
await git.exec(["push", "--delete", "origin", data.snapshot.originalId]);
|
|
108
111
|
}
|
|
109
|
-
async
|
|
112
|
+
async fetchSnapshots(data) {
|
|
110
113
|
const git = new Git_1.Git({
|
|
111
|
-
dir: await
|
|
114
|
+
dir: await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "snapshots"),
|
|
112
115
|
log: data.options.verbose,
|
|
113
116
|
});
|
|
114
117
|
const pkgPatterns = (0, string_1.makePathPatterns)(data.options.packageNames);
|
|
@@ -143,13 +146,10 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
143
146
|
}, [])
|
|
144
147
|
.sort((a, b) => a.date.localeCompare(b.date));
|
|
145
148
|
}
|
|
146
|
-
async
|
|
149
|
+
async backup(data) {
|
|
147
150
|
const pkg = data.package;
|
|
148
|
-
const
|
|
149
|
-
(0,
|
|
150
|
-
if (!(await (0, fs_1.existsDir)(sourcePath)))
|
|
151
|
-
throw new AppError_1.AppError(`Package path not exists: ${sourcePath}`);
|
|
152
|
-
const tmpPath = await this.mkTmpDir(GitRepository.name + "-backup");
|
|
151
|
+
const path = pkg.path;
|
|
152
|
+
const tmpPath = await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "backup");
|
|
153
153
|
const branchName = GitRepository.buildBranchName(data.package.name);
|
|
154
154
|
const git = new Git_1.Git({
|
|
155
155
|
dir: tmpPath,
|
|
@@ -169,7 +169,7 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
169
169
|
const backupPathsOptions = {
|
|
170
170
|
package: data.package,
|
|
171
171
|
snapshot: data.snapshot,
|
|
172
|
-
|
|
172
|
+
path: path,
|
|
173
173
|
verbose: data.options.verbose,
|
|
174
174
|
};
|
|
175
175
|
const include = await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions);
|
|
@@ -177,13 +177,13 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
177
177
|
? await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions)
|
|
178
178
|
: undefined;
|
|
179
179
|
const stream = await (0, fast_glob_1.default)(include, {
|
|
180
|
-
cwd:
|
|
180
|
+
cwd: path,
|
|
181
181
|
ignore: exclude,
|
|
182
182
|
dot: true,
|
|
183
183
|
});
|
|
184
184
|
let files = 0;
|
|
185
185
|
for await (const entry of stream) {
|
|
186
|
-
const source = (0, path_1.join)(
|
|
186
|
+
const source = (0, path_1.join)(path, entry);
|
|
187
187
|
const target = (0, path_1.join)(tmpPath, entry);
|
|
188
188
|
const dir = (0, path_1.dirname)(target);
|
|
189
189
|
if (!createdPaths.includes(dir)) {
|
|
@@ -215,16 +215,13 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
215
215
|
await git.addTag(meta.name, meta.message);
|
|
216
216
|
await git.push({ branchName });
|
|
217
217
|
await git.pushTags();
|
|
218
|
-
await (0, promises_1.rm)(tmpPath, {
|
|
219
|
-
recursive: true,
|
|
220
|
-
});
|
|
218
|
+
await (0, promises_1.rm)(tmpPath, { recursive: true });
|
|
221
219
|
}
|
|
222
|
-
|
|
220
|
+
copy(data) {
|
|
223
221
|
throw new Error("Method not implemented.");
|
|
224
222
|
}
|
|
225
|
-
async
|
|
226
|
-
const restorePath = data.
|
|
227
|
-
(0, assert_1.ok)(restorePath);
|
|
223
|
+
async restore(data) {
|
|
224
|
+
const restorePath = data.snapshotPath;
|
|
228
225
|
const tagName = GitRepository.buildSnapshotTagName({
|
|
229
226
|
id: data.snapshot.id,
|
|
230
227
|
package: data.package.name,
|