@datatruck/cli 0.33.0 → 0.34.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.schema.json +3 -0
- package/lib/actions/BackupAction.d.ts +8 -2
- package/lib/actions/BackupAction.js +22 -8
- package/lib/actions/CopyAction.d.ts +4 -1
- package/lib/actions/CopyAction.js +15 -4
- package/lib/commands/BackupCommand.d.ts +2 -0
- package/lib/repositories/DatatruckRepository.d.ts +6 -2
- package/lib/repositories/DatatruckRepository.js +11 -3
- package/lib/repositories/GitRepository.d.ts +6 -2
- package/lib/repositories/GitRepository.js +8 -3
- package/lib/repositories/RepositoryAbstract.d.ts +6 -2
- package/lib/repositories/ResticRepository.d.ts +6 -2
- package/lib/repositories/ResticRepository.js +11 -5
- package/lib/tasks/GitTask.js +32 -50
- package/lib/tasks/MariadbTask.js +43 -56
- package/lib/tasks/MssqlTask.js +5 -11
- package/lib/tasks/MysqlDumpTask.js +4 -4
- package/lib/tasks/PostgresqlDumpTask.d.ts +1 -1
- package/lib/tasks/PostgresqlDumpTask.js +9 -32
- package/lib/tasks/SqlDumpTaskAbstract.d.ts +1 -2
- package/lib/tasks/SqlDumpTaskAbstract.js +1 -1
- package/lib/utils/Git.d.ts +9 -7
- package/lib/utils/Git.js +30 -29
- package/lib/utils/Restic.d.ts +11 -11
- package/lib/utils/Restic.js +72 -90
- package/lib/utils/async-process.d.ts +66 -0
- package/lib/utils/async-process.js +237 -0
- package/lib/utils/async.d.ts +3 -5
- package/lib/utils/async.js +2 -2
- package/lib/utils/datatruck/client.d.ts +3 -1
- package/lib/utils/datatruck/client.js +1 -1
- package/lib/utils/datatruck/config-type.d.ts +1 -0
- package/lib/utils/datatruck/cron-server.js +2 -2
- package/lib/utils/datatruck/report-list.d.ts +2 -0
- package/lib/utils/datatruck/report-list.js +1 -1
- package/lib/utils/fs.d.ts +7 -2
- package/lib/utils/fs.js +0 -1
- package/lib/utils/http.d.ts +3 -1
- package/lib/utils/http.js +6 -1
- package/lib/utils/mysql.d.ts +8 -10
- package/lib/utils/mysql.js +60 -79
- package/lib/utils/process.d.ts +3 -92
- package/lib/utils/process.js +7 -311
- package/lib/utils/spawnSteps.js +9 -10
- package/lib/utils/tar.js +29 -49
- package/lib/utils/virtual-fs.d.ts +6 -2
- package/lib/utils/virtual-fs.js +4 -1
- package/package.json +2 -2
package/config.schema.json
CHANGED
|
@@ -31,12 +31,14 @@ type Context = {
|
|
|
31
31
|
backup: {
|
|
32
32
|
packageName: string;
|
|
33
33
|
repositoryName: string;
|
|
34
|
+
bytes: number;
|
|
34
35
|
};
|
|
35
36
|
cleanup: {};
|
|
36
37
|
copy: {
|
|
37
38
|
packageName: string;
|
|
38
39
|
repositoryName: string;
|
|
39
40
|
mirrorRepositoryName: string;
|
|
41
|
+
bytes: number;
|
|
40
42
|
};
|
|
41
43
|
prune: {
|
|
42
44
|
packageName: string;
|
|
@@ -60,14 +62,18 @@ export declare class BackupAction<TRequired extends boolean = true> {
|
|
|
60
62
|
snapshotPath: string | undefined;
|
|
61
63
|
pkg: PackageConfig;
|
|
62
64
|
onProgress: (data: Progress) => void;
|
|
63
|
-
}): Promise<
|
|
65
|
+
}): Promise<{
|
|
66
|
+
bytes: number;
|
|
67
|
+
}>;
|
|
64
68
|
protected copy(data: {
|
|
65
69
|
repositoryName: string;
|
|
66
70
|
mirrorRepositoryName: string;
|
|
67
71
|
snapshot: PreSnapshot;
|
|
68
72
|
pkg: PackageConfig;
|
|
69
73
|
onProgress: (data: Progress) => void;
|
|
70
|
-
}): Promise<
|
|
74
|
+
}): Promise<{
|
|
75
|
+
bytes: number;
|
|
76
|
+
}>;
|
|
71
77
|
dataFormat(result: Listr3TaskResultEnd<Context>[], options?: {
|
|
72
78
|
streams?: Streams;
|
|
73
79
|
verbose?: number;
|
|
@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.BackupAction = void 0;
|
|
7
7
|
const DataFormat_1 = require("../utils/DataFormat");
|
|
8
|
+
const bytes_1 = require("../utils/bytes");
|
|
8
9
|
const cli_1 = require("../utils/cli");
|
|
9
10
|
const config_1 = require("../utils/datatruck/config");
|
|
10
11
|
const report_list_1 = require("../utils/datatruck/report-list");
|
|
@@ -20,6 +21,7 @@ const assert_1 = require("assert");
|
|
|
20
21
|
const chalk_1 = __importDefault(require("chalk"));
|
|
21
22
|
const crypto_1 = require("crypto");
|
|
22
23
|
const dayjs_1 = __importDefault(require("dayjs"));
|
|
24
|
+
const os_1 = require("os");
|
|
23
25
|
class BackupAction {
|
|
24
26
|
config;
|
|
25
27
|
options;
|
|
@@ -72,7 +74,7 @@ class BackupAction {
|
|
|
72
74
|
await repo.ensureFreeDiskSpace(repoConfig.config, this.config.minFreeDiskSpace);
|
|
73
75
|
const packageConfig = pkg.repositoryConfigs?.find((config) => config.type === repoConfig.type &&
|
|
74
76
|
(!config.names || config.names.includes(repoConfig.name)))?.config;
|
|
75
|
-
await repo.backup({
|
|
77
|
+
return await repo.backup({
|
|
76
78
|
options: this.options,
|
|
77
79
|
snapshot: data.snapshot,
|
|
78
80
|
package: pkg,
|
|
@@ -87,7 +89,7 @@ class BackupAction {
|
|
|
87
89
|
const mirrorRepo = await (0, repository_1.createAndInitRepo)(mirrorRepoConfig, this.options.verbose);
|
|
88
90
|
if (this.config.minFreeDiskSpace)
|
|
89
91
|
await mirrorRepo.ensureFreeDiskSpace(mirrorRepoConfig.config, this.config.minFreeDiskSpace);
|
|
90
|
-
await repo.copy({
|
|
92
|
+
return await repo.copy({
|
|
91
93
|
options: this.options,
|
|
92
94
|
package: data.pkg,
|
|
93
95
|
snapshot: data.snapshot,
|
|
@@ -105,8 +107,14 @@ class BackupAction {
|
|
|
105
107
|
return (0, cli_1.renderListTaskItem)(item, color, {
|
|
106
108
|
snapshot: (data) => data.id,
|
|
107
109
|
task: (data) => [data.packageName, data.taskName],
|
|
108
|
-
backup: (data) => [
|
|
109
|
-
|
|
110
|
+
backup: (data) => [
|
|
111
|
+
data.packageName,
|
|
112
|
+
g([data.repositoryName, (0, bytes_1.formatBytes)(data.bytes)].join(" ")),
|
|
113
|
+
],
|
|
114
|
+
copy: (data) => [
|
|
115
|
+
data.packageName,
|
|
116
|
+
g([data.mirrorRepositoryName, (0, bytes_1.formatBytes)(data.bytes)].join(" ")),
|
|
117
|
+
],
|
|
110
118
|
prune: (data) => [data.packageName, g(`${data.pruned}/${data.total}`)],
|
|
111
119
|
cleanup: () => "",
|
|
112
120
|
report: (data) => data.type,
|
|
@@ -220,6 +228,7 @@ class BackupAction {
|
|
|
220
228
|
data: {
|
|
221
229
|
packageName: pkg.name,
|
|
222
230
|
repositoryName: repositoryName,
|
|
231
|
+
bytes: 0,
|
|
223
232
|
},
|
|
224
233
|
title: {
|
|
225
234
|
initial: `Create backup: ${pkg.name} (${repositoryName})`,
|
|
@@ -229,19 +238,20 @@ class BackupAction {
|
|
|
229
238
|
},
|
|
230
239
|
exitOnError: false,
|
|
231
240
|
runWrapper: gc.cleanupOnFinish.bind(gc),
|
|
232
|
-
run: async (task) => {
|
|
241
|
+
run: async (task, data) => {
|
|
233
242
|
const taskSummary = pkg.task
|
|
234
243
|
? l.result("task", pkg.name)
|
|
235
244
|
: undefined;
|
|
236
245
|
if (taskSummary?.error)
|
|
237
246
|
throw new Error(`Task failed`);
|
|
238
|
-
await this.backup({
|
|
247
|
+
const backup = await this.backup({
|
|
239
248
|
pkg,
|
|
240
249
|
repositoryName,
|
|
241
250
|
snapshot,
|
|
242
251
|
snapshotPath: taskResult?.snapshotPath,
|
|
243
252
|
onProgress: (p) => pm.update(p, (t) => (task.output = t)),
|
|
244
253
|
});
|
|
254
|
+
data.bytes = backup.bytes;
|
|
245
255
|
},
|
|
246
256
|
})), l.$task({
|
|
247
257
|
key: "cleanup",
|
|
@@ -263,6 +273,7 @@ class BackupAction {
|
|
|
263
273
|
packageName: pkg.name,
|
|
264
274
|
repositoryName: name,
|
|
265
275
|
mirrorRepositoryName: mirror,
|
|
276
|
+
bytes: 0,
|
|
266
277
|
},
|
|
267
278
|
title: {
|
|
268
279
|
initial: `Copy snapshot: ${pkg.name} (${mirror})`,
|
|
@@ -272,20 +283,21 @@ class BackupAction {
|
|
|
272
283
|
},
|
|
273
284
|
exitOnError: false,
|
|
274
285
|
runWrapper: gc.cleanup.bind(gc),
|
|
275
|
-
run: async (task) => {
|
|
286
|
+
run: async (task, data) => {
|
|
276
287
|
const backupSummary = l.result("backup", [
|
|
277
288
|
pkg.name,
|
|
278
289
|
name,
|
|
279
290
|
]);
|
|
280
291
|
if (backupSummary.error)
|
|
281
292
|
throw new Error(`Backup failed`);
|
|
282
|
-
await this.copy({
|
|
293
|
+
const copy = await this.copy({
|
|
283
294
|
repositoryName: name,
|
|
284
295
|
mirrorRepositoryName: mirror,
|
|
285
296
|
pkg,
|
|
286
297
|
snapshot,
|
|
287
298
|
onProgress: (p) => pm.update(p, (t) => (task.output = t)),
|
|
288
299
|
});
|
|
300
|
+
data.bytes = copy.bytes;
|
|
289
301
|
},
|
|
290
302
|
})), !!this.options.prune &&
|
|
291
303
|
l.$task({
|
|
@@ -317,6 +329,8 @@ class BackupAction {
|
|
|
317
329
|
}));
|
|
318
330
|
}),
|
|
319
331
|
...(0, report_list_1.createReportListTasks)(l, {
|
|
332
|
+
hostname: this.config.hostname ?? (0, os_1.hostname)(),
|
|
333
|
+
action: "backup",
|
|
320
334
|
reports: this.config.reports || [],
|
|
321
335
|
verbose: this.options.verbose,
|
|
322
336
|
onMessage: (result, report) => this.dataFormat(result).format(report.format ?? "list"),
|
|
@@ -31,6 +31,7 @@ export type Context = {
|
|
|
31
31
|
repositoryName: string;
|
|
32
32
|
mirrorRepositoryName: string;
|
|
33
33
|
skipped: boolean;
|
|
34
|
+
bytes: number;
|
|
34
35
|
};
|
|
35
36
|
} & ReportListTaskContext;
|
|
36
37
|
export declare class CopyAction<TRequired extends boolean = true> {
|
|
@@ -50,6 +51,8 @@ export declare class CopyAction<TRequired extends boolean = true> {
|
|
|
50
51
|
mirrorConfig: RepositoryConfig;
|
|
51
52
|
snapshot: Snapshot;
|
|
52
53
|
onProgress: (p: Progress) => void;
|
|
53
|
-
}): Promise<
|
|
54
|
+
}): Promise<{
|
|
55
|
+
bytes: number;
|
|
56
|
+
}>;
|
|
54
57
|
exec(): Promise<(import("../utils/list").List3SummaryResult | import("../utils/list").Listr3TaskResult<Context>)[]>;
|
|
55
58
|
}
|
|
@@ -50,6 +50,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
50
50
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
51
51
|
exports.CopyAction = void 0;
|
|
52
52
|
const DataFormat_1 = require("../utils/DataFormat");
|
|
53
|
+
const bytes_1 = require("../utils/bytes");
|
|
53
54
|
const cli_1 = require("../utils/cli");
|
|
54
55
|
const config_1 = require("../utils/datatruck/config");
|
|
55
56
|
const report_list_1 = require("../utils/datatruck/report-list");
|
|
@@ -61,6 +62,7 @@ const object_1 = require("../utils/object");
|
|
|
61
62
|
const progress_1 = require("../utils/progress");
|
|
62
63
|
const temp_1 = require("../utils/temp");
|
|
63
64
|
const chalk_1 = __importDefault(require("chalk"));
|
|
65
|
+
const os_1 = require("os");
|
|
64
66
|
class CopyAction {
|
|
65
67
|
config;
|
|
66
68
|
options;
|
|
@@ -79,7 +81,11 @@ class CopyAction {
|
|
|
79
81
|
snapshots: (data) => data.snapshots.length,
|
|
80
82
|
copy: (data) => [
|
|
81
83
|
data.packageName,
|
|
82
|
-
g([
|
|
84
|
+
g([
|
|
85
|
+
data.snapshotId.slice(0, 8),
|
|
86
|
+
data.mirrorRepositoryName,
|
|
87
|
+
(0, bytes_1.formatBytes)(data.bytes),
|
|
88
|
+
].join(" ")),
|
|
83
89
|
],
|
|
84
90
|
report: (data) => data.type,
|
|
85
91
|
summary: (data) => ({
|
|
@@ -150,7 +156,7 @@ class CopyAction {
|
|
|
150
156
|
});
|
|
151
157
|
if (this.config.minFreeDiskSpace)
|
|
152
158
|
await mirrorRepo.ensureFreeDiskSpace(mirrorConfig.config, this.config.minFreeDiskSpace);
|
|
153
|
-
await mirrorRepo.backup({
|
|
159
|
+
return await mirrorRepo.backup({
|
|
154
160
|
options: {
|
|
155
161
|
verbose: this.options.verbose,
|
|
156
162
|
tags: snapshot.tags,
|
|
@@ -219,6 +225,7 @@ class CopyAction {
|
|
|
219
225
|
packageName: snapshot.packageName,
|
|
220
226
|
repositoryName: repoConfig.name,
|
|
221
227
|
mirrorRepositoryName: repo2.name,
|
|
228
|
+
bytes: 0,
|
|
222
229
|
skipped: false,
|
|
223
230
|
},
|
|
224
231
|
title: {
|
|
@@ -249,16 +256,17 @@ class CopyAction {
|
|
|
249
256
|
mirrorConfig,
|
|
250
257
|
]);
|
|
251
258
|
if (sourceRepo.has()) {
|
|
252
|
-
await sourceRepo.get().copy({
|
|
259
|
+
const copy = await sourceRepo.get().copy({
|
|
253
260
|
mirrorRepositoryConfig: mirrorConfig.config,
|
|
254
261
|
options: { verbose: this.options.verbose },
|
|
255
262
|
package: { name: snapshot.packageName },
|
|
256
263
|
snapshot,
|
|
257
264
|
onProgress: (p) => pm.update(p, (d) => (task.output = d)),
|
|
258
265
|
});
|
|
266
|
+
data.bytes = copy.bytes;
|
|
259
267
|
}
|
|
260
268
|
else {
|
|
261
|
-
await this.copyCrossRepository({
|
|
269
|
+
const copy = await this.copyCrossRepository({
|
|
262
270
|
mirrorConfig,
|
|
263
271
|
mirrorRepo,
|
|
264
272
|
repo,
|
|
@@ -266,6 +274,7 @@ class CopyAction {
|
|
|
266
274
|
snapshot,
|
|
267
275
|
onProgress: (p) => pm.update(p, (d) => (task.output = d)),
|
|
268
276
|
});
|
|
277
|
+
data.bytes = copy.bytes;
|
|
269
278
|
sourceRepo.set(mirrorRepo);
|
|
270
279
|
}
|
|
271
280
|
},
|
|
@@ -274,6 +283,8 @@ class CopyAction {
|
|
|
274
283
|
},
|
|
275
284
|
}),
|
|
276
285
|
...(0, report_list_1.createReportListTasks)(l, {
|
|
286
|
+
hostname: this.config.hostname ?? (0, os_1.hostname)(),
|
|
287
|
+
action: "copy",
|
|
277
288
|
reports: this.config.reports || [],
|
|
278
289
|
verbose: this.options.verbose,
|
|
279
290
|
onMessage: (result, report) => this.dataFormat(result).format(report.format ?? "list"),
|
|
@@ -25,12 +25,14 @@ export declare class BackupCommand extends CommandAbstract<BackupCommandOptions<
|
|
|
25
25
|
backup: {
|
|
26
26
|
packageName: string;
|
|
27
27
|
repositoryName: string;
|
|
28
|
+
bytes: number;
|
|
28
29
|
};
|
|
29
30
|
cleanup: {};
|
|
30
31
|
copy: {
|
|
31
32
|
packageName: string;
|
|
32
33
|
repositoryName: string;
|
|
33
34
|
mirrorRepositoryName: string;
|
|
35
|
+
bytes: number;
|
|
34
36
|
};
|
|
35
37
|
prune: {
|
|
36
38
|
packageName: string;
|
|
@@ -50,8 +50,12 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
|
|
|
50
50
|
init(data: RepoInitData): Promise<void>;
|
|
51
51
|
prune(data: RepoPruneData): Promise<void>;
|
|
52
52
|
fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
53
|
-
backup(data: RepoBackupData<DatatruckPackageRepositoryConfig>): Promise<
|
|
54
|
-
|
|
53
|
+
backup(data: RepoBackupData<DatatruckPackageRepositoryConfig>): Promise<{
|
|
54
|
+
bytes: number;
|
|
55
|
+
}>;
|
|
56
|
+
copy(data: RepoCopyData<DatatruckRepositoryConfig>): Promise<{
|
|
57
|
+
bytes: number;
|
|
58
|
+
}>;
|
|
55
59
|
restore(data: RepoRestoreData<DatatruckPackageRepositoryConfig>): Promise<void>;
|
|
56
60
|
}
|
|
57
61
|
export {};
|
|
@@ -203,6 +203,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
203
203
|
}
|
|
204
204
|
scanner.end();
|
|
205
205
|
// Meta
|
|
206
|
+
const size = Object.values(tarStats).reduce((total, { size }) => total + size, 0);
|
|
206
207
|
const metaPath = `${snapshotName}/meta.json`;
|
|
207
208
|
const nodePkg = (0, fs_1.parsePackageFile)();
|
|
208
209
|
const meta = {
|
|
@@ -212,12 +213,15 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
212
213
|
package: data.package.name,
|
|
213
214
|
task: data.package.task?.name,
|
|
214
215
|
version: nodePkg.version,
|
|
215
|
-
size
|
|
216
|
+
size,
|
|
216
217
|
tarStats,
|
|
217
218
|
};
|
|
218
219
|
if (data.options.verbose)
|
|
219
220
|
(0, cli_1.logExec)(`Writing metadata into ${fs.resolvePath(metaPath)}`);
|
|
220
221
|
await fs.writeFile(`${snapshotName}/meta.json`, JSON.stringify(meta));
|
|
222
|
+
return {
|
|
223
|
+
bytes: size,
|
|
224
|
+
};
|
|
221
225
|
}
|
|
222
226
|
async copy(data) {
|
|
223
227
|
const sourceFs = (0, client_1.createFs)(this.config.backend);
|
|
@@ -236,6 +240,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
236
240
|
await targetFs.ensureEmptyDir(tmpSnapshotName);
|
|
237
241
|
const entries = await sourceFs.readdir(snapshotName);
|
|
238
242
|
const total = entries.length;
|
|
243
|
+
let bytes = 0;
|
|
239
244
|
let current = 0;
|
|
240
245
|
for (const entry of entries) {
|
|
241
246
|
const absolute = {
|
|
@@ -250,7 +255,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
250
255
|
const sourceEntry = `${snapshotName}/${entry}`;
|
|
251
256
|
const targetEntry = `${tmpSnapshotName}/${entry}`;
|
|
252
257
|
if (targetFs.isLocal()) {
|
|
253
|
-
await sourceFs.download(sourceEntry, targetFs.resolvePath(targetEntry), {
|
|
258
|
+
const downloaded = await sourceFs.download(sourceEntry, targetFs.resolvePath(targetEntry), {
|
|
254
259
|
onProgress: (progress) => data.onProgress({
|
|
255
260
|
absolute,
|
|
256
261
|
relative: {
|
|
@@ -260,12 +265,13 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
260
265
|
},
|
|
261
266
|
}),
|
|
262
267
|
});
|
|
268
|
+
bytes += downloaded.bytes;
|
|
263
269
|
}
|
|
264
270
|
else {
|
|
265
271
|
const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "remote-copy", entry);
|
|
266
272
|
const tempFile = (0, path_1.join)(tempDir, entry);
|
|
267
273
|
try {
|
|
268
|
-
await sourceFs.download(sourceEntry, tempFile, {
|
|
274
|
+
const downloaded = await sourceFs.download(sourceEntry, tempFile, {
|
|
269
275
|
onProgress: (progress) => data.onProgress({
|
|
270
276
|
absolute,
|
|
271
277
|
relative: {
|
|
@@ -275,6 +281,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
275
281
|
},
|
|
276
282
|
}),
|
|
277
283
|
});
|
|
284
|
+
bytes += downloaded.bytes;
|
|
278
285
|
await targetFs.upload(tempFile, targetEntry);
|
|
279
286
|
}
|
|
280
287
|
finally {
|
|
@@ -283,6 +290,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
283
290
|
}
|
|
284
291
|
}
|
|
285
292
|
await targetFs.rename(tmpSnapshotName, snapshotName);
|
|
293
|
+
return { bytes };
|
|
286
294
|
}
|
|
287
295
|
async restore(data) {
|
|
288
296
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
@@ -22,7 +22,11 @@ export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfi
|
|
|
22
22
|
init(data: RepoInitData): Promise<void>;
|
|
23
23
|
prune(data: RepoPruneData): Promise<void>;
|
|
24
24
|
fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
25
|
-
backup(data: RepoBackupData<GitPackageRepositoryConfig>): Promise<
|
|
26
|
-
|
|
25
|
+
backup(data: RepoBackupData<GitPackageRepositoryConfig>): Promise<{
|
|
26
|
+
bytes: number;
|
|
27
|
+
}>;
|
|
28
|
+
copy(data: RepoCopyData<GitRepositoryConfig>): Promise<{
|
|
29
|
+
bytes: number;
|
|
30
|
+
}>;
|
|
27
31
|
restore(data: RepoRestoreData<GitPackageRepositoryConfig>): Promise<void>;
|
|
28
32
|
}
|
|
@@ -187,6 +187,8 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
187
187
|
if (await git.haveChanges())
|
|
188
188
|
await git.exec(["commit", "-m", data.snapshot.id]);
|
|
189
189
|
const nodePkg = (0, fs_1.parsePackageFile)();
|
|
190
|
+
const size = (await (0, fs_1.fastFolderSizeAsync)(tmpPath)) -
|
|
191
|
+
(await (0, fs_1.fastFolderSizeAsync)((0, path_1.join)(tmpPath, ".git")));
|
|
190
192
|
const meta = GitRepository.buildSnapshotTag({
|
|
191
193
|
id: data.snapshot.id,
|
|
192
194
|
shortId: data.snapshot.id.slice(0, 8),
|
|
@@ -195,16 +197,19 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
195
197
|
package: data.package.name,
|
|
196
198
|
task: data.package.task?.name,
|
|
197
199
|
version: nodePkg.version,
|
|
198
|
-
size:
|
|
199
|
-
(await (0, fs_1.fastFolderSizeAsync)((0, path_1.join)(tmpPath, ".git")))).toString(),
|
|
200
|
+
size: size.toString(),
|
|
200
201
|
});
|
|
201
202
|
await git.addTag(meta.name, meta.message);
|
|
202
203
|
await git.push({ branchName });
|
|
203
204
|
await git.pushTags();
|
|
204
205
|
await (0, promises_1.rm)(tmpPath, { recursive: true });
|
|
206
|
+
return {
|
|
207
|
+
bytes: size,
|
|
208
|
+
};
|
|
205
209
|
}
|
|
206
|
-
copy(data) {
|
|
210
|
+
async copy(data) {
|
|
207
211
|
throw new Error("Method not implemented.");
|
|
212
|
+
return { bytes: 0 };
|
|
208
213
|
}
|
|
209
214
|
async restore(data) {
|
|
210
215
|
const restorePath = data.snapshotPath;
|
|
@@ -83,7 +83,11 @@ export declare abstract class RepositoryAbstract<TConfig> {
|
|
|
83
83
|
abstract init(data: RepoInitData): Promise<void>;
|
|
84
84
|
abstract prune(data: RepoPruneData): Promise<void>;
|
|
85
85
|
abstract fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
86
|
-
abstract copy(data: RepoCopyData<TConfig>): Promise<
|
|
87
|
-
|
|
86
|
+
abstract copy(data: RepoCopyData<TConfig>): Promise<{
|
|
87
|
+
bytes: number;
|
|
88
|
+
}>;
|
|
89
|
+
abstract backup(data: RepoBackupData<unknown>): Promise<{
|
|
90
|
+
bytes: number;
|
|
91
|
+
}>;
|
|
88
92
|
abstract restore(data: RepoRestoreData<unknown>): Promise<void>;
|
|
89
93
|
}
|
|
@@ -33,7 +33,11 @@ export declare class ResticRepository extends RepositoryAbstract<ResticRepositor
|
|
|
33
33
|
init(data: RepoInitData): Promise<void>;
|
|
34
34
|
fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
35
35
|
prune(data: RepoPruneData): Promise<void>;
|
|
36
|
-
backup(data: RepoBackupData<ResticPackageRepositoryConfig>): Promise<
|
|
37
|
-
|
|
36
|
+
backup(data: RepoBackupData<ResticPackageRepositoryConfig>): Promise<{
|
|
37
|
+
bytes: number;
|
|
38
|
+
}>;
|
|
39
|
+
copy(data: RepoCopyData<ResticRepositoryConfig>): Promise<{
|
|
40
|
+
bytes: number;
|
|
41
|
+
}>;
|
|
38
42
|
restore(data: RepoRestoreData<ResticPackageRepositoryConfig>): Promise<void>;
|
|
39
43
|
}
|
|
@@ -204,10 +204,6 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
204
204
|
allowEmptySnapshot: true,
|
|
205
205
|
excludeFile: gitignorePath ? [gitignorePath] : undefined,
|
|
206
206
|
parent: lastSnapshot?.id,
|
|
207
|
-
// https://github.com/restic/restic/pull/3200
|
|
208
|
-
...((await restic.checkBackupSetPathSupport()) && {
|
|
209
|
-
setPaths: [`/datatruck/${data.package.name}`],
|
|
210
|
-
}),
|
|
211
207
|
tags: [
|
|
212
208
|
ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.ID, data.snapshot.id),
|
|
213
209
|
ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.SHORT_ID, data.snapshot.id.slice(0, 8)),
|
|
@@ -267,6 +263,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
267
263
|
percent: 100,
|
|
268
264
|
},
|
|
269
265
|
});
|
|
266
|
+
return {
|
|
267
|
+
bytes: resticTotalBytes,
|
|
268
|
+
};
|
|
270
269
|
}
|
|
271
270
|
async copy(data) {
|
|
272
271
|
const config = data.mirrorRepositoryConfig;
|
|
@@ -289,9 +288,16 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
289
288
|
},
|
|
290
289
|
log: data.options.verbose,
|
|
291
290
|
});
|
|
291
|
+
let bytes = 0;
|
|
292
292
|
await restic.copy({
|
|
293
293
|
id: snapshot.originalId,
|
|
294
|
+
onStream(data) {
|
|
295
|
+
if (data.message_type === "status") {
|
|
296
|
+
bytes = data.total_bytes;
|
|
297
|
+
}
|
|
298
|
+
},
|
|
294
299
|
});
|
|
300
|
+
return { bytes };
|
|
295
301
|
}
|
|
296
302
|
async restore(data) {
|
|
297
303
|
const restorePath = data.snapshotPath;
|
|
@@ -310,7 +316,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
310
316
|
await restic.restore({
|
|
311
317
|
id: snapshot.originalId,
|
|
312
318
|
target: restorePath,
|
|
313
|
-
onStream:
|
|
319
|
+
onStream: (streamData) => {
|
|
314
320
|
if (streamData.message_type === "restore-status") {
|
|
315
321
|
const current = Math.min(streamData.total_bytes, snapshot.size);
|
|
316
322
|
data.onProgress({
|
package/lib/tasks/GitTask.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.GitTask = exports.gitTaskName = void 0;
|
|
4
|
+
const async_process_1 = require("../utils/async-process");
|
|
4
5
|
const cli_1 = require("../utils/cli");
|
|
5
6
|
const fs_1 = require("../utils/fs");
|
|
6
7
|
const math_1 = require("../utils/math");
|
|
7
|
-
const process_1 = require("../utils/process");
|
|
8
8
|
const temp_1 = require("../utils/temp");
|
|
9
9
|
const TaskAbstract_1 = require("./TaskAbstract");
|
|
10
10
|
const assert_1 = require("assert");
|
|
@@ -12,7 +12,6 @@ const fs_2 = require("fs");
|
|
|
12
12
|
const promises_1 = require("fs/promises");
|
|
13
13
|
const micromatch_1 = require("micromatch");
|
|
14
14
|
const path_1 = require("path");
|
|
15
|
-
const readline_1 = require("readline");
|
|
16
15
|
exports.gitTaskName = "git";
|
|
17
16
|
class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
18
17
|
verbose;
|
|
@@ -34,10 +33,9 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
34
33
|
description: "Creating bundle",
|
|
35
34
|
},
|
|
36
35
|
});
|
|
37
|
-
await
|
|
36
|
+
await async_process_1.AsyncProcess.exec(this.command, ["bundle", "create", bundlePath, "--all"], {
|
|
38
37
|
cwd: path,
|
|
39
|
-
|
|
40
|
-
log: this.verbose,
|
|
38
|
+
$log: this.verbose,
|
|
41
39
|
});
|
|
42
40
|
// Config
|
|
43
41
|
if (this.config.includeConfig ?? true) {
|
|
@@ -70,48 +68,33 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
70
68
|
continue;
|
|
71
69
|
option.pathsPath = (0, path_1.join)(snapshotPath, `repo.${option.name}-paths.txt`);
|
|
72
70
|
const stream = (0, fs_2.createWriteStream)(option.pathsPath);
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
found = (0, micromatch_1.isMatch)(path, option.include);
|
|
101
|
-
}
|
|
102
|
-
if (found) {
|
|
103
|
-
total++;
|
|
104
|
-
stream.write(`${path}\n`);
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
|
-
},
|
|
108
|
-
});
|
|
109
|
-
}
|
|
110
|
-
finally {
|
|
111
|
-
await new Promise((resolve) => stream.end(resolve));
|
|
112
|
-
if (streamError)
|
|
113
|
-
throw streamError;
|
|
114
|
-
}
|
|
71
|
+
const p = new async_process_1.AsyncProcess(this.command, [
|
|
72
|
+
"-c",
|
|
73
|
+
"core.quotepath=off",
|
|
74
|
+
"ls-files",
|
|
75
|
+
...option.argv,
|
|
76
|
+
"--exclude-standard",
|
|
77
|
+
], {
|
|
78
|
+
cwd: data.package.path,
|
|
79
|
+
$log: this.verbose,
|
|
80
|
+
});
|
|
81
|
+
await Promise.all([
|
|
82
|
+
p.stdout.parseLines((path) => {
|
|
83
|
+
let found = false;
|
|
84
|
+
if (option.include === true) {
|
|
85
|
+
found = true;
|
|
86
|
+
}
|
|
87
|
+
else if (option.include) {
|
|
88
|
+
found = (0, micromatch_1.isMatch)(path, option.include);
|
|
89
|
+
}
|
|
90
|
+
if (found) {
|
|
91
|
+
total++;
|
|
92
|
+
stream.write(`${path}\n`);
|
|
93
|
+
}
|
|
94
|
+
}),
|
|
95
|
+
p.child.on("close", () => stream.end()),
|
|
96
|
+
(0, fs_1.waitForClose)(stream),
|
|
97
|
+
]);
|
|
115
98
|
}
|
|
116
99
|
// Copy
|
|
117
100
|
for (const option of lsFilesConfig) {
|
|
@@ -176,10 +159,9 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
176
159
|
};
|
|
177
160
|
// Bundle
|
|
178
161
|
const bundlePath = (0, path_1.join)(snapshotPath, "repo.bundle");
|
|
179
|
-
await
|
|
162
|
+
await async_process_1.AsyncProcess.exec(this.command, ["clone", bundlePath, "."], {
|
|
180
163
|
cwd: restorePath,
|
|
181
|
-
|
|
182
|
-
log: this.verbose,
|
|
164
|
+
$log: this.verbose,
|
|
183
165
|
});
|
|
184
166
|
await incrementProgress();
|
|
185
167
|
// Config
|