@datatruck/cli 0.32.2 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.schema.json +374 -31
- package/lib/{Action → actions}/BackupAction.d.ts +5 -17
- package/lib/{Action → actions}/BackupAction.js +32 -85
- package/lib/{Action → actions}/ConfigAction.d.ts +4 -2
- package/lib/{Action → actions}/ConfigAction.js +16 -8
- package/lib/{Action → actions}/CopyAction.d.ts +17 -13
- package/lib/actions/CopyAction.js +285 -0
- package/lib/{Action → actions}/InitAction.d.ts +1 -1
- package/lib/{Action → actions}/InitAction.js +3 -3
- package/lib/{Action → actions}/PruneAction.d.ts +1 -2
- package/lib/{Action → actions}/PruneAction.js +2 -2
- package/lib/{Action → actions}/RestoreAction.d.ts +4 -13
- package/lib/{Action → actions}/RestoreAction.js +17 -17
- package/lib/{Action → actions}/SnapshotsAction.d.ts +2 -3
- package/lib/{Action → actions}/SnapshotsAction.js +3 -3
- package/lib/cli.d.ts +3 -3
- package/lib/cli.js +18 -17
- package/lib/commands/BackupCommand.d.ts +43 -0
- package/lib/{Command → commands}/BackupCommand.js +7 -6
- package/lib/{Command → commands}/CleanCacheCommand.d.ts +4 -2
- package/lib/{Command → commands}/CleanCacheCommand.js +6 -5
- package/lib/{Command → commands}/CommandAbstract.d.ts +10 -6
- package/lib/{Command → commands}/CommandAbstract.js +5 -3
- package/lib/commands/ConfigCommand.d.ts +22 -0
- package/lib/{Command → commands}/ConfigCommand.js +5 -5
- package/lib/commands/CopyCommand.d.ts +17 -0
- package/lib/{Command → commands}/CopyCommand.js +7 -6
- package/lib/commands/InitCommand.d.ts +19 -0
- package/lib/{Command → commands}/InitCommand.js +9 -9
- package/lib/{Command → commands}/PruneCommand.d.ts +7 -4
- package/lib/{Command → commands}/PruneCommand.js +11 -11
- package/lib/commands/RestoreCommand.d.ts +38 -0
- package/lib/{Command → commands}/RestoreCommand.js +7 -6
- package/lib/{Command → commands}/SnapshotsCommand.d.ts +8 -6
- package/lib/{Command → commands}/SnapshotsCommand.js +12 -12
- package/lib/{Command → commands}/StartServerCommand.d.ts +4 -2
- package/lib/{Command → commands}/StartServerCommand.js +5 -5
- package/lib/index.d.ts +20 -23
- package/lib/index.js +8 -8
- package/lib/{Repository → repositories}/DatatruckRepository.js +2 -2
- package/lib/{Repository → repositories}/RepositoryAbstract.d.ts +7 -7
- package/lib/{Repository → repositories}/ResticRepository.js +7 -6
- package/lib/{Task → tasks}/MssqlTask.js +2 -2
- package/lib/{Task → tasks}/MysqlDumpTask.js +4 -4
- package/lib/{Task → tasks}/ScriptTask.d.ts +2 -2
- package/lib/{Task → tasks}/SqlDumpTaskAbstract.js +4 -4
- package/lib/{Task → tasks}/TaskAbstract.d.ts +4 -4
- package/lib/utils/DataFormat.js +3 -3
- package/lib/utils/Restic.d.ts +1 -0
- package/lib/utils/Restic.js +12 -2
- package/lib/utils/cli.d.ts +9 -3
- package/lib/utils/cli.js +17 -1
- package/lib/utils/cron.d.ts +11 -0
- package/lib/utils/cron.js +27 -0
- package/lib/utils/datatruck/command.d.ts +29 -0
- package/lib/utils/datatruck/command.js +61 -0
- package/lib/{Config/RepositoryConfig.d.ts → utils/datatruck/config-repository-type.d.ts} +19 -8
- package/lib/{Config/TaskConfig.d.ts → utils/datatruck/config-task-type.d.ts} +6 -6
- package/lib/utils/datatruck/config-type.d.ts +50 -0
- package/lib/utils/datatruck/config.d.ts +19 -10
- package/lib/utils/datatruck/config.js +43 -7
- package/lib/utils/datatruck/cron-server.d.ts +27 -6
- package/lib/utils/datatruck/cron-server.js +38 -20
- package/lib/utils/datatruck/paths.d.ts +2 -2
- package/lib/utils/datatruck/report-list.d.ts +12 -0
- package/lib/utils/datatruck/report-list.js +57 -0
- package/lib/utils/datatruck/repository-server.js +3 -2
- package/lib/utils/datatruck/repository.d.ts +16 -0
- package/lib/utils/datatruck/repository.js +30 -0
- package/lib/utils/datatruck/snapshot.d.ts +2 -2
- package/lib/utils/datatruck/task.d.ts +3 -0
- package/lib/{Factory/TaskFactory.js → utils/datatruck/task.js} +8 -8
- package/lib/utils/date.js +6 -2
- package/lib/utils/fs.d.ts +3 -0
- package/lib/utils/fs.js +24 -4
- package/lib/utils/list.d.ts +5 -5
- package/lib/utils/mysql.js +5 -5
- package/lib/utils/object.d.ts +13 -0
- package/lib/utils/object.js +32 -1
- package/lib/utils/process.js +4 -1
- package/lib/utils/string.d.ts +1 -0
- package/lib/utils/string.js +7 -3
- package/lib/utils/ts.d.ts +16 -0
- package/lib/utils/watcher.d.ts +10 -0
- package/lib/utils/watcher.js +34 -0
- package/package.json +3 -3
- package/lib/Action/CopyAction.js +0 -164
- package/lib/Command/BackupCommand.d.ts +0 -19
- package/lib/Command/ConfigCommand.d.ts +0 -15
- package/lib/Command/CopyCommand.d.ts +0 -16
- package/lib/Command/InitCommand.d.ts +0 -13
- package/lib/Command/RestoreCommand.d.ts +0 -17
- package/lib/Config/Config.d.ts +0 -28
- package/lib/Config/PackageConfig.d.ts +0 -24
- package/lib/Config/PackageRepositoryConfig.d.ts +0 -15
- package/lib/Config/PrunePolicyConfig.d.ts +0 -2
- package/lib/Config/RepositoryConfig.js +0 -2
- package/lib/Config/TaskConfig.js +0 -2
- package/lib/Factory/CommandFactory.d.ts +0 -45
- package/lib/Factory/CommandFactory.js +0 -96
- package/lib/Factory/RepositoryFactory.d.ts +0 -3
- package/lib/Factory/RepositoryFactory.js +0 -23
- package/lib/Factory/TaskFactory.d.ts +0 -3
- /package/lib/{Action → actions}/CleanCacheAction.d.ts +0 -0
- /package/lib/{Action → actions}/CleanCacheAction.js +0 -0
- /package/lib/{Repository → repositories}/DatatruckRepository.d.ts +0 -0
- /package/lib/{Repository → repositories}/GitRepository.d.ts +0 -0
- /package/lib/{Repository → repositories}/GitRepository.js +0 -0
- /package/lib/{Repository → repositories}/RepositoryAbstract.js +0 -0
- /package/lib/{Repository → repositories}/ResticRepository.d.ts +0 -0
- /package/lib/{Task → tasks}/GitTask.d.ts +0 -0
- /package/lib/{Task → tasks}/GitTask.js +0 -0
- /package/lib/{Task → tasks}/MariadbTask.d.ts +0 -0
- /package/lib/{Task → tasks}/MariadbTask.js +0 -0
- /package/lib/{Task → tasks}/MssqlTask.d.ts +0 -0
- /package/lib/{Task → tasks}/MysqlDumpTask.d.ts +0 -0
- /package/lib/{Task → tasks}/PostgresqlDumpTask.d.ts +0 -0
- /package/lib/{Task → tasks}/PostgresqlDumpTask.js +0 -0
- /package/lib/{Task → tasks}/ScriptTask.js +0 -0
- /package/lib/{Task → tasks}/SqlDumpTaskAbstract.d.ts +0 -0
- /package/lib/{Task → tasks}/TaskAbstract.js +0 -0
- /package/lib/{Config/Config.js → utils/datatruck/config-repository-type.js} +0 -0
- /package/lib/{Config/PackageConfig.js → utils/datatruck/config-task-type.js} +0 -0
- /package/lib/{Config/PackageRepositoryConfig.js → utils/datatruck/config-type.js} +0 -0
- /package/lib/{Error/AppError.d.ts → utils/datatruck/error.d.ts} +0 -0
- /package/lib/{Error/AppError.js → utils/datatruck/error.js} +0 -0
- /package/lib/{Config/PrunePolicyConfig.js → utils/ts.js} +0 -0
|
@@ -4,17 +4,16 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.BackupAction = void 0;
|
|
7
|
-
const RepositoryFactory_1 = require("../Factory/RepositoryFactory");
|
|
8
|
-
const TaskFactory_1 = require("../Factory/TaskFactory");
|
|
9
7
|
const DataFormat_1 = require("../utils/DataFormat");
|
|
10
8
|
const cli_1 = require("../utils/cli");
|
|
11
9
|
const config_1 = require("../utils/datatruck/config");
|
|
10
|
+
const report_list_1 = require("../utils/datatruck/report-list");
|
|
11
|
+
const repository_1 = require("../utils/datatruck/repository");
|
|
12
|
+
const task_1 = require("../utils/datatruck/task");
|
|
12
13
|
const date_1 = require("../utils/date");
|
|
13
14
|
const fs_1 = require("../utils/fs");
|
|
14
15
|
const list_1 = require("../utils/list");
|
|
15
16
|
const progress_1 = require("../utils/progress");
|
|
16
|
-
const reportSteps_1 = require("../utils/reportSteps");
|
|
17
|
-
const spawnSteps_1 = require("../utils/spawnSteps");
|
|
18
17
|
const temp_1 = require("../utils/temp");
|
|
19
18
|
const PruneAction_1 = require("./PruneAction");
|
|
20
19
|
const assert_1 = require("assert");
|
|
@@ -68,7 +67,7 @@ class BackupAction {
|
|
|
68
67
|
const pkg = { ...data.pkg, path: data.snapshotPath ?? data.pkg.path };
|
|
69
68
|
(0, assert_1.ok)(pkg.path);
|
|
70
69
|
await (0, fs_1.ensureExistsDir)(pkg.path);
|
|
71
|
-
const repo = (0,
|
|
70
|
+
const repo = await (0, repository_1.createAndInitRepo)(repoConfig, this.options.verbose);
|
|
72
71
|
if (this.config.minFreeDiskSpace)
|
|
73
72
|
await repo.ensureFreeDiskSpace(repoConfig.config, this.config.minFreeDiskSpace);
|
|
74
73
|
const packageConfig = pkg.repositoryConfigs?.find((config) => config.type === repoConfig.type &&
|
|
@@ -84,8 +83,8 @@ class BackupAction {
|
|
|
84
83
|
async copy(data) {
|
|
85
84
|
const repoConfig = (0, config_1.findRepositoryOrFail)(this.config, data.repositoryName);
|
|
86
85
|
const mirrorRepoConfig = (0, config_1.findRepositoryOrFail)(this.config, data.mirrorRepositoryName);
|
|
87
|
-
const repo = (0,
|
|
88
|
-
const mirrorRepo = (0,
|
|
86
|
+
const repo = await (0, repository_1.createAndInitRepo)(repoConfig, this.options.verbose);
|
|
87
|
+
const mirrorRepo = await (0, repository_1.createAndInitRepo)(mirrorRepoConfig, this.options.verbose);
|
|
89
88
|
if (this.config.minFreeDiskSpace)
|
|
90
89
|
await mirrorRepo.ensureFreeDiskSpace(mirrorRepoConfig.config, this.config.minFreeDiskSpace);
|
|
91
90
|
await repo.copy({
|
|
@@ -103,33 +102,27 @@ class BackupAction {
|
|
|
103
102
|
};
|
|
104
103
|
const renderData = (item, color, result = []) => {
|
|
105
104
|
const g = (v) => (color ? `${chalk_1.default.gray(`(${v})`)}` : `(${v})`);
|
|
106
|
-
return item
|
|
107
|
-
|
|
108
|
-
:
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
return result;
|
|
128
|
-
}, 0),
|
|
129
|
-
}, color)
|
|
130
|
-
: item.key === "report"
|
|
131
|
-
? item.data.type
|
|
132
|
-
: "";
|
|
105
|
+
return (0, cli_1.renderListTaskItem)(item, color, {
|
|
106
|
+
snapshot: (data) => data.id,
|
|
107
|
+
task: (data) => [data.packageName, data.taskName],
|
|
108
|
+
backup: (data) => [data.packageName, g(data.repositoryName)],
|
|
109
|
+
copy: (data) => [data.packageName, g(data.mirrorRepositoryName)],
|
|
110
|
+
prune: (data) => [data.packageName, g(`${data.pruned}/${data.total}`)],
|
|
111
|
+
cleanup: () => "",
|
|
112
|
+
report: (data) => data.type,
|
|
113
|
+
summary: (data) => ({
|
|
114
|
+
errors: data.errors,
|
|
115
|
+
backups: result.filter((r) => !r.error && r.key === "backup").length,
|
|
116
|
+
copies: result.filter((r) => !r.error && r.key === "copy").length,
|
|
117
|
+
prunes: result
|
|
118
|
+
.filter((r) => !r.error && r.key === "prune")
|
|
119
|
+
.reduce((result, item) => {
|
|
120
|
+
if (item.key === "prune")
|
|
121
|
+
result += item.data.pruned;
|
|
122
|
+
return result;
|
|
123
|
+
}, 0),
|
|
124
|
+
}),
|
|
125
|
+
});
|
|
133
126
|
};
|
|
134
127
|
return new DataFormat_1.DataFormat({
|
|
135
128
|
streams: options.streams,
|
|
@@ -214,7 +207,7 @@ class BackupAction {
|
|
|
214
207
|
exitOnError: false,
|
|
215
208
|
runWrapper: gc.cleanupIfFail.bind(gc),
|
|
216
209
|
run: async (task) => {
|
|
217
|
-
taskResult = await (0,
|
|
210
|
+
taskResult = await (0, task_1.createTask)(pkg.task).backup({
|
|
218
211
|
options,
|
|
219
212
|
package: pkg,
|
|
220
213
|
snapshot,
|
|
@@ -323,56 +316,10 @@ class BackupAction {
|
|
|
323
316
|
},
|
|
324
317
|
}));
|
|
325
318
|
}),
|
|
326
|
-
...(
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
initial: `Send report ${reportIndex}`,
|
|
331
|
-
started: `Sending report ${reportIndex}`,
|
|
332
|
-
completed: `Report sent: ${reportIndex}`,
|
|
333
|
-
failed: `Report send failed: ${reportIndex}`,
|
|
334
|
-
},
|
|
335
|
-
key: "report",
|
|
336
|
-
keyIndex: index,
|
|
337
|
-
data: { type: report.run.type },
|
|
338
|
-
exitOnError: false,
|
|
339
|
-
run: async (task) => {
|
|
340
|
-
const result = l
|
|
341
|
-
.getResult()
|
|
342
|
-
.filter((r) => r.key !== "report");
|
|
343
|
-
const success = result.every((r) => !r.error);
|
|
344
|
-
const enabled = !report.when ||
|
|
345
|
-
(report.when === "success" && success) ||
|
|
346
|
-
(report.when === "error" && !success);
|
|
347
|
-
if (!enabled)
|
|
348
|
-
return task.skip(`Report send skipped: ${reportIndex}`);
|
|
349
|
-
const message = this.dataFormat(result).format(report.format ?? "list");
|
|
350
|
-
if ((0, spawnSteps_1.isSpawnStep)(report.run)) {
|
|
351
|
-
await (0, spawnSteps_1.runSpawnSteps)(report.run, {
|
|
352
|
-
data: {
|
|
353
|
-
dtt: {
|
|
354
|
-
message,
|
|
355
|
-
result,
|
|
356
|
-
success,
|
|
357
|
-
},
|
|
358
|
-
},
|
|
359
|
-
verbose: this.options.verbose,
|
|
360
|
-
});
|
|
361
|
-
}
|
|
362
|
-
else if ((0, reportSteps_1.isReportStep)(report.run)) {
|
|
363
|
-
await (0, reportSteps_1.runReportSteps)(report.run, {
|
|
364
|
-
data: {
|
|
365
|
-
title: "DTT Backup",
|
|
366
|
-
message,
|
|
367
|
-
success,
|
|
368
|
-
},
|
|
369
|
-
});
|
|
370
|
-
}
|
|
371
|
-
else {
|
|
372
|
-
throw new Error(`Invalid step type: ${report.run.type}`);
|
|
373
|
-
}
|
|
374
|
-
},
|
|
375
|
-
});
|
|
319
|
+
...(0, report_list_1.createReportListTasks)(l, {
|
|
320
|
+
reports: this.config.reports || [],
|
|
321
|
+
verbose: this.options.verbose,
|
|
322
|
+
onMessage: (result, report) => this.dataFormat(result).format(report.format ?? "list"),
|
|
376
323
|
}),
|
|
377
324
|
];
|
|
378
325
|
},
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { GlobalOptions } from "../
|
|
2
|
-
import type { Config } from "../
|
|
1
|
+
import { GlobalOptions } from "../commands/CommandAbstract";
|
|
2
|
+
import type { Config } from "../utils/datatruck/config-type";
|
|
3
3
|
import { IfRequireKeys } from "../utils/ts";
|
|
4
4
|
export type ConfigActionOptions = {
|
|
5
5
|
path: string;
|
|
@@ -11,7 +11,9 @@ export declare class ConfigAction<TRequired extends boolean = true> {
|
|
|
11
11
|
static validate(config: Config): void;
|
|
12
12
|
static check(config: Config): void;
|
|
13
13
|
static normalize(config: Config): Config;
|
|
14
|
+
static findFile(path: string): Promise<string>;
|
|
14
15
|
static parseFile(path: string): Promise<Config>;
|
|
16
|
+
static findAndParseFile(inputPath: string): Promise<Config>;
|
|
15
17
|
static fromGlobalOptionsWithPath(globalOptions: GlobalOptions<true>): Promise<{
|
|
16
18
|
path: string;
|
|
17
19
|
data: Config;
|
|
@@ -4,9 +4,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.ConfigAction = void 0;
|
|
7
|
-
const AppError_1 = require("../Error/AppError");
|
|
8
7
|
const config_schema_1 = require("../config.schema");
|
|
9
8
|
const config_1 = require("../utils/datatruck/config");
|
|
9
|
+
const error_1 = require("../utils/datatruck/error");
|
|
10
10
|
const fs_1 = require("../utils/fs");
|
|
11
11
|
const ajv_1 = __importDefault(require("ajv"));
|
|
12
12
|
const assert_1 = require("assert");
|
|
@@ -20,7 +20,7 @@ class ConfigAction {
|
|
|
20
20
|
allowUnionTypes: true,
|
|
21
21
|
}).compile(config_schema_1.configSchema);
|
|
22
22
|
if (!validate(config))
|
|
23
|
-
throw new
|
|
23
|
+
throw new error_1.AppError("Json schema error: " + JSON.stringify(validate.errors, null, 2));
|
|
24
24
|
}
|
|
25
25
|
static check(config) {
|
|
26
26
|
const repositoryNames = [];
|
|
@@ -29,18 +29,18 @@ class ConfigAction {
|
|
|
29
29
|
for (const repo of config.repositories) {
|
|
30
30
|
repos[repo.name] = repo;
|
|
31
31
|
if (repositoryNames.includes(repo.name))
|
|
32
|
-
throw new
|
|
32
|
+
throw new error_1.AppError(`Duplicated repository name: ${repo.name}`);
|
|
33
33
|
repositoryNames.push(repo.name);
|
|
34
34
|
}
|
|
35
35
|
for (const repo of config.repositories) {
|
|
36
36
|
if (repo.mirrorRepoNames) {
|
|
37
37
|
for (const mirrorRepoName of repo.mirrorRepoNames) {
|
|
38
38
|
if (!repos[mirrorRepoName])
|
|
39
|
-
throw new
|
|
39
|
+
throw new error_1.AppError(`Mirror repository name not found: ${mirrorRepoName}`);
|
|
40
40
|
if (repos[mirrorRepoName].type !== repo.type)
|
|
41
|
-
throw new
|
|
41
|
+
throw new error_1.AppError(`Mirror repository type is incompatible: ${mirrorRepoName}`);
|
|
42
42
|
if (mirrorRepoNames.includes(mirrorRepoName))
|
|
43
|
-
throw new
|
|
43
|
+
throw new error_1.AppError(`Mirror repository is already used`);
|
|
44
44
|
mirrorRepoNames.push(mirrorRepoName);
|
|
45
45
|
}
|
|
46
46
|
}
|
|
@@ -48,7 +48,7 @@ class ConfigAction {
|
|
|
48
48
|
const packageNames = [];
|
|
49
49
|
for (const pkg of config.packages) {
|
|
50
50
|
if (packageNames.includes(pkg.name))
|
|
51
|
-
throw new
|
|
51
|
+
throw new error_1.AppError(`Duplicated package name: ${pkg.name}`);
|
|
52
52
|
repositoryNames.push(pkg.name);
|
|
53
53
|
}
|
|
54
54
|
}
|
|
@@ -65,12 +65,20 @@ class ConfigAction {
|
|
|
65
65
|
});
|
|
66
66
|
return config;
|
|
67
67
|
}
|
|
68
|
+
static async findFile(path) {
|
|
69
|
+
return await (0, fs_1.findFile)(path, "datatruck.config", fs_1.parseFileExtensions, "Config path not found");
|
|
70
|
+
}
|
|
68
71
|
static async parseFile(path) {
|
|
69
72
|
const config = await (0, fs_1.parseFile)(path, "config");
|
|
70
73
|
ConfigAction.validate(config);
|
|
71
74
|
ConfigAction.check(config);
|
|
72
75
|
return ConfigAction.normalize(config);
|
|
73
76
|
}
|
|
77
|
+
static async findAndParseFile(inputPath) {
|
|
78
|
+
const path = await ConfigAction.findFile(inputPath);
|
|
79
|
+
const data = await ConfigAction.parseFile(path);
|
|
80
|
+
return data;
|
|
81
|
+
}
|
|
74
82
|
static async fromGlobalOptionsWithPath(globalOptions) {
|
|
75
83
|
if (typeof globalOptions.config !== "string")
|
|
76
84
|
return {
|
|
@@ -88,7 +96,7 @@ class ConfigAction {
|
|
|
88
96
|
return config.data;
|
|
89
97
|
}
|
|
90
98
|
async exec() {
|
|
91
|
-
const path = await
|
|
99
|
+
const path = await ConfigAction.findFile(this.options.path);
|
|
92
100
|
const data = await ConfigAction.parseFile(path);
|
|
93
101
|
return { path, data };
|
|
94
102
|
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { Snapshot } from "../Repository/RepositoryAbstract";
|
|
1
|
+
import { RepositoryAbstract, Snapshot } from "../repositories/RepositoryAbstract";
|
|
3
2
|
import { DataFormat } from "../utils/DataFormat";
|
|
3
|
+
import type { Config, RepositoryConfig } from "../utils/datatruck/config-type";
|
|
4
|
+
import { ReportListTaskContext } from "../utils/datatruck/report-list";
|
|
4
5
|
import { Listr3TaskResultEnd } from "../utils/list";
|
|
5
|
-
import {
|
|
6
|
+
import { StrictMap } from "../utils/object";
|
|
7
|
+
import { Progress, ProgressMode } from "../utils/progress";
|
|
6
8
|
import { Streams } from "../utils/stream";
|
|
7
9
|
import { IfRequireKeys } from "../utils/ts";
|
|
8
10
|
export type CopyActionOptions = {
|
|
@@ -30,7 +32,7 @@ export type Context = {
|
|
|
30
32
|
mirrorRepositoryName: string;
|
|
31
33
|
skipped: boolean;
|
|
32
34
|
};
|
|
33
|
-
};
|
|
35
|
+
} & ReportListTaskContext;
|
|
34
36
|
export declare class CopyAction<TRequired extends boolean = true> {
|
|
35
37
|
readonly config: Config;
|
|
36
38
|
readonly options: IfRequireKeys<TRequired, CopyActionOptions>;
|
|
@@ -39,13 +41,15 @@ export declare class CopyAction<TRequired extends boolean = true> {
|
|
|
39
41
|
streams?: Streams;
|
|
40
42
|
verbose?: number;
|
|
41
43
|
}): DataFormat;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
44
|
+
protected fetchSnapshots(repo: RepositoryAbstract<any>): Promise<Snapshot[]>;
|
|
45
|
+
protected createSourceRepoMap(): StrictMap<[Pick<Snapshot, "packageName" | "id">, Pick<RepositoryConfig, "type">], RepositoryAbstract<any>>;
|
|
46
|
+
protected copyCrossRepository(options: {
|
|
47
|
+
repo: RepositoryAbstract<any>;
|
|
48
|
+
repoConfig: RepositoryConfig;
|
|
49
|
+
mirrorRepo: RepositoryAbstract<any>;
|
|
50
|
+
mirrorConfig: RepositoryConfig;
|
|
51
|
+
snapshot: Snapshot;
|
|
52
|
+
onProgress: (p: Progress) => void;
|
|
53
|
+
}): Promise<void>;
|
|
54
|
+
exec(): Promise<(import("../utils/list").List3SummaryResult | import("../utils/list").Listr3TaskResult<Context>)[]>;
|
|
51
55
|
}
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
3
|
+
if (value !== null && value !== void 0) {
|
|
4
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
5
|
+
var dispose;
|
|
6
|
+
if (async) {
|
|
7
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
8
|
+
dispose = value[Symbol.asyncDispose];
|
|
9
|
+
}
|
|
10
|
+
if (dispose === void 0) {
|
|
11
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
12
|
+
dispose = value[Symbol.dispose];
|
|
13
|
+
}
|
|
14
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
16
|
+
}
|
|
17
|
+
else if (async) {
|
|
18
|
+
env.stack.push({ async: true });
|
|
19
|
+
}
|
|
20
|
+
return value;
|
|
21
|
+
};
|
|
22
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
23
|
+
return function (env) {
|
|
24
|
+
function fail(e) {
|
|
25
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
26
|
+
env.hasError = true;
|
|
27
|
+
}
|
|
28
|
+
function next() {
|
|
29
|
+
while (env.stack.length) {
|
|
30
|
+
var rec = env.stack.pop();
|
|
31
|
+
try {
|
|
32
|
+
var result = rec.dispose && rec.dispose.call(rec.value);
|
|
33
|
+
if (rec.async) return Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
34
|
+
}
|
|
35
|
+
catch (e) {
|
|
36
|
+
fail(e);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
if (env.hasError) throw env.error;
|
|
40
|
+
}
|
|
41
|
+
return next();
|
|
42
|
+
};
|
|
43
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
44
|
+
var e = new Error(message);
|
|
45
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
46
|
+
});
|
|
47
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
48
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
49
|
+
};
|
|
50
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
51
|
+
exports.CopyAction = void 0;
|
|
52
|
+
const DataFormat_1 = require("../utils/DataFormat");
|
|
53
|
+
const cli_1 = require("../utils/cli");
|
|
54
|
+
const config_1 = require("../utils/datatruck/config");
|
|
55
|
+
const report_list_1 = require("../utils/datatruck/report-list");
|
|
56
|
+
const repository_1 = require("../utils/datatruck/repository");
|
|
57
|
+
const snapshot_1 = require("../utils/datatruck/snapshot");
|
|
58
|
+
const date_1 = require("../utils/date");
|
|
59
|
+
const list_1 = require("../utils/list");
|
|
60
|
+
const object_1 = require("../utils/object");
|
|
61
|
+
const progress_1 = require("../utils/progress");
|
|
62
|
+
const temp_1 = require("../utils/temp");
|
|
63
|
+
const chalk_1 = __importDefault(require("chalk"));
|
|
64
|
+
class CopyAction {
|
|
65
|
+
config;
|
|
66
|
+
options;
|
|
67
|
+
constructor(config, options) {
|
|
68
|
+
this.config = config;
|
|
69
|
+
this.options = options;
|
|
70
|
+
}
|
|
71
|
+
dataFormat(result, options = {}) {
|
|
72
|
+
const renderTitle = (item, color) => {
|
|
73
|
+
let title = item.key.slice(0, 1).toUpperCase() + item.key.slice(1);
|
|
74
|
+
return item.key === "copy" && color ? chalk_1.default.cyan(title) : title;
|
|
75
|
+
};
|
|
76
|
+
const renderData = (item, color, items = []) => {
|
|
77
|
+
const g = (v) => (color ? `${chalk_1.default.gray(`(${v})`)}` : `(${v})`);
|
|
78
|
+
return (0, cli_1.renderListTaskItem)(item, color, {
|
|
79
|
+
snapshots: (data) => data.snapshots.length,
|
|
80
|
+
copy: (data) => [
|
|
81
|
+
data.packageName,
|
|
82
|
+
g([data.snapshotId.slice(0, 8), data.mirrorRepositoryName].join(" ")),
|
|
83
|
+
],
|
|
84
|
+
report: (data) => data.type,
|
|
85
|
+
summary: (data) => ({
|
|
86
|
+
errors: data.errors,
|
|
87
|
+
copied: items.filter((i) => i.key === "copy" && !i.error && !i.data.skipped).length,
|
|
88
|
+
skipped: items.filter((i) => i.key === "copy" && !i.error && i.data.skipped).length,
|
|
89
|
+
}),
|
|
90
|
+
});
|
|
91
|
+
};
|
|
92
|
+
return new DataFormat_1.DataFormat({
|
|
93
|
+
streams: options.streams,
|
|
94
|
+
json: result,
|
|
95
|
+
table: {
|
|
96
|
+
headers: [
|
|
97
|
+
{ value: "", width: 3 },
|
|
98
|
+
{ value: "Title", width: 15 },
|
|
99
|
+
{ value: "Data" },
|
|
100
|
+
{ value: "Duration", width: 10 },
|
|
101
|
+
{ value: "Error", width: 50 },
|
|
102
|
+
],
|
|
103
|
+
rows: () => result.map((item) => [
|
|
104
|
+
(0, cli_1.renderResult)(item.error),
|
|
105
|
+
renderTitle(item, true),
|
|
106
|
+
renderData(item, true, result),
|
|
107
|
+
(0, date_1.duration)(item.elapsed),
|
|
108
|
+
(0, cli_1.renderError)(item.error, options.verbose),
|
|
109
|
+
]),
|
|
110
|
+
},
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
async fetchSnapshots(repo) {
|
|
114
|
+
const snapshots = await repo.fetchSnapshots({
|
|
115
|
+
options: {
|
|
116
|
+
ids: this.options.ids,
|
|
117
|
+
packageNames: this.options.packageNames,
|
|
118
|
+
packageTaskNames: this.options.packageTaskNames,
|
|
119
|
+
verbose: this.options.verbose,
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
const result = this.options.last
|
|
123
|
+
? (0, snapshot_1.groupAndFilter)(snapshots, ["packageName"], {
|
|
124
|
+
last: this.options.last,
|
|
125
|
+
}).map(({ item }) => item)
|
|
126
|
+
: snapshots;
|
|
127
|
+
if (!result.length)
|
|
128
|
+
throw new Error("No snapshots found");
|
|
129
|
+
return result;
|
|
130
|
+
}
|
|
131
|
+
createSourceRepoMap() {
|
|
132
|
+
return new object_1.StrictMap(([snapshot, config]) => [config.type, snapshot.id, snapshot.packageName].join("|"));
|
|
133
|
+
}
|
|
134
|
+
async copyCrossRepository(options) {
|
|
135
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
136
|
+
try {
|
|
137
|
+
const { repo, repoConfig, mirrorRepo, mirrorConfig, snapshot } = options;
|
|
138
|
+
const tmp = __addDisposableResource(env_1, await (0, temp_1.useTempDir)("copy", "restore"), true);
|
|
139
|
+
const pkg = (0, config_1.findPackageOrFail)(this.config, snapshot.packageName);
|
|
140
|
+
await repo.restore({
|
|
141
|
+
options: {
|
|
142
|
+
verbose: this.options.verbose,
|
|
143
|
+
snapshotId: snapshot.id,
|
|
144
|
+
},
|
|
145
|
+
snapshot: { id: snapshot.id, date: snapshot.date },
|
|
146
|
+
package: { name: snapshot.packageName },
|
|
147
|
+
packageConfig: (0, config_1.findPackageRepositoryConfig)(pkg, repoConfig),
|
|
148
|
+
snapshotPath: tmp.path,
|
|
149
|
+
onProgress: options.onProgress,
|
|
150
|
+
});
|
|
151
|
+
if (this.config.minFreeDiskSpace)
|
|
152
|
+
await mirrorRepo.ensureFreeDiskSpace(mirrorConfig.config, this.config.minFreeDiskSpace);
|
|
153
|
+
await mirrorRepo.backup({
|
|
154
|
+
options: {
|
|
155
|
+
verbose: this.options.verbose,
|
|
156
|
+
tags: snapshot.tags,
|
|
157
|
+
},
|
|
158
|
+
snapshot: { id: snapshot.id, date: snapshot.date },
|
|
159
|
+
package: {
|
|
160
|
+
name: snapshot.packageName,
|
|
161
|
+
path: tmp.path,
|
|
162
|
+
},
|
|
163
|
+
packageConfig: (0, config_1.findPackageRepositoryConfig)(pkg, mirrorConfig),
|
|
164
|
+
onProgress: options.onProgress,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
catch (e_1) {
|
|
168
|
+
env_1.error = e_1;
|
|
169
|
+
env_1.hasError = true;
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
const result_1 = __disposeResources(env_1);
|
|
173
|
+
if (result_1)
|
|
174
|
+
await result_1;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
async exec() {
|
|
178
|
+
const { options } = this;
|
|
179
|
+
const pm = new progress_1.ProgressManager({
|
|
180
|
+
verbose: options.verbose,
|
|
181
|
+
tty: options.tty,
|
|
182
|
+
mode: options.progress,
|
|
183
|
+
});
|
|
184
|
+
const l = new list_1.Listr3({ progressManager: pm });
|
|
185
|
+
return l
|
|
186
|
+
.add([
|
|
187
|
+
l.$task({
|
|
188
|
+
key: "snapshots",
|
|
189
|
+
data: { snapshots: [] },
|
|
190
|
+
title: {
|
|
191
|
+
initial: "Fetch snapshots",
|
|
192
|
+
started: "Fetching snapshots",
|
|
193
|
+
completed: "Snapshots fetched",
|
|
194
|
+
failed: "Snapshot fetch failed",
|
|
195
|
+
},
|
|
196
|
+
run: async (task, data) => {
|
|
197
|
+
if (this.config.minFreeDiskSpace)
|
|
198
|
+
await (0, temp_1.ensureFreeDiskTempSpace)(this.config.minFreeDiskSpace);
|
|
199
|
+
const repoConfig = (0, config_1.findRepositoryOrFail)(this.config, this.options.repositoryName);
|
|
200
|
+
const repo = await (0, repository_1.createAndInitRepo)(repoConfig, this.options.verbose);
|
|
201
|
+
data.snapshots = await this.fetchSnapshots(repo);
|
|
202
|
+
task.title = `Snapshots fetched: ${data.snapshots.length}`;
|
|
203
|
+
const repositoryNames2 = (0, config_1.sortReposByType)((0, config_1.filterRepository)(this.config.repositories, {
|
|
204
|
+
include: this.options.repositoryNames2,
|
|
205
|
+
exclude: [repoConfig.name],
|
|
206
|
+
action: "backup",
|
|
207
|
+
}), [repoConfig.type]);
|
|
208
|
+
if (!repositoryNames2.length)
|
|
209
|
+
throw new Error("No mirror snapshots found");
|
|
210
|
+
const sourceRepoMap = this.createSourceRepoMap();
|
|
211
|
+
return data.snapshots.flatMap((snapshot) => repositoryNames2.map((repo2) => {
|
|
212
|
+
const id = snapshot.id.slice(0, 8);
|
|
213
|
+
const pkgName = snapshot.packageName;
|
|
214
|
+
return l.$task({
|
|
215
|
+
key: "copy",
|
|
216
|
+
keyIndex: [snapshot.packageName, repo2.name, snapshot.id],
|
|
217
|
+
data: {
|
|
218
|
+
snapshotId: snapshot.id,
|
|
219
|
+
packageName: snapshot.packageName,
|
|
220
|
+
repositoryName: repoConfig.name,
|
|
221
|
+
mirrorRepositoryName: repo2.name,
|
|
222
|
+
skipped: false,
|
|
223
|
+
},
|
|
224
|
+
title: {
|
|
225
|
+
initial: `Copy snapshot: ${pkgName} (${id}) » ${repo2.name}`,
|
|
226
|
+
started: `Copying snapshot: ${pkgName} (${id}) » ${repo2.name}`,
|
|
227
|
+
completed: `Snapshot copied: ${pkgName} (${id}) » ${repo2.name}`,
|
|
228
|
+
failed: `Snapshot copy failed: ${pkgName} (${id}) » ${repo2.name}`,
|
|
229
|
+
},
|
|
230
|
+
exitOnError: false,
|
|
231
|
+
run: async (task, data) => {
|
|
232
|
+
const mirrorConfig = (0, config_1.findRepositoryOrFail)(this.config, repo2.name);
|
|
233
|
+
const mirrorRepo = await (0, repository_1.createAndInitRepo)(mirrorConfig, this.options.verbose);
|
|
234
|
+
const currentCopies = await mirrorRepo.fetchSnapshots({
|
|
235
|
+
options: {
|
|
236
|
+
ids: [snapshot.id],
|
|
237
|
+
packageNames: [snapshot.packageName],
|
|
238
|
+
verbose: this.options.verbose,
|
|
239
|
+
},
|
|
240
|
+
});
|
|
241
|
+
if (currentCopies.length) {
|
|
242
|
+
data.skipped = true;
|
|
243
|
+
return task.skip(`Already exists at ${mirrorConfig.name}: ${pkgName} (${id})`);
|
|
244
|
+
}
|
|
245
|
+
if (this.config.minFreeDiskSpace)
|
|
246
|
+
await mirrorRepo.ensureFreeDiskSpace(mirrorConfig.config, this.config.minFreeDiskSpace);
|
|
247
|
+
const sourceRepo = sourceRepoMap.with([
|
|
248
|
+
snapshot,
|
|
249
|
+
mirrorConfig,
|
|
250
|
+
]);
|
|
251
|
+
if (sourceRepo.has()) {
|
|
252
|
+
await sourceRepo.get().copy({
|
|
253
|
+
mirrorRepositoryConfig: mirrorConfig.config,
|
|
254
|
+
options: { verbose: this.options.verbose },
|
|
255
|
+
package: { name: snapshot.packageName },
|
|
256
|
+
snapshot,
|
|
257
|
+
onProgress: (p) => pm.update(p, (d) => (task.output = d)),
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
261
|
+
await this.copyCrossRepository({
|
|
262
|
+
mirrorConfig,
|
|
263
|
+
mirrorRepo,
|
|
264
|
+
repo,
|
|
265
|
+
repoConfig,
|
|
266
|
+
snapshot,
|
|
267
|
+
onProgress: (p) => pm.update(p, (d) => (task.output = d)),
|
|
268
|
+
});
|
|
269
|
+
sourceRepo.set(mirrorRepo);
|
|
270
|
+
}
|
|
271
|
+
},
|
|
272
|
+
});
|
|
273
|
+
}));
|
|
274
|
+
},
|
|
275
|
+
}),
|
|
276
|
+
...(0, report_list_1.createReportListTasks)(l, {
|
|
277
|
+
reports: this.config.reports || [],
|
|
278
|
+
verbose: this.options.verbose,
|
|
279
|
+
onMessage: (result, report) => this.dataFormat(result).format(report.format ?? "list"),
|
|
280
|
+
}),
|
|
281
|
+
])
|
|
282
|
+
.exec();
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
exports.CopyAction = CopyAction;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.InitAction = void 0;
|
|
4
|
-
const RepositoryFactory_1 = require("../Factory/RepositoryFactory");
|
|
5
4
|
const config_1 = require("../utils/datatruck/config");
|
|
5
|
+
const repository_1 = require("../utils/datatruck/repository");
|
|
6
6
|
class InitAction {
|
|
7
7
|
config;
|
|
8
8
|
options;
|
|
@@ -13,7 +13,7 @@ class InitAction {
|
|
|
13
13
|
async exec() {
|
|
14
14
|
const result = [];
|
|
15
15
|
for (const repoConfig of this.config.repositories) {
|
|
16
|
-
if (!(0, config_1.
|
|
16
|
+
if (!(0, config_1.filterRepositoryByEnabled)(repoConfig, "init"))
|
|
17
17
|
continue;
|
|
18
18
|
if (this.options.repositoryNames &&
|
|
19
19
|
!this.options.repositoryNames.includes(repoConfig.name))
|
|
@@ -21,7 +21,7 @@ class InitAction {
|
|
|
21
21
|
if (this.options.repositoryTypes &&
|
|
22
22
|
!this.options.repositoryTypes.includes(repoConfig.type))
|
|
23
23
|
continue;
|
|
24
|
-
const repo = (0,
|
|
24
|
+
const repo = (0, repository_1.createRepo)(repoConfig);
|
|
25
25
|
let initError = null;
|
|
26
26
|
try {
|
|
27
27
|
await repo.init({
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import type { Config } from "../
|
|
2
|
-
import { RepositoryConfig } from "../Config/RepositoryConfig";
|
|
1
|
+
import type { Config, RepositoryConfig } from "../utils/datatruck/config-type";
|
|
3
2
|
import { KeepObject } from "../utils/date";
|
|
4
3
|
import { IfRequireKeys } from "../utils/ts";
|
|
5
4
|
import { ExtendedSnapshot, SnapshotsActionOptions } from "./SnapshotsAction";
|