@datatruck/cli 0.27.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/BackupAction.d.ts +75 -34
- package/Action/BackupAction.js +302 -246
- package/Action/CleanCacheAction.d.ts +8 -4
- package/Action/CleanCacheAction.js +8 -5
- package/Action/ConfigAction.d.ts +12 -5
- package/Action/ConfigAction.js +14 -18
- package/Action/CopyAction.d.ts +51 -0
- package/Action/CopyAction.js +165 -0
- package/Action/InitAction.d.ts +3 -3
- package/Action/InitAction.js +9 -9
- package/Action/PruneAction.d.ts +10 -16
- package/Action/PruneAction.js +37 -34
- package/Action/RestoreAction.d.ts +49 -24
- package/Action/RestoreAction.js +164 -195
- package/Action/SnapshotsAction.d.ts +8 -8
- package/Action/SnapshotsAction.js +9 -9
- package/CHANGELOG.md +513 -0
- package/Command/BackupCommand.d.ts +7 -4
- package/Command/BackupCommand.js +14 -26
- package/Command/CleanCacheCommand.d.ts +4 -4
- package/Command/CleanCacheCommand.js +26 -5
- package/Command/CommandAbstract.d.ts +10 -7
- package/Command/CommandAbstract.js +4 -1
- package/Command/ConfigCommand.d.ts +6 -9
- package/Command/ConfigCommand.js +13 -8
- package/Command/CopyCommand.d.ts +16 -0
- package/Command/CopyCommand.js +66 -0
- package/Command/InitCommand.d.ts +4 -4
- package/Command/InitCommand.js +13 -17
- package/Command/PruneCommand.d.ts +4 -10
- package/Command/PruneCommand.js +13 -12
- package/Command/RestoreCommand.d.ts +1 -1
- package/Command/RestoreCommand.js +13 -21
- package/Command/SnapshotsCommand.d.ts +4 -4
- package/Command/SnapshotsCommand.js +16 -15
- package/Command/StartServerCommand.d.ts +3 -3
- package/Config/Config.d.ts +11 -0
- package/Config/Config.js +18 -0
- package/Config/PrunePolicyConfig.d.ts +2 -2
- package/Factory/CommandFactory.d.ts +27 -34
- package/Factory/CommandFactory.js +27 -54
- package/Factory/RepositoryFactory.d.ts +1 -1
- package/Factory/RepositoryFactory.js +3 -3
- package/Factory/TaskFactory.d.ts +1 -1
- package/Factory/TaskFactory.js +3 -3
- package/Repository/DatatruckRepository.d.ts +10 -8
- package/Repository/DatatruckRepository.js +47 -25
- package/Repository/GitRepository.d.ts +9 -8
- package/Repository/GitRepository.js +22 -25
- package/Repository/RepositoryAbstract.d.ts +39 -37
- package/Repository/RepositoryAbstract.js +4 -5
- package/Repository/ResticRepository.d.ts +9 -8
- package/Repository/ResticRepository.js +30 -28
- package/Task/GitTask.d.ts +6 -7
- package/Task/GitTask.js +24 -30
- package/Task/MariadbTask.d.ts +4 -5
- package/Task/MariadbTask.js +26 -32
- package/Task/MssqlTask.d.ts +5 -3
- package/Task/MssqlTask.js +11 -12
- package/Task/MysqlDumpTask.d.ts +10 -3
- package/Task/MysqlDumpTask.js +102 -32
- package/Task/ScriptTask.d.ts +23 -18
- package/Task/ScriptTask.js +34 -24
- package/Task/SqlDumpTaskAbstract.d.ts +8 -3
- package/Task/SqlDumpTaskAbstract.js +32 -20
- package/Task/TaskAbstract.d.ts +24 -25
- package/Task/TaskAbstract.js +6 -10
- package/cli.js +13 -5
- package/config.schema.json +89 -1
- package/package.json +4 -5
- package/utils/DataFormat.d.ts +23 -12
- package/utils/DataFormat.js +36 -14
- package/utils/cli.d.ts +3 -9
- package/utils/cli.js +19 -55
- package/utils/crypto.d.ts +1 -0
- package/utils/crypto.js +15 -0
- package/utils/datatruck/client.d.ts +2 -0
- package/utils/datatruck/client.js +3 -0
- package/utils/datatruck/config.d.ts +2 -0
- package/utils/datatruck/config.js +18 -3
- package/utils/datatruck/paths.d.ts +5 -9
- package/utils/datatruck/paths.js +2 -2
- package/utils/datatruck/snapshot.d.ts +5 -2
- package/utils/datatruck/snapshot.js +12 -22
- package/utils/date.d.ts +21 -4
- package/utils/date.js +46 -24
- package/utils/fs.d.ts +16 -11
- package/utils/fs.js +81 -48
- package/utils/list.d.ts +64 -0
- package/utils/list.js +145 -0
- package/utils/mysql.d.ts +2 -0
- package/utils/mysql.js +21 -2
- package/utils/process.d.ts +1 -0
- package/utils/process.js +24 -31
- package/utils/progress.d.ts +33 -0
- package/utils/progress.js +113 -0
- package/utils/steps.d.ts +11 -0
- package/utils/steps.js +22 -10
- package/utils/stream.d.ts +7 -0
- package/utils/stream.js +10 -0
- package/utils/string.d.ts +0 -1
- package/utils/string.js +1 -13
- package/utils/tar.d.ts +10 -3
- package/utils/tar.js +73 -45
- package/utils/temp.d.ts +26 -0
- package/utils/temp.js +133 -0
- package/utils/virtual-fs.d.ts +6 -2
- package/utils/virtual-fs.js +6 -0
- package/Action/BackupSessionsAction.d.ts +0 -13
- package/Action/BackupSessionsAction.js +0 -18
- package/Action/RestoreSessionsAction.d.ts +0 -13
- package/Action/RestoreSessionsAction.js +0 -18
- package/Command/BackupSessionsCommand.d.ts +0 -12
- package/Command/BackupSessionsCommand.js +0 -92
- package/Command/RestoreSessionsCommand.d.ts +0 -12
- package/Command/RestoreSessionsCommand.js +0 -91
- package/Decorator/EntityDecorator.d.ts +0 -11
- package/Decorator/EntityDecorator.js +0 -17
- package/Entity/BackupSessionEntity.d.ts +0 -6
- package/Entity/BackupSessionEntity.js +0 -25
- package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
- package/Entity/BackupSessionRepositoryEntity.js +0 -25
- package/Entity/BackupSessionTaskEntity.d.ts +0 -5
- package/Entity/BackupSessionTaskEntity.js +0 -24
- package/Entity/CrudEntityAbstract.d.ts +0 -5
- package/Entity/CrudEntityAbstract.js +0 -9
- package/Entity/RestoreSessionEntity.d.ts +0 -5
- package/Entity/RestoreSessionEntity.js +0 -24
- package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
- package/Entity/RestoreSessionRepositoryEntity.js +0 -25
- package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
- package/Entity/RestoreSessionTaskEntity.js +0 -24
- package/Entity/StateEntityAbstract.d.ts +0 -9
- package/Entity/StateEntityAbstract.js +0 -12
- package/Factory/EntityFactory.d.ts +0 -6
- package/Factory/EntityFactory.js +0 -40
- package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
- package/SessionDriver/ConsoleSessionDriver.js +0 -208
- package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
- package/SessionDriver/SessionDriverAbstract.js +0 -28
- package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
- package/SessionDriver/SqliteSessionDriver.js +0 -173
- package/SessionManager/BackupSessionManager.d.ts +0 -45
- package/SessionManager/BackupSessionManager.js +0 -218
- package/SessionManager/RestoreSessionManager.d.ts +0 -47
- package/SessionManager/RestoreSessionManager.js +0 -218
- package/SessionManager/SessionManagerAbstract.d.ts +0 -18
- package/SessionManager/SessionManagerAbstract.js +0 -36
- package/migrations/001-initial.sql +0 -98
- package/utils/entity.d.ts +0 -4
- package/utils/entity.js +0 -10
|
@@ -1,52 +1,45 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import { PruneCommandOptionsType } from "../Command/PruneCommand";
|
|
1
|
+
import { CleanCacheActionOptions } from "../Action/CleanCacheAction";
|
|
2
|
+
import { BackupCommandOptions, BackupCommandResult } from "../Command/BackupCommand";
|
|
3
|
+
import { GlobalOptions } from "../Command/CommandAbstract";
|
|
4
|
+
import { ConfigCommandResult, ConfigCommandOptions } from "../Command/ConfigCommand";
|
|
5
|
+
import { CopyCommandOptionsType } from "../Command/CopyCommand";
|
|
6
|
+
import { InitCommandResult, InitCommandOptions } from "../Command/InitCommand";
|
|
7
|
+
import { PruneCommandOptions } from "../Command/PruneCommand";
|
|
9
8
|
import { RestoreCommandOptionsType } from "../Command/RestoreCommand";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
9
|
+
import { SnapshotsCommandResult, SnapshotsCommandOptions } from "../Command/SnapshotsCommand";
|
|
10
|
+
import { StartServerCommand, StartServerCommandOptions } from "../Command/StartServerCommand";
|
|
11
|
+
import { Streams } from "../utils/stream";
|
|
13
12
|
export declare enum CommandEnum {
|
|
14
13
|
config = "config",
|
|
15
14
|
init = "init",
|
|
16
15
|
snapshots = "snapshots",
|
|
17
16
|
prune = "prune",
|
|
18
17
|
backup = "backup",
|
|
19
|
-
backupSessions = "backup-sessions",
|
|
20
18
|
restore = "restore",
|
|
21
|
-
|
|
19
|
+
copy = "copy",
|
|
22
20
|
cleanCache = "clean-cache",
|
|
23
21
|
startServer = "start-server"
|
|
24
22
|
}
|
|
25
23
|
export type OptionsMapType = {
|
|
26
|
-
[CommandEnum.config]:
|
|
27
|
-
[CommandEnum.init]:
|
|
28
|
-
[CommandEnum.snapshots]:
|
|
29
|
-
[CommandEnum.prune]:
|
|
30
|
-
[CommandEnum.backup]:
|
|
31
|
-
[CommandEnum.backupSessions]: BackupSessionsCommandOptionsType;
|
|
24
|
+
[CommandEnum.config]: ConfigCommandOptions;
|
|
25
|
+
[CommandEnum.init]: InitCommandOptions;
|
|
26
|
+
[CommandEnum.snapshots]: SnapshotsCommandOptions;
|
|
27
|
+
[CommandEnum.prune]: PruneCommandOptions;
|
|
28
|
+
[CommandEnum.backup]: BackupCommandOptions;
|
|
32
29
|
[CommandEnum.restore]: RestoreCommandOptionsType;
|
|
33
|
-
[CommandEnum.
|
|
34
|
-
[CommandEnum.cleanCache]:
|
|
35
|
-
[CommandEnum.startServer]:
|
|
30
|
+
[CommandEnum.copy]: CopyCommandOptionsType;
|
|
31
|
+
[CommandEnum.cleanCache]: CleanCacheActionOptions;
|
|
32
|
+
[CommandEnum.startServer]: StartServerCommandOptions;
|
|
36
33
|
};
|
|
37
34
|
export type LogMapType = {
|
|
38
|
-
[CommandEnum.config]:
|
|
39
|
-
[CommandEnum.init]:
|
|
40
|
-
[CommandEnum.snapshots]:
|
|
35
|
+
[CommandEnum.config]: ConfigCommandResult;
|
|
36
|
+
[CommandEnum.init]: InitCommandResult;
|
|
37
|
+
[CommandEnum.snapshots]: SnapshotsCommandResult;
|
|
38
|
+
[CommandEnum.backup]: BackupCommandResult;
|
|
41
39
|
};
|
|
42
|
-
export declare function CommandFactory<TCommand extends keyof OptionsMapType>(type: TCommand, globalOptions:
|
|
43
|
-
export declare function exec<TCommand extends keyof OptionsMapType>(type: TCommand, globalOptions:
|
|
44
|
-
export declare function createActionInterface(globalOptions:
|
|
40
|
+
export declare function CommandFactory<TCommand extends keyof OptionsMapType>(type: TCommand, globalOptions: GlobalOptions<true>, options: OptionsMapType[TCommand], streams?: Partial<Streams>): StartServerCommand;
|
|
41
|
+
export declare function exec<TCommand extends keyof OptionsMapType>(type: TCommand, globalOptions: GlobalOptions<true>, options: OptionsMapType[TCommand], streams?: Partial<Streams>): Promise<number>;
|
|
42
|
+
export declare function createActionInterface(globalOptions: GlobalOptions<true>): {
|
|
45
43
|
[K in keyof OptionsMapType as `${K}`]: (options: OptionsMapType[K]) => Promise<K extends keyof LogMapType ? LogMapType[K] : never>;
|
|
46
44
|
};
|
|
47
|
-
export declare function
|
|
48
|
-
exitCode: any;
|
|
49
|
-
log: LogMapType[TCommand];
|
|
50
|
-
}>;
|
|
51
|
-
export declare function makeParseLog<TCommand extends keyof LogMapType>(type: TCommand): () => LogMapType[TCommand];
|
|
52
|
-
export declare function CommandConstructorFactory(type: CommandEnum): typeof CleanCacheCommand;
|
|
45
|
+
export declare function CommandConstructorFactory(type: CommandEnum): typeof StartServerCommand;
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.CommandConstructorFactory = exports.
|
|
3
|
+
exports.CommandConstructorFactory = exports.createActionInterface = exports.exec = exports.CommandFactory = exports.CommandEnum = void 0;
|
|
4
4
|
const BackupCommand_1 = require("../Command/BackupCommand");
|
|
5
|
-
const BackupSessionsCommand_1 = require("../Command/BackupSessionsCommand");
|
|
6
5
|
const CleanCacheCommand_1 = require("../Command/CleanCacheCommand");
|
|
7
6
|
const ConfigCommand_1 = require("../Command/ConfigCommand");
|
|
7
|
+
const CopyCommand_1 = require("../Command/CopyCommand");
|
|
8
8
|
const InitCommand_1 = require("../Command/InitCommand");
|
|
9
9
|
const PruneCommand_1 = require("../Command/PruneCommand");
|
|
10
10
|
const RestoreCommand_1 = require("../Command/RestoreCommand");
|
|
11
|
-
const RestoreSessionsCommand_1 = require("../Command/RestoreSessionsCommand");
|
|
12
11
|
const SnapshotsCommand_1 = require("../Command/SnapshotsCommand");
|
|
13
12
|
const StartServerCommand_1 = require("../Command/StartServerCommand");
|
|
14
13
|
const AppError_1 = require("../Error/AppError");
|
|
14
|
+
const stream_1 = require("stream");
|
|
15
15
|
var CommandEnum;
|
|
16
16
|
(function (CommandEnum) {
|
|
17
17
|
CommandEnum["config"] = "config";
|
|
@@ -19,72 +19,48 @@ var CommandEnum;
|
|
|
19
19
|
CommandEnum["snapshots"] = "snapshots";
|
|
20
20
|
CommandEnum["prune"] = "prune";
|
|
21
21
|
CommandEnum["backup"] = "backup";
|
|
22
|
-
CommandEnum["backupSessions"] = "backup-sessions";
|
|
23
22
|
CommandEnum["restore"] = "restore";
|
|
24
|
-
CommandEnum["
|
|
23
|
+
CommandEnum["copy"] = "copy";
|
|
25
24
|
CommandEnum["cleanCache"] = "clean-cache";
|
|
26
25
|
CommandEnum["startServer"] = "start-server";
|
|
27
26
|
})(CommandEnum || (exports.CommandEnum = CommandEnum = {}));
|
|
28
|
-
function CommandFactory(type, globalOptions, options) {
|
|
27
|
+
function CommandFactory(type, globalOptions, options, streams) {
|
|
29
28
|
const constructor = CommandConstructorFactory(type);
|
|
30
|
-
return new constructor(globalOptions, options);
|
|
29
|
+
return new constructor(globalOptions, options, streams);
|
|
31
30
|
}
|
|
32
31
|
exports.CommandFactory = CommandFactory;
|
|
33
|
-
async function exec(type, globalOptions, options) {
|
|
34
|
-
return await CommandFactory(type, globalOptions, options).onExec();
|
|
32
|
+
async function exec(type, globalOptions, options, streams) {
|
|
33
|
+
return await CommandFactory(type, globalOptions, options, streams).onExec();
|
|
35
34
|
}
|
|
36
35
|
exports.exec = exec;
|
|
37
36
|
function createActionInterface(globalOptions) {
|
|
38
37
|
const object = {};
|
|
39
38
|
for (const type of Object.values(CommandEnum)) {
|
|
40
39
|
object[type] = async (options) => {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
40
|
+
let stdoutData = "";
|
|
41
|
+
const stdout = new stream_1.Writable({
|
|
42
|
+
write(chunk, encoding, callback) {
|
|
43
|
+
stdoutData += chunk.toString();
|
|
44
|
+
process.stdout.write(chunk, encoding, callback);
|
|
45
|
+
},
|
|
46
|
+
}).on("data", (chunk) => (stdoutData += chunk.toString()));
|
|
47
|
+
const end = () => !stdout.closed &&
|
|
48
|
+
new Promise((resolve) => stdout.end().on("close", resolve));
|
|
49
|
+
try {
|
|
50
|
+
const exitCode = await exec(type, { ...globalOptions, outputFormat: "json", verbose: 1 }, options, { stdout });
|
|
51
|
+
if (exitCode !== 0)
|
|
52
|
+
throw new Error(`Invalid exit code: ${exitCode}`);
|
|
53
|
+
await end();
|
|
54
|
+
return JSON.parse(stdoutData);
|
|
48
55
|
}
|
|
49
|
-
|
|
50
|
-
|
|
56
|
+
finally {
|
|
57
|
+
await end();
|
|
51
58
|
}
|
|
52
|
-
if (exitCode !== 0)
|
|
53
|
-
throw new Error(`Invalid exit code: ${exitCode}`);
|
|
54
|
-
return log;
|
|
55
59
|
};
|
|
56
60
|
}
|
|
57
61
|
return object;
|
|
58
62
|
}
|
|
59
63
|
exports.createActionInterface = createActionInterface;
|
|
60
|
-
async function runAndParse(type, run) {
|
|
61
|
-
const parseLog = makeParseLog(type);
|
|
62
|
-
try {
|
|
63
|
-
const exitCode = await run();
|
|
64
|
-
return { exitCode, log: parseLog() };
|
|
65
|
-
}
|
|
66
|
-
catch (error) {
|
|
67
|
-
try {
|
|
68
|
-
parseLog();
|
|
69
|
-
}
|
|
70
|
-
catch (_) { }
|
|
71
|
-
throw error;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
exports.runAndParse = runAndParse;
|
|
75
|
-
function makeParseLog(type) {
|
|
76
|
-
const data = [];
|
|
77
|
-
const consoleLog = console.log;
|
|
78
|
-
console.log = console.info = (...items) => {
|
|
79
|
-
consoleLog.bind(console)(...items);
|
|
80
|
-
data.push(...items);
|
|
81
|
-
};
|
|
82
|
-
return function parseLog() {
|
|
83
|
-
console.log = console.info = consoleLog;
|
|
84
|
-
return JSON.parse(data.flat().join("\n"));
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
exports.makeParseLog = makeParseLog;
|
|
88
64
|
function CommandConstructorFactory(type) {
|
|
89
65
|
if (type === CommandEnum.config) {
|
|
90
66
|
return ConfigCommand_1.ConfigCommand;
|
|
@@ -101,14 +77,11 @@ function CommandConstructorFactory(type) {
|
|
|
101
77
|
else if (type === CommandEnum.backup) {
|
|
102
78
|
return BackupCommand_1.BackupCommand;
|
|
103
79
|
}
|
|
104
|
-
else if (type === CommandEnum.backupSessions) {
|
|
105
|
-
return BackupSessionsCommand_1.BackupSessionsCommand;
|
|
106
|
-
}
|
|
107
80
|
else if (type === CommandEnum.restore) {
|
|
108
81
|
return RestoreCommand_1.RestoreCommand;
|
|
109
82
|
}
|
|
110
|
-
else if (type === CommandEnum.
|
|
111
|
-
return
|
|
83
|
+
else if (type === CommandEnum.copy) {
|
|
84
|
+
return CopyCommand_1.CopyCommand;
|
|
112
85
|
}
|
|
113
86
|
else if (type === CommandEnum.cleanCache) {
|
|
114
87
|
return CleanCacheCommand_1.CleanCacheCommand;
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { RepositoryConfigType } from "../Config/RepositoryConfig";
|
|
2
2
|
import type { RepositoryAbstract } from "../Repository/RepositoryAbstract";
|
|
3
|
-
export declare function
|
|
3
|
+
export declare function createRepo(repository: RepositoryConfigType): RepositoryAbstract<any>;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.createRepo = void 0;
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const DatatruckRepository_1 = require("../Repository/DatatruckRepository");
|
|
6
6
|
const GitRepository_1 = require("../Repository/GitRepository");
|
|
7
7
|
const ResticRepository_1 = require("../Repository/ResticRepository");
|
|
8
|
-
function
|
|
8
|
+
function createRepo(repository) {
|
|
9
9
|
const type = repository.type;
|
|
10
10
|
if (type === GitRepository_1.gitRepositoryName) {
|
|
11
11
|
return new GitRepository_1.GitRepository(repository);
|
|
@@ -20,4 +20,4 @@ function RepositoryFactory(repository) {
|
|
|
20
20
|
throw new AppError_1.AppError(`Invalid repository type: ${type}`);
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
exports.
|
|
23
|
+
exports.createRepo = createRepo;
|
package/Factory/TaskFactory.d.ts
CHANGED
package/Factory/TaskFactory.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.createTask = void 0;
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const GitTask_1 = require("../Task/GitTask");
|
|
6
6
|
const MariadbTask_1 = require("../Task/MariadbTask");
|
|
@@ -8,7 +8,7 @@ const MssqlTask_1 = require("../Task/MssqlTask");
|
|
|
8
8
|
const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
|
|
9
9
|
const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
|
|
10
10
|
const ScriptTask_1 = require("../Task/ScriptTask");
|
|
11
|
-
function
|
|
11
|
+
function createTask(task) {
|
|
12
12
|
if (task.name === GitTask_1.gitTaskName) {
|
|
13
13
|
return new GitTask_1.GitTask(task.config ?? {});
|
|
14
14
|
}
|
|
@@ -31,4 +31,4 @@ function TaskFactory(task) {
|
|
|
31
31
|
throw new AppError_1.AppError(`Invalid task name: ${task["name"]}`);
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
|
-
exports.
|
|
34
|
+
exports.createTask = createTask;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CompressOptions } from "../utils/tar";
|
|
2
|
-
import { RepositoryAbstract,
|
|
2
|
+
import { RepositoryAbstract, RepoBackupData, RepoInitData, RepoRestoreData, RepoFetchSnapshotsData, Snapshot, RepoPruneData, RepoCopyData } from "./RepositoryAbstract";
|
|
3
3
|
import type { JSONSchema7 } from "json-schema";
|
|
4
4
|
export type MetaDataType = {
|
|
5
5
|
id: string;
|
|
@@ -12,6 +12,7 @@ export type MetaDataType = {
|
|
|
12
12
|
tarStats?: Record<string, {
|
|
13
13
|
files: number;
|
|
14
14
|
size: number;
|
|
15
|
+
checksum: string;
|
|
15
16
|
}>;
|
|
16
17
|
};
|
|
17
18
|
export type DatatruckRepositoryConfigType = {
|
|
@@ -47,12 +48,13 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
|
|
|
47
48
|
sourcePath: string;
|
|
48
49
|
} | null;
|
|
49
50
|
static parseMetaData(data: string): Promise<MetaDataType>;
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
51
|
+
getSource(): string;
|
|
52
|
+
fetchDiskStats(config: DatatruckRepositoryConfigType): Promise<import("../utils/fs").DiskStats>;
|
|
53
|
+
init(data: RepoInitData): Promise<void>;
|
|
54
|
+
prune(data: RepoPruneData): Promise<void>;
|
|
55
|
+
fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
56
|
+
backup(data: RepoBackupData<DatatruckPackageRepositoryConfigType>): Promise<void>;
|
|
57
|
+
copy(data: RepoCopyData<DatatruckRepositoryConfigType>): Promise<void>;
|
|
58
|
+
restore(data: RepoRestoreData<DatatruckPackageRepositoryConfigType>): Promise<void>;
|
|
57
59
|
}
|
|
58
60
|
export {};
|
|
@@ -4,11 +4,14 @@ exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exp
|
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
6
6
|
const cli_1 = require("../utils/cli");
|
|
7
|
+
const crypto_1 = require("../utils/crypto");
|
|
7
8
|
const client_1 = require("../utils/datatruck/client");
|
|
8
9
|
const paths_1 = require("../utils/datatruck/paths");
|
|
9
10
|
const fs_1 = require("../utils/fs");
|
|
11
|
+
const math_1 = require("../utils/math");
|
|
10
12
|
const string_1 = require("../utils/string");
|
|
11
13
|
const tar_1 = require("../utils/tar");
|
|
14
|
+
const temp_1 = require("../utils/temp");
|
|
12
15
|
const RepositoryAbstract_1 = require("./RepositoryAbstract");
|
|
13
16
|
const assert_1 = require("assert");
|
|
14
17
|
const promises_1 = require("fs/promises");
|
|
@@ -73,14 +76,18 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
73
76
|
static async parseMetaData(data) {
|
|
74
77
|
return JSON.parse(data.toString());
|
|
75
78
|
}
|
|
76
|
-
|
|
79
|
+
getSource() {
|
|
77
80
|
return this.config.backend;
|
|
78
81
|
}
|
|
79
|
-
|
|
82
|
+
fetchDiskStats(config) {
|
|
83
|
+
const fs = (0, client_1.createFs)(config.backend);
|
|
84
|
+
return fs.fetchDiskStats(".");
|
|
85
|
+
}
|
|
86
|
+
async init(data) {
|
|
80
87
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
81
88
|
await fs.mkdir(".");
|
|
82
89
|
}
|
|
83
|
-
async
|
|
90
|
+
async prune(data) {
|
|
84
91
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
85
92
|
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, {
|
|
86
93
|
name: data.snapshot.packageName,
|
|
@@ -90,7 +97,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
90
97
|
if (await fs.existsDir(snapshotName))
|
|
91
98
|
await fs.rmAll(snapshotName);
|
|
92
99
|
}
|
|
93
|
-
async
|
|
100
|
+
async fetchSnapshots(data) {
|
|
94
101
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
95
102
|
if (!(await fs.existsDir(".")))
|
|
96
103
|
throw new Error(`Repository (${this.repository.name}) out path does not exist: ${fs.resolvePath(".")}`);
|
|
@@ -132,26 +139,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
132
139
|
}
|
|
133
140
|
return snapshots;
|
|
134
141
|
}
|
|
135
|
-
async
|
|
142
|
+
async backup(data) {
|
|
136
143
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
137
144
|
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
138
145
|
const outPath = fs.isLocal()
|
|
139
146
|
? fs.resolvePath(snapshotName)
|
|
140
|
-
: await
|
|
147
|
+
: await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "fs-remote");
|
|
141
148
|
const pkg = data.package;
|
|
142
|
-
const
|
|
143
|
-
(0, assert_1.ok)(sourcePath);
|
|
149
|
+
const path = pkg.path;
|
|
144
150
|
await fs.mkdir(snapshotName);
|
|
145
151
|
const backupPathsOptions = {
|
|
146
152
|
package: data.package,
|
|
147
153
|
snapshot: data.snapshot,
|
|
148
|
-
|
|
154
|
+
path: path,
|
|
149
155
|
verbose: data.options.verbose,
|
|
150
156
|
};
|
|
151
157
|
const scanner = await (0, fs_1.createFileScanner)({
|
|
152
158
|
onProgress: data.onProgress,
|
|
153
159
|
glob: {
|
|
154
|
-
cwd:
|
|
160
|
+
cwd: path,
|
|
155
161
|
onlyFiles: false,
|
|
156
162
|
include: await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions),
|
|
157
163
|
ignore: pkg.exclude
|
|
@@ -170,7 +176,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
170
176
|
const packs = [defaultsPack, ...configPacks];
|
|
171
177
|
const defaultsPackIndex = packs.findIndex((p) => p === defaultsPack);
|
|
172
178
|
const stream = (0, fs_1.createWriteStreamPool)({
|
|
173
|
-
path: await
|
|
179
|
+
path: await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "stream-pool"),
|
|
174
180
|
onStreamPath: (key) => `files-${key}.txt`,
|
|
175
181
|
});
|
|
176
182
|
scanner.total++;
|
|
@@ -192,7 +198,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
192
198
|
scanner.total++;
|
|
193
199
|
stream.writeLine(packIndex, ".");
|
|
194
200
|
}
|
|
195
|
-
stream.writeLine(packIndex, entry.path);
|
|
201
|
+
stream.writeLine(packIndex, (0, tar_1.normalizeTarPath)(entry.path));
|
|
196
202
|
return true;
|
|
197
203
|
});
|
|
198
204
|
await stream.end();
|
|
@@ -208,25 +214,29 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
208
214
|
tarStats[packBasename] = {
|
|
209
215
|
files: stream.lines(packIndex),
|
|
210
216
|
size: 0,
|
|
217
|
+
checksum: "",
|
|
211
218
|
};
|
|
212
219
|
const tarPath = (0, path_1.join)(outPath, packBasename);
|
|
213
220
|
await (0, tar_1.createTar)({
|
|
214
221
|
compress: pack.compress,
|
|
215
222
|
verbose: data.options.verbose,
|
|
216
223
|
includeList,
|
|
217
|
-
path:
|
|
224
|
+
path: path,
|
|
218
225
|
output: tarPath,
|
|
219
|
-
onEntry: async (data) =>
|
|
226
|
+
onEntry: async (data) => scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
|
|
220
227
|
});
|
|
228
|
+
scanner.progress("Fetching tar stats", (0, path_1.basename)(tarPath));
|
|
229
|
+
tarStats[packBasename].checksum = await (0, crypto_1.calcFileHash)(tarPath, "sha1");
|
|
221
230
|
tarStats[packBasename].size = (await (0, promises_1.stat)(tarPath)).size;
|
|
222
231
|
if (!fs.isLocal()) {
|
|
232
|
+
scanner.progress("Uploading tar", (0, path_1.basename)(tarPath));
|
|
223
233
|
await fs.upload(tarPath, `${snapshotName}/${packBasename}`);
|
|
224
234
|
await (0, promises_1.rm)(tarPath);
|
|
225
235
|
}
|
|
226
236
|
}
|
|
227
237
|
packIndex++;
|
|
228
238
|
}
|
|
229
|
-
|
|
239
|
+
scanner.end();
|
|
230
240
|
// Meta
|
|
231
241
|
const metaPath = `${snapshotName}/meta.json`;
|
|
232
242
|
const nodePkg = (0, fs_1.parsePackageFile)();
|
|
@@ -244,7 +254,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
244
254
|
(0, cli_1.logExec)(`Writing metadata into ${fs.resolvePath(metaPath)}`);
|
|
245
255
|
await fs.writeFile(`${snapshotName}/meta.json`, JSON.stringify(meta));
|
|
246
256
|
}
|
|
247
|
-
async
|
|
257
|
+
async copy(data) {
|
|
248
258
|
const sourceFs = (0, client_1.createFs)(this.config.backend);
|
|
249
259
|
const targetFs = (0, client_1.createFs)(data.mirrorRepositoryConfig.backend);
|
|
250
260
|
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
@@ -253,13 +263,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
253
263
|
await targetFs.mkdir(snapshotName);
|
|
254
264
|
await targetFs.ensureEmptyDir(snapshotName);
|
|
255
265
|
const entries = await sourceFs.readdir(snapshotName);
|
|
266
|
+
const total = entries.length;
|
|
267
|
+
let current = 0;
|
|
256
268
|
for (const entry of entries) {
|
|
269
|
+
data.onProgress({
|
|
270
|
+
absolute: {
|
|
271
|
+
current,
|
|
272
|
+
description: "Copying",
|
|
273
|
+
payload: entry,
|
|
274
|
+
total,
|
|
275
|
+
percent: (0, math_1.progressPercent)(total, current),
|
|
276
|
+
},
|
|
277
|
+
});
|
|
278
|
+
current++;
|
|
257
279
|
const sourceEntry = `${snapshotName}/${entry}`;
|
|
258
280
|
if (targetFs.isLocal()) {
|
|
259
281
|
await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
|
|
260
282
|
}
|
|
261
283
|
else {
|
|
262
|
-
const tempDir = await
|
|
284
|
+
const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "remote-copy", entry);
|
|
263
285
|
const tempFile = (0, path_1.join)(tempDir, entry);
|
|
264
286
|
try {
|
|
265
287
|
await sourceFs.download(sourceEntry, tempFile);
|
|
@@ -271,12 +293,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
271
293
|
}
|
|
272
294
|
}
|
|
273
295
|
}
|
|
274
|
-
async
|
|
296
|
+
async restore(data) {
|
|
275
297
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
276
|
-
const relRestorePath = data.
|
|
298
|
+
const relRestorePath = data.snapshotPath;
|
|
277
299
|
(0, assert_1.ok)(relRestorePath);
|
|
278
300
|
const restorePath = (0, path_1.resolve)(relRestorePath);
|
|
279
|
-
const [snapshot] = await this.
|
|
301
|
+
const [snapshot] = await this.fetchSnapshots({
|
|
280
302
|
options: {
|
|
281
303
|
ids: [data.options.snapshotId],
|
|
282
304
|
},
|
|
@@ -286,12 +308,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
286
308
|
const snapshotName = DatatruckRepository.buildSnapshotName(snapshot, data.package);
|
|
287
309
|
const meta = await DatatruckRepository.parseMetaData(await fs.readFile(`${snapshotName}/meta.json`));
|
|
288
310
|
const progress = (0, fs_1.createProgress)({ onProgress: data.onProgress });
|
|
289
|
-
|
|
311
|
+
progress.update("Scanning files");
|
|
290
312
|
const entries = (await fs.readdir(snapshotName)).filter((v) => v.endsWith(".tar") || v.endsWith(".tar.gz"));
|
|
291
313
|
const tarStats = meta?.tarStats || {};
|
|
292
314
|
for (const file in tarStats)
|
|
293
315
|
progress.total += tarStats[file].files;
|
|
294
|
-
|
|
316
|
+
progress.update(`Scanned files: ${progress.total}`);
|
|
295
317
|
if (data.options.verbose)
|
|
296
318
|
(0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
|
|
297
319
|
for (const entry of entries) {
|
|
@@ -299,7 +321,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
299
321
|
try {
|
|
300
322
|
const sourceEntry = `${snapshotName}/${entry}`;
|
|
301
323
|
if (!fs.isLocal()) {
|
|
302
|
-
const tempDir = await
|
|
324
|
+
const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "restore", "remote-fs", entry);
|
|
303
325
|
tempEntry = `${tempDir}/${entry}`;
|
|
304
326
|
await fs.download(sourceEntry, tempEntry);
|
|
305
327
|
}
|
|
@@ -309,7 +331,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
309
331
|
output: restorePath,
|
|
310
332
|
decompress: entry.endsWith(".tar.gz"),
|
|
311
333
|
verbose: data.options.verbose,
|
|
312
|
-
onEntry:
|
|
334
|
+
onEntry: (data) => progress.update(entry.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
|
|
313
335
|
});
|
|
314
336
|
}
|
|
315
337
|
finally {
|
|
@@ -317,7 +339,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
317
339
|
await (0, fs_1.tryRm)(tempEntry);
|
|
318
340
|
}
|
|
319
341
|
}
|
|
320
|
-
|
|
342
|
+
progress.update("Finished");
|
|
321
343
|
}
|
|
322
344
|
}
|
|
323
345
|
exports.DatatruckRepository = DatatruckRepository;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RepositoryAbstract,
|
|
1
|
+
import { RepositoryAbstract, RepoBackupData, RepoInitData, RepoRestoreData, RepoFetchSnapshotsData, Snapshot, SnapshotTagEnum, SnapshotTagObjectType, RepoPruneData, RepoCopyData } from "./RepositoryAbstract";
|
|
2
2
|
import { JSONSchema7 } from "json-schema";
|
|
3
3
|
export type GitRepositoryConfigType = {
|
|
4
4
|
repo: string;
|
|
@@ -10,7 +10,8 @@ export declare const gitRepositoryDefinition: JSONSchema7;
|
|
|
10
10
|
export declare const gitPackageRepositoryDefinition: JSONSchema7;
|
|
11
11
|
export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfigType> {
|
|
12
12
|
static refPrefix: string;
|
|
13
|
-
|
|
13
|
+
getSource(): string;
|
|
14
|
+
fetchDiskStats(config: GitRepositoryConfigType): Promise<import("../utils/fs").DiskStats | undefined>;
|
|
14
15
|
static buildSnapshotTagName(tag: Pick<SnapshotTagObjectType, SnapshotTagEnum.PACKAGE | SnapshotTagEnum.ID>): string;
|
|
15
16
|
static buildSnapshotTag(tag: SnapshotTagObjectType): {
|
|
16
17
|
name: string;
|
|
@@ -21,10 +22,10 @@ export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfi
|
|
|
21
22
|
tags: string[];
|
|
22
23
|
}) | null;
|
|
23
24
|
static buildBranchName(packageName: string): string;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
init(data: RepoInitData): Promise<void>;
|
|
26
|
+
prune(data: RepoPruneData): Promise<void>;
|
|
27
|
+
fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
28
|
+
backup(data: RepoBackupData<GitPackageRepositoryConfigType>): Promise<void>;
|
|
29
|
+
copy(data: RepoCopyData<GitRepositoryConfigType>): Promise<void>;
|
|
30
|
+
restore(data: RepoRestoreData<GitPackageRepositoryConfigType>): Promise<void>;
|
|
30
31
|
}
|