@datatruck/cli 0.27.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/BackupAction.d.ts +69 -34
- package/Action/BackupAction.js +284 -244
- package/Action/CleanCacheAction.d.ts +8 -4
- package/Action/CleanCacheAction.js +8 -5
- package/Action/ConfigAction.d.ts +12 -5
- package/Action/ConfigAction.js +14 -18
- package/Action/CopyAction.d.ts +49 -0
- package/Action/CopyAction.js +144 -0
- package/Action/InitAction.d.ts +3 -3
- package/Action/InitAction.js +9 -9
- package/Action/PruneAction.d.ts +9 -9
- package/Action/PruneAction.js +39 -23
- package/Action/RestoreAction.d.ts +48 -23
- package/Action/RestoreAction.js +158 -195
- package/Action/SnapshotsAction.d.ts +8 -8
- package/Action/SnapshotsAction.js +8 -8
- package/CHANGELOG.md +495 -0
- package/Command/BackupCommand.d.ts +6 -4
- package/Command/BackupCommand.js +9 -26
- package/Command/CleanCacheCommand.d.ts +4 -4
- package/Command/CleanCacheCommand.js +26 -5
- package/Command/CommandAbstract.d.ts +10 -7
- package/Command/CommandAbstract.js +4 -1
- package/Command/ConfigCommand.d.ts +6 -9
- package/Command/ConfigCommand.js +13 -8
- package/Command/CopyCommand.d.ts +15 -0
- package/Command/CopyCommand.js +61 -0
- package/Command/InitCommand.d.ts +4 -4
- package/Command/InitCommand.js +11 -15
- package/Command/PruneCommand.d.ts +3 -3
- package/Command/PruneCommand.js +13 -12
- package/Command/RestoreCommand.js +9 -17
- package/Command/SnapshotsCommand.d.ts +4 -4
- package/Command/SnapshotsCommand.js +16 -15
- package/Command/StartServerCommand.d.ts +3 -3
- package/Config/Config.d.ts +9 -0
- package/Config/Config.js +17 -0
- package/Config/PrunePolicyConfig.d.ts +2 -2
- package/Factory/CommandFactory.d.ts +27 -34
- package/Factory/CommandFactory.js +27 -54
- package/Factory/RepositoryFactory.d.ts +1 -1
- package/Factory/RepositoryFactory.js +3 -3
- package/Factory/TaskFactory.d.ts +1 -1
- package/Factory/TaskFactory.js +3 -3
- package/Repository/DatatruckRepository.d.ts +9 -8
- package/Repository/DatatruckRepository.js +42 -25
- package/Repository/GitRepository.d.ts +9 -8
- package/Repository/GitRepository.js +22 -25
- package/Repository/RepositoryAbstract.d.ts +39 -37
- package/Repository/RepositoryAbstract.js +4 -5
- package/Repository/ResticRepository.d.ts +9 -8
- package/Repository/ResticRepository.js +30 -28
- package/Task/GitTask.d.ts +6 -7
- package/Task/GitTask.js +24 -30
- package/Task/MariadbTask.d.ts +4 -5
- package/Task/MariadbTask.js +26 -32
- package/Task/MssqlTask.d.ts +5 -3
- package/Task/MssqlTask.js +11 -12
- package/Task/MysqlDumpTask.d.ts +10 -3
- package/Task/MysqlDumpTask.js +107 -31
- package/Task/ScriptTask.d.ts +23 -18
- package/Task/ScriptTask.js +34 -24
- package/Task/SqlDumpTaskAbstract.d.ts +8 -3
- package/Task/SqlDumpTaskAbstract.js +31 -19
- package/Task/TaskAbstract.d.ts +24 -25
- package/Task/TaskAbstract.js +6 -10
- package/cli.js +13 -5
- package/config.schema.json +86 -1
- package/package.json +4 -5
- package/utils/DataFormat.d.ts +23 -12
- package/utils/DataFormat.js +36 -14
- package/utils/cli.d.ts +2 -9
- package/utils/cli.js +9 -52
- package/utils/datatruck/client.d.ts +2 -0
- package/utils/datatruck/client.js +3 -0
- package/utils/datatruck/config.d.ts +2 -0
- package/utils/datatruck/config.js +18 -3
- package/utils/datatruck/paths.d.ts +5 -9
- package/utils/datatruck/paths.js +2 -2
- package/utils/datatruck/snapshot.d.ts +2 -2
- package/utils/date.d.ts +7 -3
- package/utils/date.js +22 -14
- package/utils/fs.d.ts +16 -11
- package/utils/fs.js +81 -48
- package/utils/list.d.ts +64 -0
- package/utils/list.js +145 -0
- package/utils/mysql.d.ts +2 -0
- package/utils/mysql.js +21 -2
- package/utils/process.d.ts +1 -0
- package/utils/process.js +24 -31
- package/utils/progress.d.ts +33 -0
- package/utils/progress.js +113 -0
- package/utils/steps.d.ts +11 -0
- package/utils/steps.js +22 -10
- package/utils/stream.d.ts +7 -0
- package/utils/stream.js +10 -0
- package/utils/string.d.ts +0 -1
- package/utils/string.js +1 -13
- package/utils/tar.d.ts +10 -3
- package/utils/tar.js +70 -44
- package/utils/temp.d.ts +26 -0
- package/utils/temp.js +133 -0
- package/utils/virtual-fs.d.ts +6 -2
- package/utils/virtual-fs.js +6 -0
- package/Action/BackupSessionsAction.d.ts +0 -13
- package/Action/BackupSessionsAction.js +0 -18
- package/Action/RestoreSessionsAction.d.ts +0 -13
- package/Action/RestoreSessionsAction.js +0 -18
- package/Command/BackupSessionsCommand.d.ts +0 -12
- package/Command/BackupSessionsCommand.js +0 -92
- package/Command/RestoreSessionsCommand.d.ts +0 -12
- package/Command/RestoreSessionsCommand.js +0 -91
- package/Decorator/EntityDecorator.d.ts +0 -11
- package/Decorator/EntityDecorator.js +0 -17
- package/Entity/BackupSessionEntity.d.ts +0 -6
- package/Entity/BackupSessionEntity.js +0 -25
- package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
- package/Entity/BackupSessionRepositoryEntity.js +0 -25
- package/Entity/BackupSessionTaskEntity.d.ts +0 -5
- package/Entity/BackupSessionTaskEntity.js +0 -24
- package/Entity/CrudEntityAbstract.d.ts +0 -5
- package/Entity/CrudEntityAbstract.js +0 -9
- package/Entity/RestoreSessionEntity.d.ts +0 -5
- package/Entity/RestoreSessionEntity.js +0 -24
- package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
- package/Entity/RestoreSessionRepositoryEntity.js +0 -25
- package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
- package/Entity/RestoreSessionTaskEntity.js +0 -24
- package/Entity/StateEntityAbstract.d.ts +0 -9
- package/Entity/StateEntityAbstract.js +0 -12
- package/Factory/EntityFactory.d.ts +0 -6
- package/Factory/EntityFactory.js +0 -40
- package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
- package/SessionDriver/ConsoleSessionDriver.js +0 -208
- package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
- package/SessionDriver/SessionDriverAbstract.js +0 -28
- package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
- package/SessionDriver/SqliteSessionDriver.js +0 -173
- package/SessionManager/BackupSessionManager.d.ts +0 -45
- package/SessionManager/BackupSessionManager.js +0 -218
- package/SessionManager/RestoreSessionManager.d.ts +0 -47
- package/SessionManager/RestoreSessionManager.js +0 -218
- package/SessionManager/SessionManagerAbstract.d.ts +0 -18
- package/SessionManager/SessionManagerAbstract.js +0 -36
- package/migrations/001-initial.sql +0 -98
- package/utils/entity.d.ts +0 -4
- package/utils/entity.js +0 -10
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.CommandConstructorFactory = exports.
|
|
3
|
+
exports.CommandConstructorFactory = exports.createActionInterface = exports.exec = exports.CommandFactory = exports.CommandEnum = void 0;
|
|
4
4
|
const BackupCommand_1 = require("../Command/BackupCommand");
|
|
5
|
-
const BackupSessionsCommand_1 = require("../Command/BackupSessionsCommand");
|
|
6
5
|
const CleanCacheCommand_1 = require("../Command/CleanCacheCommand");
|
|
7
6
|
const ConfigCommand_1 = require("../Command/ConfigCommand");
|
|
7
|
+
const CopyCommand_1 = require("../Command/CopyCommand");
|
|
8
8
|
const InitCommand_1 = require("../Command/InitCommand");
|
|
9
9
|
const PruneCommand_1 = require("../Command/PruneCommand");
|
|
10
10
|
const RestoreCommand_1 = require("../Command/RestoreCommand");
|
|
11
|
-
const RestoreSessionsCommand_1 = require("../Command/RestoreSessionsCommand");
|
|
12
11
|
const SnapshotsCommand_1 = require("../Command/SnapshotsCommand");
|
|
13
12
|
const StartServerCommand_1 = require("../Command/StartServerCommand");
|
|
14
13
|
const AppError_1 = require("../Error/AppError");
|
|
14
|
+
const stream_1 = require("stream");
|
|
15
15
|
var CommandEnum;
|
|
16
16
|
(function (CommandEnum) {
|
|
17
17
|
CommandEnum["config"] = "config";
|
|
@@ -19,72 +19,48 @@ var CommandEnum;
|
|
|
19
19
|
CommandEnum["snapshots"] = "snapshots";
|
|
20
20
|
CommandEnum["prune"] = "prune";
|
|
21
21
|
CommandEnum["backup"] = "backup";
|
|
22
|
-
CommandEnum["backupSessions"] = "backup-sessions";
|
|
23
22
|
CommandEnum["restore"] = "restore";
|
|
24
|
-
CommandEnum["
|
|
23
|
+
CommandEnum["copy"] = "copy";
|
|
25
24
|
CommandEnum["cleanCache"] = "clean-cache";
|
|
26
25
|
CommandEnum["startServer"] = "start-server";
|
|
27
26
|
})(CommandEnum || (exports.CommandEnum = CommandEnum = {}));
|
|
28
|
-
function CommandFactory(type, globalOptions, options) {
|
|
27
|
+
function CommandFactory(type, globalOptions, options, streams) {
|
|
29
28
|
const constructor = CommandConstructorFactory(type);
|
|
30
|
-
return new constructor(globalOptions, options);
|
|
29
|
+
return new constructor(globalOptions, options, streams);
|
|
31
30
|
}
|
|
32
31
|
exports.CommandFactory = CommandFactory;
|
|
33
|
-
async function exec(type, globalOptions, options) {
|
|
34
|
-
return await CommandFactory(type, globalOptions, options).onExec();
|
|
32
|
+
async function exec(type, globalOptions, options, streams) {
|
|
33
|
+
return await CommandFactory(type, globalOptions, options, streams).onExec();
|
|
35
34
|
}
|
|
36
35
|
exports.exec = exec;
|
|
37
36
|
function createActionInterface(globalOptions) {
|
|
38
37
|
const object = {};
|
|
39
38
|
for (const type of Object.values(CommandEnum)) {
|
|
40
39
|
object[type] = async (options) => {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
40
|
+
let stdoutData = "";
|
|
41
|
+
const stdout = new stream_1.Writable({
|
|
42
|
+
write(chunk, encoding, callback) {
|
|
43
|
+
stdoutData += chunk.toString();
|
|
44
|
+
process.stdout.write(chunk, encoding, callback);
|
|
45
|
+
},
|
|
46
|
+
}).on("data", (chunk) => (stdoutData += chunk.toString()));
|
|
47
|
+
const end = () => !stdout.closed &&
|
|
48
|
+
new Promise((resolve) => stdout.end().on("close", resolve));
|
|
49
|
+
try {
|
|
50
|
+
const exitCode = await exec(type, { ...globalOptions, outputFormat: "json", verbose: 1 }, options, { stdout });
|
|
51
|
+
if (exitCode !== 0)
|
|
52
|
+
throw new Error(`Invalid exit code: ${exitCode}`);
|
|
53
|
+
await end();
|
|
54
|
+
return JSON.parse(stdoutData);
|
|
48
55
|
}
|
|
49
|
-
|
|
50
|
-
|
|
56
|
+
finally {
|
|
57
|
+
await end();
|
|
51
58
|
}
|
|
52
|
-
if (exitCode !== 0)
|
|
53
|
-
throw new Error(`Invalid exit code: ${exitCode}`);
|
|
54
|
-
return log;
|
|
55
59
|
};
|
|
56
60
|
}
|
|
57
61
|
return object;
|
|
58
62
|
}
|
|
59
63
|
exports.createActionInterface = createActionInterface;
|
|
60
|
-
async function runAndParse(type, run) {
|
|
61
|
-
const parseLog = makeParseLog(type);
|
|
62
|
-
try {
|
|
63
|
-
const exitCode = await run();
|
|
64
|
-
return { exitCode, log: parseLog() };
|
|
65
|
-
}
|
|
66
|
-
catch (error) {
|
|
67
|
-
try {
|
|
68
|
-
parseLog();
|
|
69
|
-
}
|
|
70
|
-
catch (_) { }
|
|
71
|
-
throw error;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
exports.runAndParse = runAndParse;
|
|
75
|
-
function makeParseLog(type) {
|
|
76
|
-
const data = [];
|
|
77
|
-
const consoleLog = console.log;
|
|
78
|
-
console.log = console.info = (...items) => {
|
|
79
|
-
consoleLog.bind(console)(...items);
|
|
80
|
-
data.push(...items);
|
|
81
|
-
};
|
|
82
|
-
return function parseLog() {
|
|
83
|
-
console.log = console.info = consoleLog;
|
|
84
|
-
return JSON.parse(data.flat().join("\n"));
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
exports.makeParseLog = makeParseLog;
|
|
88
64
|
function CommandConstructorFactory(type) {
|
|
89
65
|
if (type === CommandEnum.config) {
|
|
90
66
|
return ConfigCommand_1.ConfigCommand;
|
|
@@ -101,14 +77,11 @@ function CommandConstructorFactory(type) {
|
|
|
101
77
|
else if (type === CommandEnum.backup) {
|
|
102
78
|
return BackupCommand_1.BackupCommand;
|
|
103
79
|
}
|
|
104
|
-
else if (type === CommandEnum.backupSessions) {
|
|
105
|
-
return BackupSessionsCommand_1.BackupSessionsCommand;
|
|
106
|
-
}
|
|
107
80
|
else if (type === CommandEnum.restore) {
|
|
108
81
|
return RestoreCommand_1.RestoreCommand;
|
|
109
82
|
}
|
|
110
|
-
else if (type === CommandEnum.
|
|
111
|
-
return
|
|
83
|
+
else if (type === CommandEnum.copy) {
|
|
84
|
+
return CopyCommand_1.CopyCommand;
|
|
112
85
|
}
|
|
113
86
|
else if (type === CommandEnum.cleanCache) {
|
|
114
87
|
return CleanCacheCommand_1.CleanCacheCommand;
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { RepositoryConfigType } from "../Config/RepositoryConfig";
|
|
2
2
|
import type { RepositoryAbstract } from "../Repository/RepositoryAbstract";
|
|
3
|
-
export declare function
|
|
3
|
+
export declare function createRepo(repository: RepositoryConfigType): RepositoryAbstract<any>;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.createRepo = void 0;
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const DatatruckRepository_1 = require("../Repository/DatatruckRepository");
|
|
6
6
|
const GitRepository_1 = require("../Repository/GitRepository");
|
|
7
7
|
const ResticRepository_1 = require("../Repository/ResticRepository");
|
|
8
|
-
function
|
|
8
|
+
function createRepo(repository) {
|
|
9
9
|
const type = repository.type;
|
|
10
10
|
if (type === GitRepository_1.gitRepositoryName) {
|
|
11
11
|
return new GitRepository_1.GitRepository(repository);
|
|
@@ -20,4 +20,4 @@ function RepositoryFactory(repository) {
|
|
|
20
20
|
throw new AppError_1.AppError(`Invalid repository type: ${type}`);
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
exports.
|
|
23
|
+
exports.createRepo = createRepo;
|
package/Factory/TaskFactory.d.ts
CHANGED
package/Factory/TaskFactory.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.createTask = void 0;
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const GitTask_1 = require("../Task/GitTask");
|
|
6
6
|
const MariadbTask_1 = require("../Task/MariadbTask");
|
|
@@ -8,7 +8,7 @@ const MssqlTask_1 = require("../Task/MssqlTask");
|
|
|
8
8
|
const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
|
|
9
9
|
const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
|
|
10
10
|
const ScriptTask_1 = require("../Task/ScriptTask");
|
|
11
|
-
function
|
|
11
|
+
function createTask(task) {
|
|
12
12
|
if (task.name === GitTask_1.gitTaskName) {
|
|
13
13
|
return new GitTask_1.GitTask(task.config ?? {});
|
|
14
14
|
}
|
|
@@ -31,4 +31,4 @@ function TaskFactory(task) {
|
|
|
31
31
|
throw new AppError_1.AppError(`Invalid task name: ${task["name"]}`);
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
|
-
exports.
|
|
34
|
+
exports.createTask = createTask;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CompressOptions } from "../utils/tar";
|
|
2
|
-
import { RepositoryAbstract,
|
|
2
|
+
import { RepositoryAbstract, RepoBackupData, RepoInitData, RepoRestoreData, RepoFetchSnapshotsData, Snapshot, RepoPruneData, RepoCopyData } from "./RepositoryAbstract";
|
|
3
3
|
import type { JSONSchema7 } from "json-schema";
|
|
4
4
|
export type MetaDataType = {
|
|
5
5
|
id: string;
|
|
@@ -47,12 +47,13 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
|
|
|
47
47
|
sourcePath: string;
|
|
48
48
|
} | null;
|
|
49
49
|
static parseMetaData(data: string): Promise<MetaDataType>;
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
50
|
+
getSource(): string;
|
|
51
|
+
fetchDiskStats(config: DatatruckRepositoryConfigType): Promise<import("../utils/fs").DiskStats>;
|
|
52
|
+
init(data: RepoInitData): Promise<void>;
|
|
53
|
+
prune(data: RepoPruneData): Promise<void>;
|
|
54
|
+
fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
55
|
+
backup(data: RepoBackupData<DatatruckPackageRepositoryConfigType>): Promise<void>;
|
|
56
|
+
copy(data: RepoCopyData<DatatruckRepositoryConfigType>): Promise<void>;
|
|
57
|
+
restore(data: RepoRestoreData<DatatruckPackageRepositoryConfigType>): Promise<void>;
|
|
57
58
|
}
|
|
58
59
|
export {};
|
|
@@ -7,8 +7,10 @@ const cli_1 = require("../utils/cli");
|
|
|
7
7
|
const client_1 = require("../utils/datatruck/client");
|
|
8
8
|
const paths_1 = require("../utils/datatruck/paths");
|
|
9
9
|
const fs_1 = require("../utils/fs");
|
|
10
|
+
const math_1 = require("../utils/math");
|
|
10
11
|
const string_1 = require("../utils/string");
|
|
11
12
|
const tar_1 = require("../utils/tar");
|
|
13
|
+
const temp_1 = require("../utils/temp");
|
|
12
14
|
const RepositoryAbstract_1 = require("./RepositoryAbstract");
|
|
13
15
|
const assert_1 = require("assert");
|
|
14
16
|
const promises_1 = require("fs/promises");
|
|
@@ -73,14 +75,18 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
73
75
|
static async parseMetaData(data) {
|
|
74
76
|
return JSON.parse(data.toString());
|
|
75
77
|
}
|
|
76
|
-
|
|
78
|
+
getSource() {
|
|
77
79
|
return this.config.backend;
|
|
78
80
|
}
|
|
79
|
-
|
|
81
|
+
fetchDiskStats(config) {
|
|
82
|
+
const fs = (0, client_1.createFs)(config.backend);
|
|
83
|
+
return fs.fetchDiskStats(".");
|
|
84
|
+
}
|
|
85
|
+
async init(data) {
|
|
80
86
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
81
87
|
await fs.mkdir(".");
|
|
82
88
|
}
|
|
83
|
-
async
|
|
89
|
+
async prune(data) {
|
|
84
90
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
85
91
|
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, {
|
|
86
92
|
name: data.snapshot.packageName,
|
|
@@ -90,7 +96,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
90
96
|
if (await fs.existsDir(snapshotName))
|
|
91
97
|
await fs.rmAll(snapshotName);
|
|
92
98
|
}
|
|
93
|
-
async
|
|
99
|
+
async fetchSnapshots(data) {
|
|
94
100
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
95
101
|
if (!(await fs.existsDir(".")))
|
|
96
102
|
throw new Error(`Repository (${this.repository.name}) out path does not exist: ${fs.resolvePath(".")}`);
|
|
@@ -132,26 +138,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
132
138
|
}
|
|
133
139
|
return snapshots;
|
|
134
140
|
}
|
|
135
|
-
async
|
|
141
|
+
async backup(data) {
|
|
136
142
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
137
143
|
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
138
144
|
const outPath = fs.isLocal()
|
|
139
145
|
? fs.resolvePath(snapshotName)
|
|
140
|
-
: await
|
|
146
|
+
: await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "fs-remote");
|
|
141
147
|
const pkg = data.package;
|
|
142
|
-
const
|
|
143
|
-
(0, assert_1.ok)(sourcePath);
|
|
148
|
+
const path = pkg.path;
|
|
144
149
|
await fs.mkdir(snapshotName);
|
|
145
150
|
const backupPathsOptions = {
|
|
146
151
|
package: data.package,
|
|
147
152
|
snapshot: data.snapshot,
|
|
148
|
-
|
|
153
|
+
path: path,
|
|
149
154
|
verbose: data.options.verbose,
|
|
150
155
|
};
|
|
151
156
|
const scanner = await (0, fs_1.createFileScanner)({
|
|
152
157
|
onProgress: data.onProgress,
|
|
153
158
|
glob: {
|
|
154
|
-
cwd:
|
|
159
|
+
cwd: path,
|
|
155
160
|
onlyFiles: false,
|
|
156
161
|
include: await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions),
|
|
157
162
|
ignore: pkg.exclude
|
|
@@ -170,7 +175,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
170
175
|
const packs = [defaultsPack, ...configPacks];
|
|
171
176
|
const defaultsPackIndex = packs.findIndex((p) => p === defaultsPack);
|
|
172
177
|
const stream = (0, fs_1.createWriteStreamPool)({
|
|
173
|
-
path: await
|
|
178
|
+
path: await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "stream-pool"),
|
|
174
179
|
onStreamPath: (key) => `files-${key}.txt`,
|
|
175
180
|
});
|
|
176
181
|
scanner.total++;
|
|
@@ -192,7 +197,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
192
197
|
scanner.total++;
|
|
193
198
|
stream.writeLine(packIndex, ".");
|
|
194
199
|
}
|
|
195
|
-
stream.writeLine(packIndex, entry.path);
|
|
200
|
+
stream.writeLine(packIndex, (0, tar_1.normalizeTarPath)(entry.path));
|
|
196
201
|
return true;
|
|
197
202
|
});
|
|
198
203
|
await stream.end();
|
|
@@ -214,9 +219,9 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
214
219
|
compress: pack.compress,
|
|
215
220
|
verbose: data.options.verbose,
|
|
216
221
|
includeList,
|
|
217
|
-
path:
|
|
222
|
+
path: path,
|
|
218
223
|
output: tarPath,
|
|
219
|
-
onEntry: async (data) =>
|
|
224
|
+
onEntry: async (data) => scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
|
|
220
225
|
});
|
|
221
226
|
tarStats[packBasename].size = (await (0, promises_1.stat)(tarPath)).size;
|
|
222
227
|
if (!fs.isLocal()) {
|
|
@@ -226,7 +231,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
226
231
|
}
|
|
227
232
|
packIndex++;
|
|
228
233
|
}
|
|
229
|
-
|
|
234
|
+
scanner.end();
|
|
230
235
|
// Meta
|
|
231
236
|
const metaPath = `${snapshotName}/meta.json`;
|
|
232
237
|
const nodePkg = (0, fs_1.parsePackageFile)();
|
|
@@ -244,7 +249,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
244
249
|
(0, cli_1.logExec)(`Writing metadata into ${fs.resolvePath(metaPath)}`);
|
|
245
250
|
await fs.writeFile(`${snapshotName}/meta.json`, JSON.stringify(meta));
|
|
246
251
|
}
|
|
247
|
-
async
|
|
252
|
+
async copy(data) {
|
|
248
253
|
const sourceFs = (0, client_1.createFs)(this.config.backend);
|
|
249
254
|
const targetFs = (0, client_1.createFs)(data.mirrorRepositoryConfig.backend);
|
|
250
255
|
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
@@ -253,13 +258,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
253
258
|
await targetFs.mkdir(snapshotName);
|
|
254
259
|
await targetFs.ensureEmptyDir(snapshotName);
|
|
255
260
|
const entries = await sourceFs.readdir(snapshotName);
|
|
261
|
+
const total = entries.length;
|
|
262
|
+
let current = 0;
|
|
256
263
|
for (const entry of entries) {
|
|
264
|
+
data.onProgress({
|
|
265
|
+
absolute: {
|
|
266
|
+
current,
|
|
267
|
+
description: "Copying",
|
|
268
|
+
payload: entry,
|
|
269
|
+
total,
|
|
270
|
+
percent: (0, math_1.progressPercent)(total, current),
|
|
271
|
+
},
|
|
272
|
+
});
|
|
273
|
+
current++;
|
|
257
274
|
const sourceEntry = `${snapshotName}/${entry}`;
|
|
258
275
|
if (targetFs.isLocal()) {
|
|
259
276
|
await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
|
|
260
277
|
}
|
|
261
278
|
else {
|
|
262
|
-
const tempDir = await
|
|
279
|
+
const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "remote-copy", entry);
|
|
263
280
|
const tempFile = (0, path_1.join)(tempDir, entry);
|
|
264
281
|
try {
|
|
265
282
|
await sourceFs.download(sourceEntry, tempFile);
|
|
@@ -271,12 +288,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
271
288
|
}
|
|
272
289
|
}
|
|
273
290
|
}
|
|
274
|
-
async
|
|
291
|
+
async restore(data) {
|
|
275
292
|
const fs = (0, client_1.createFs)(this.config.backend);
|
|
276
|
-
const relRestorePath = data.
|
|
293
|
+
const relRestorePath = data.snapshotPath;
|
|
277
294
|
(0, assert_1.ok)(relRestorePath);
|
|
278
295
|
const restorePath = (0, path_1.resolve)(relRestorePath);
|
|
279
|
-
const [snapshot] = await this.
|
|
296
|
+
const [snapshot] = await this.fetchSnapshots({
|
|
280
297
|
options: {
|
|
281
298
|
ids: [data.options.snapshotId],
|
|
282
299
|
},
|
|
@@ -286,12 +303,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
286
303
|
const snapshotName = DatatruckRepository.buildSnapshotName(snapshot, data.package);
|
|
287
304
|
const meta = await DatatruckRepository.parseMetaData(await fs.readFile(`${snapshotName}/meta.json`));
|
|
288
305
|
const progress = (0, fs_1.createProgress)({ onProgress: data.onProgress });
|
|
289
|
-
|
|
306
|
+
progress.update("Scanning files");
|
|
290
307
|
const entries = (await fs.readdir(snapshotName)).filter((v) => v.endsWith(".tar") || v.endsWith(".tar.gz"));
|
|
291
308
|
const tarStats = meta?.tarStats || {};
|
|
292
309
|
for (const file in tarStats)
|
|
293
310
|
progress.total += tarStats[file].files;
|
|
294
|
-
|
|
311
|
+
progress.update(`Scanned files: ${progress.total}`);
|
|
295
312
|
if (data.options.verbose)
|
|
296
313
|
(0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
|
|
297
314
|
for (const entry of entries) {
|
|
@@ -299,7 +316,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
299
316
|
try {
|
|
300
317
|
const sourceEntry = `${snapshotName}/${entry}`;
|
|
301
318
|
if (!fs.isLocal()) {
|
|
302
|
-
const tempDir = await
|
|
319
|
+
const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "restore", "remote-fs", entry);
|
|
303
320
|
tempEntry = `${tempDir}/${entry}`;
|
|
304
321
|
await fs.download(sourceEntry, tempEntry);
|
|
305
322
|
}
|
|
@@ -309,7 +326,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
309
326
|
output: restorePath,
|
|
310
327
|
decompress: entry.endsWith(".tar.gz"),
|
|
311
328
|
verbose: data.options.verbose,
|
|
312
|
-
onEntry:
|
|
329
|
+
onEntry: (data) => progress.update(entry.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
|
|
313
330
|
});
|
|
314
331
|
}
|
|
315
332
|
finally {
|
|
@@ -317,7 +334,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
317
334
|
await (0, fs_1.tryRm)(tempEntry);
|
|
318
335
|
}
|
|
319
336
|
}
|
|
320
|
-
|
|
337
|
+
progress.update("Finished");
|
|
321
338
|
}
|
|
322
339
|
}
|
|
323
340
|
exports.DatatruckRepository = DatatruckRepository;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RepositoryAbstract,
|
|
1
|
+
import { RepositoryAbstract, RepoBackupData, RepoInitData, RepoRestoreData, RepoFetchSnapshotsData, Snapshot, SnapshotTagEnum, SnapshotTagObjectType, RepoPruneData, RepoCopyData } from "./RepositoryAbstract";
|
|
2
2
|
import { JSONSchema7 } from "json-schema";
|
|
3
3
|
export type GitRepositoryConfigType = {
|
|
4
4
|
repo: string;
|
|
@@ -10,7 +10,8 @@ export declare const gitRepositoryDefinition: JSONSchema7;
|
|
|
10
10
|
export declare const gitPackageRepositoryDefinition: JSONSchema7;
|
|
11
11
|
export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfigType> {
|
|
12
12
|
static refPrefix: string;
|
|
13
|
-
|
|
13
|
+
getSource(): string;
|
|
14
|
+
fetchDiskStats(config: GitRepositoryConfigType): Promise<import("../utils/fs").DiskStats | undefined>;
|
|
14
15
|
static buildSnapshotTagName(tag: Pick<SnapshotTagObjectType, SnapshotTagEnum.PACKAGE | SnapshotTagEnum.ID>): string;
|
|
15
16
|
static buildSnapshotTag(tag: SnapshotTagObjectType): {
|
|
16
17
|
name: string;
|
|
@@ -21,10 +22,10 @@ export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfi
|
|
|
21
22
|
tags: string[];
|
|
22
23
|
}) | null;
|
|
23
24
|
static buildBranchName(packageName: string): string;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
init(data: RepoInitData): Promise<void>;
|
|
26
|
+
prune(data: RepoPruneData): Promise<void>;
|
|
27
|
+
fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
28
|
+
backup(data: RepoBackupData<GitPackageRepositoryConfigType>): Promise<void>;
|
|
29
|
+
copy(data: RepoCopyData<GitRepositoryConfigType>): Promise<void>;
|
|
30
|
+
restore(data: RepoRestoreData<GitPackageRepositoryConfigType>): Promise<void>;
|
|
30
31
|
}
|
|
@@ -4,14 +4,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.GitRepository = exports.gitPackageRepositoryDefinition = exports.gitRepositoryDefinition = exports.gitRepositoryName = void 0;
|
|
7
|
-
const AppError_1 = require("../Error/AppError");
|
|
8
7
|
const Git_1 = require("../utils/Git");
|
|
9
8
|
const cli_1 = require("../utils/cli");
|
|
10
9
|
const paths_1 = require("../utils/datatruck/paths");
|
|
11
10
|
const fs_1 = require("../utils/fs");
|
|
12
11
|
const string_1 = require("../utils/string");
|
|
12
|
+
const temp_1 = require("../utils/temp");
|
|
13
13
|
const RepositoryAbstract_1 = require("./RepositoryAbstract");
|
|
14
|
-
const assert_1 = require("assert");
|
|
15
14
|
const fast_glob_1 = __importDefault(require("fast-glob"));
|
|
16
15
|
const promises_1 = require("fs/promises");
|
|
17
16
|
const micromatch_1 = require("micromatch");
|
|
@@ -33,9 +32,13 @@ exports.gitPackageRepositoryDefinition = {
|
|
|
33
32
|
};
|
|
34
33
|
class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
35
34
|
static refPrefix = "dt";
|
|
36
|
-
|
|
35
|
+
getSource() {
|
|
37
36
|
return this.config.repo;
|
|
38
37
|
}
|
|
38
|
+
async fetchDiskStats(config) {
|
|
39
|
+
if ((0, fs_1.isLocalDir)(config.repo))
|
|
40
|
+
return await (0, fs_1.fetchDiskStats)(config.repo);
|
|
41
|
+
}
|
|
39
42
|
static buildSnapshotTagName(tag) {
|
|
40
43
|
return `${GitRepository.refPrefix}/${tag.package}/${tag.id}`;
|
|
41
44
|
}
|
|
@@ -56,9 +59,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
56
59
|
static buildBranchName(packageName) {
|
|
57
60
|
return `${GitRepository.refPrefix}/${packageName}`;
|
|
58
61
|
}
|
|
59
|
-
async
|
|
62
|
+
async init(data) {
|
|
60
63
|
const git = new Git_1.Git({
|
|
61
|
-
dir: (0,
|
|
64
|
+
dir: (0, temp_1.tmpDir)(exports.gitRepositoryName, "repository", "init"),
|
|
62
65
|
log: data.options.verbose,
|
|
63
66
|
});
|
|
64
67
|
if (await git.canBeInit(this.config.repo)) {
|
|
@@ -83,9 +86,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
83
86
|
await git.push({ branchName });
|
|
84
87
|
}
|
|
85
88
|
}
|
|
86
|
-
async
|
|
89
|
+
async prune(data) {
|
|
87
90
|
const git = new Git_1.Git({
|
|
88
|
-
dir: await
|
|
91
|
+
dir: await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "prune"),
|
|
89
92
|
log: data.options.verbose,
|
|
90
93
|
});
|
|
91
94
|
const branchName = GitRepository.buildBranchName(data.snapshot.packageName);
|
|
@@ -106,9 +109,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
106
109
|
await git.exec(["push", "origin", branchName, "--force-with-lease"]);
|
|
107
110
|
await git.exec(["push", "--delete", "origin", data.snapshot.originalId]);
|
|
108
111
|
}
|
|
109
|
-
async
|
|
112
|
+
async fetchSnapshots(data) {
|
|
110
113
|
const git = new Git_1.Git({
|
|
111
|
-
dir: await
|
|
114
|
+
dir: await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "snapshots"),
|
|
112
115
|
log: data.options.verbose,
|
|
113
116
|
});
|
|
114
117
|
const pkgPatterns = (0, string_1.makePathPatterns)(data.options.packageNames);
|
|
@@ -143,13 +146,10 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
143
146
|
}, [])
|
|
144
147
|
.sort((a, b) => a.date.localeCompare(b.date));
|
|
145
148
|
}
|
|
146
|
-
async
|
|
149
|
+
async backup(data) {
|
|
147
150
|
const pkg = data.package;
|
|
148
|
-
const
|
|
149
|
-
(0,
|
|
150
|
-
if (!(await (0, fs_1.existsDir)(sourcePath)))
|
|
151
|
-
throw new AppError_1.AppError(`Package path not exists: ${sourcePath}`);
|
|
152
|
-
const tmpPath = await this.mkTmpDir(GitRepository.name + "-backup");
|
|
151
|
+
const path = pkg.path;
|
|
152
|
+
const tmpPath = await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "backup");
|
|
153
153
|
const branchName = GitRepository.buildBranchName(data.package.name);
|
|
154
154
|
const git = new Git_1.Git({
|
|
155
155
|
dir: tmpPath,
|
|
@@ -169,7 +169,7 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
169
169
|
const backupPathsOptions = {
|
|
170
170
|
package: data.package,
|
|
171
171
|
snapshot: data.snapshot,
|
|
172
|
-
|
|
172
|
+
path: path,
|
|
173
173
|
verbose: data.options.verbose,
|
|
174
174
|
};
|
|
175
175
|
const include = await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions);
|
|
@@ -177,13 +177,13 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
177
177
|
? await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions)
|
|
178
178
|
: undefined;
|
|
179
179
|
const stream = await (0, fast_glob_1.default)(include, {
|
|
180
|
-
cwd:
|
|
180
|
+
cwd: path,
|
|
181
181
|
ignore: exclude,
|
|
182
182
|
dot: true,
|
|
183
183
|
});
|
|
184
184
|
let files = 0;
|
|
185
185
|
for await (const entry of stream) {
|
|
186
|
-
const source = (0, path_1.join)(
|
|
186
|
+
const source = (0, path_1.join)(path, entry);
|
|
187
187
|
const target = (0, path_1.join)(tmpPath, entry);
|
|
188
188
|
const dir = (0, path_1.dirname)(target);
|
|
189
189
|
if (!createdPaths.includes(dir)) {
|
|
@@ -215,16 +215,13 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
215
215
|
await git.addTag(meta.name, meta.message);
|
|
216
216
|
await git.push({ branchName });
|
|
217
217
|
await git.pushTags();
|
|
218
|
-
await (0, promises_1.rm)(tmpPath, {
|
|
219
|
-
recursive: true,
|
|
220
|
-
});
|
|
218
|
+
await (0, promises_1.rm)(tmpPath, { recursive: true });
|
|
221
219
|
}
|
|
222
|
-
|
|
220
|
+
copy(data) {
|
|
223
221
|
throw new Error("Method not implemented.");
|
|
224
222
|
}
|
|
225
|
-
async
|
|
226
|
-
const restorePath = data.
|
|
227
|
-
(0, assert_1.ok)(restorePath);
|
|
223
|
+
async restore(data) {
|
|
224
|
+
const restorePath = data.snapshotPath;
|
|
228
225
|
const tagName = GitRepository.buildSnapshotTagName({
|
|
229
226
|
id: data.snapshot.id,
|
|
230
227
|
package: data.package.name,
|