@datatruck/cli 0.26.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/RestoreAction.d.ts +1 -1
- package/Action/RestoreAction.js +1 -1
- package/Command/RestoreCommand.d.ts +1 -1
- package/Command/RestoreCommand.js +2 -2
- package/Command/StartServerCommand.d.ts +6 -0
- package/Command/StartServerCommand.js +24 -0
- package/Config/Config.d.ts +2 -0
- package/Config/Config.js +26 -0
- package/Factory/CommandFactory.d.ts +4 -1
- package/Factory/CommandFactory.js +6 -0
- package/Repository/DatatruckRepository.d.ts +8 -8
- package/Repository/DatatruckRepository.js +97 -132
- package/Task/MysqlDumpTask.js +1 -1
- package/Task/SqlDumpTaskAbstract.js +1 -1
- package/cli.js +1 -0
- package/config.schema.json +38 -2
- package/package.json +1 -1
- package/utils/cli.js +3 -1
- package/utils/datatruck/client.d.ts +22 -0
- package/utils/datatruck/client.js +96 -0
- package/utils/datatruck/config.d.ts +6 -6
- package/utils/datatruck/server.d.ts +21 -0
- package/utils/datatruck/server.js +96 -0
- package/utils/fs.d.ts +13 -6
- package/utils/fs.js +34 -19
- package/utils/http.d.ts +21 -0
- package/utils/http.js +154 -0
- package/utils/virtual-fs.d.ts +33 -0
- package/utils/virtual-fs.js +59 -0
package/Action/RestoreAction.js
CHANGED
|
@@ -9,7 +9,7 @@ export type RestoreCommandOptionsType<TResolved = false> = {
|
|
|
9
9
|
repository?: If<TResolved, string[]>;
|
|
10
10
|
repositoryType?: If<TResolved, RepositoryConfigType["type"][]>;
|
|
11
11
|
tag?: If<TResolved, string[]>;
|
|
12
|
-
|
|
12
|
+
restorePath?: boolean;
|
|
13
13
|
};
|
|
14
14
|
export declare class RestoreCommand extends CommandAbstract<RestoreCommandOptionsType<false>, RestoreCommandOptionsType<true>> {
|
|
15
15
|
onOptions(): import("../utils/cli").OptionsType<RestoreCommandOptionsType<false>, RestoreCommandOptionsType<true>>;
|
|
@@ -21,7 +21,7 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
21
21
|
option: "-p,--package <values>",
|
|
22
22
|
parser: string_1.parseStringList,
|
|
23
23
|
},
|
|
24
|
-
|
|
24
|
+
restorePath: {
|
|
25
25
|
description: "Disable restore path",
|
|
26
26
|
option: "--no-restore-path",
|
|
27
27
|
},
|
|
@@ -63,7 +63,7 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
63
63
|
repositoryTypes: this.options.repositoryType,
|
|
64
64
|
tags: this.options.tag,
|
|
65
65
|
verbose: verbose > 0,
|
|
66
|
-
|
|
66
|
+
restorePath: this.options.restorePath,
|
|
67
67
|
});
|
|
68
68
|
const sessionManager = new RestoreSessionManager_1.RestoreSessionManager({
|
|
69
69
|
driver: new SqliteSessionDriver_1.SqliteSessionDriver({
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { CommandAbstract } from "./CommandAbstract";
|
|
2
|
+
export type StartServerCommandOptionsType<TResolved = false> = {};
|
|
3
|
+
export declare class StartServerCommand extends CommandAbstract<StartServerCommandOptionsType<false>, StartServerCommandOptionsType<true>> {
|
|
4
|
+
onOptions(): import("../utils/cli").OptionsType<StartServerCommandOptionsType<false>, StartServerCommandOptionsType<true>>;
|
|
5
|
+
onExec(): Promise<number>;
|
|
6
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.StartServerCommand = void 0;
|
|
4
|
+
const ConfigAction_1 = require("../Action/ConfigAction");
|
|
5
|
+
const server_1 = require("../utils/datatruck/server");
|
|
6
|
+
const CommandAbstract_1 = require("./CommandAbstract");
|
|
7
|
+
class StartServerCommand extends CommandAbstract_1.CommandAbstract {
|
|
8
|
+
onOptions() {
|
|
9
|
+
return this.returnsOptions({});
|
|
10
|
+
}
|
|
11
|
+
async onExec() {
|
|
12
|
+
const config = await ConfigAction_1.ConfigAction.fromGlobalOptions(this.globalOptions);
|
|
13
|
+
const server = (0, server_1.createDatatruckServer)(config.server || {});
|
|
14
|
+
const port = config.server?.listen?.port ?? 8888;
|
|
15
|
+
const address = config.server?.listen?.address ?? "127.0.0.1";
|
|
16
|
+
console.info(`Listening on http://${address}:${port}`);
|
|
17
|
+
await new Promise((resolve, reject) => {
|
|
18
|
+
server.listen(port, address);
|
|
19
|
+
server.on("error", reject);
|
|
20
|
+
});
|
|
21
|
+
return 0;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
exports.StartServerCommand = StartServerCommand;
|
package/Config/Config.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { DatatruckServerOptions } from "../utils/datatruck/server";
|
|
1
2
|
import { PackageConfigType } from "./PackageConfig";
|
|
2
3
|
import { RepositoryConfigType } from "./RepositoryConfig";
|
|
3
4
|
import type { JSONSchema7 } from "json-schema";
|
|
@@ -5,5 +6,6 @@ export type ConfigType = {
|
|
|
5
6
|
tempDir?: string;
|
|
6
7
|
repositories: RepositoryConfigType[];
|
|
7
8
|
packages: PackageConfigType[];
|
|
9
|
+
server?: DatatruckServerOptions;
|
|
8
10
|
};
|
|
9
11
|
export declare const configDefinition: JSONSchema7;
|
package/Config/Config.js
CHANGED
|
@@ -17,5 +17,31 @@ exports.configDefinition = {
|
|
|
17
17
|
type: "array",
|
|
18
18
|
items: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.package),
|
|
19
19
|
},
|
|
20
|
+
server: {
|
|
21
|
+
type: "object",
|
|
22
|
+
additionalProperties: false,
|
|
23
|
+
properties: {
|
|
24
|
+
path: { type: "string" },
|
|
25
|
+
users: {
|
|
26
|
+
type: "array",
|
|
27
|
+
items: {
|
|
28
|
+
type: "object",
|
|
29
|
+
additionalProperties: false,
|
|
30
|
+
properties: {
|
|
31
|
+
name: { type: "string" },
|
|
32
|
+
password: { type: "string" },
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
listen: {
|
|
37
|
+
type: "object",
|
|
38
|
+
additionalProperties: false,
|
|
39
|
+
properties: {
|
|
40
|
+
port: { type: "integer" },
|
|
41
|
+
address: { type: "string" },
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
},
|
|
20
46
|
},
|
|
21
47
|
};
|
|
@@ -9,6 +9,7 @@ import { PruneCommandOptionsType } from "../Command/PruneCommand";
|
|
|
9
9
|
import { RestoreCommandOptionsType } from "../Command/RestoreCommand";
|
|
10
10
|
import { RestoreSessionsCommandOptionsType } from "../Command/RestoreSessionsCommand";
|
|
11
11
|
import { SnapshotsCommandLogType, SnapshotsCommandOptionsType } from "../Command/SnapshotsCommand";
|
|
12
|
+
import { StartServerCommandOptionsType } from "../Command/StartServerCommand";
|
|
12
13
|
export declare enum CommandEnum {
|
|
13
14
|
config = "config",
|
|
14
15
|
init = "init",
|
|
@@ -18,7 +19,8 @@ export declare enum CommandEnum {
|
|
|
18
19
|
backupSessions = "backup-sessions",
|
|
19
20
|
restore = "restore",
|
|
20
21
|
restoreSessions = "restore-sessions",
|
|
21
|
-
cleanCache = "clean-cache"
|
|
22
|
+
cleanCache = "clean-cache",
|
|
23
|
+
startServer = "start-server"
|
|
22
24
|
}
|
|
23
25
|
export type OptionsMapType = {
|
|
24
26
|
[CommandEnum.config]: ConfigCommandOptionsType;
|
|
@@ -30,6 +32,7 @@ export type OptionsMapType = {
|
|
|
30
32
|
[CommandEnum.restore]: RestoreCommandOptionsType;
|
|
31
33
|
[CommandEnum.restoreSessions]: RestoreSessionsCommandOptionsType;
|
|
32
34
|
[CommandEnum.cleanCache]: CleanCacheActionOptionsType;
|
|
35
|
+
[CommandEnum.startServer]: StartServerCommandOptionsType;
|
|
33
36
|
};
|
|
34
37
|
export type LogMapType = {
|
|
35
38
|
[CommandEnum.config]: ConfigCommandLogType;
|
|
@@ -10,6 +10,7 @@ const PruneCommand_1 = require("../Command/PruneCommand");
|
|
|
10
10
|
const RestoreCommand_1 = require("../Command/RestoreCommand");
|
|
11
11
|
const RestoreSessionsCommand_1 = require("../Command/RestoreSessionsCommand");
|
|
12
12
|
const SnapshotsCommand_1 = require("../Command/SnapshotsCommand");
|
|
13
|
+
const StartServerCommand_1 = require("../Command/StartServerCommand");
|
|
13
14
|
const AppError_1 = require("../Error/AppError");
|
|
14
15
|
var CommandEnum;
|
|
15
16
|
(function (CommandEnum) {
|
|
@@ -22,6 +23,7 @@ var CommandEnum;
|
|
|
22
23
|
CommandEnum["restore"] = "restore";
|
|
23
24
|
CommandEnum["restoreSessions"] = "restore-sessions";
|
|
24
25
|
CommandEnum["cleanCache"] = "clean-cache";
|
|
26
|
+
CommandEnum["startServer"] = "start-server";
|
|
25
27
|
})(CommandEnum || (exports.CommandEnum = CommandEnum = {}));
|
|
26
28
|
function CommandFactory(type, globalOptions, options) {
|
|
27
29
|
const constructor = CommandConstructorFactory(type);
|
|
@@ -74,6 +76,7 @@ function makeParseLog(type) {
|
|
|
74
76
|
const data = [];
|
|
75
77
|
const consoleLog = console.log;
|
|
76
78
|
console.log = console.info = (...items) => {
|
|
79
|
+
consoleLog.bind(console)(...items);
|
|
77
80
|
data.push(...items);
|
|
78
81
|
};
|
|
79
82
|
return function parseLog() {
|
|
@@ -110,6 +113,9 @@ function CommandConstructorFactory(type) {
|
|
|
110
113
|
else if (type === CommandEnum.cleanCache) {
|
|
111
114
|
return CleanCacheCommand_1.CleanCacheCommand;
|
|
112
115
|
}
|
|
116
|
+
else if (type === CommandEnum.startServer) {
|
|
117
|
+
return StartServerCommand_1.StartServerCommand;
|
|
118
|
+
}
|
|
113
119
|
else {
|
|
114
120
|
throw new AppError_1.AppError(`Invalid command type: ${type}`);
|
|
115
121
|
}
|
|
@@ -11,10 +11,11 @@ export type MetaDataType = {
|
|
|
11
11
|
size: number;
|
|
12
12
|
tarStats?: Record<string, {
|
|
13
13
|
files: number;
|
|
14
|
+
size: number;
|
|
14
15
|
}>;
|
|
15
16
|
};
|
|
16
17
|
export type DatatruckRepositoryConfigType = {
|
|
17
|
-
|
|
18
|
+
backend: string;
|
|
18
19
|
compress?: boolean | CompressOptions;
|
|
19
20
|
};
|
|
20
21
|
type PackObject = {
|
|
@@ -33,10 +34,11 @@ export declare const datatruckRepositoryDefinition: JSONSchema7;
|
|
|
33
34
|
export declare const datatruckPackageRepositoryDefinition: JSONSchema7;
|
|
34
35
|
export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRepositoryConfigType> {
|
|
35
36
|
static zipBasenameTpl: string;
|
|
36
|
-
static buildSnapshotName(
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
static buildSnapshotName(snapshot: {
|
|
38
|
+
id: string;
|
|
39
|
+
date: string;
|
|
40
|
+
}, pkg: {
|
|
41
|
+
name: string;
|
|
40
42
|
}): string;
|
|
41
43
|
static parseSnapshotName(name: string): {
|
|
42
44
|
snapshotDate: string;
|
|
@@ -44,9 +46,7 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
|
|
|
44
46
|
snapshotShortId: string;
|
|
45
47
|
sourcePath: string;
|
|
46
48
|
} | null;
|
|
47
|
-
|
|
48
|
-
static parseMetaData(path: string): Promise<MetaDataType | undefined>;
|
|
49
|
-
static stringifyMetaData(data: MetaDataType): string;
|
|
49
|
+
static parseMetaData(data: string): Promise<MetaDataType>;
|
|
50
50
|
onGetSource(): string;
|
|
51
51
|
onInit(data: InitDataType): Promise<void>;
|
|
52
52
|
onPrune(data: PruneDataType): Promise<void>;
|
|
@@ -4,6 +4,7 @@ exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exp
|
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
6
6
|
const cli_1 = require("../utils/cli");
|
|
7
|
+
const client_1 = require("../utils/datatruck/client");
|
|
7
8
|
const paths_1 = require("../utils/datatruck/paths");
|
|
8
9
|
const fs_1 = require("../utils/fs");
|
|
9
10
|
const string_1 = require("../utils/string");
|
|
@@ -16,10 +17,10 @@ const path_1 = require("path");
|
|
|
16
17
|
exports.datatruckRepositoryName = "datatruck";
|
|
17
18
|
exports.datatruckRepositoryDefinition = {
|
|
18
19
|
type: "object",
|
|
19
|
-
required: ["
|
|
20
|
+
required: ["backend"],
|
|
20
21
|
additionalProperties: false,
|
|
21
22
|
properties: {
|
|
22
|
-
|
|
23
|
+
backend: { type: "string" },
|
|
23
24
|
compress: {
|
|
24
25
|
anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
|
|
25
26
|
},
|
|
@@ -52,10 +53,10 @@ exports.datatruckPackageRepositoryDefinition = {
|
|
|
52
53
|
};
|
|
53
54
|
class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
54
55
|
static zipBasenameTpl = `.*.dd.tar.gz`;
|
|
55
|
-
static buildSnapshotName(
|
|
56
|
-
const date =
|
|
57
|
-
const pkgName = encodeURIComponent(
|
|
58
|
-
const snapshotShortId =
|
|
56
|
+
static buildSnapshotName(snapshot, pkg) {
|
|
57
|
+
const date = snapshot.date.replace(/:/g, "-");
|
|
58
|
+
const pkgName = encodeURIComponent(pkg.name).replace(/%40/g, "@");
|
|
59
|
+
const snapshotShortId = snapshot.id.slice(0, 8);
|
|
59
60
|
return `${date}_${pkgName}_${snapshotShortId}`;
|
|
60
61
|
}
|
|
61
62
|
static parseSnapshotName(name) {
|
|
@@ -69,49 +70,32 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
69
70
|
packageName = decodeURIComponent(packageName);
|
|
70
71
|
return { snapshotDate, packageName, snapshotShortId, sourcePath: name };
|
|
71
72
|
}
|
|
72
|
-
|
|
73
|
-
return (
|
|
74
|
-
}
|
|
75
|
-
static async parseMetaData(path) {
|
|
76
|
-
let contents;
|
|
77
|
-
try {
|
|
78
|
-
contents = await (0, promises_1.readFile)(path);
|
|
79
|
-
}
|
|
80
|
-
catch (error) {
|
|
81
|
-
if ((0, fs_1.isNotFoundError)(error))
|
|
82
|
-
return;
|
|
83
|
-
throw error;
|
|
84
|
-
}
|
|
85
|
-
return JSON.parse(contents.toString());
|
|
86
|
-
}
|
|
87
|
-
static stringifyMetaData(data) {
|
|
88
|
-
return JSON.stringify(data);
|
|
73
|
+
static async parseMetaData(data) {
|
|
74
|
+
return JSON.parse(data.toString());
|
|
89
75
|
}
|
|
90
76
|
onGetSource() {
|
|
91
|
-
return this.config.
|
|
77
|
+
return this.config.backend;
|
|
92
78
|
}
|
|
93
79
|
async onInit(data) {
|
|
94
|
-
|
|
80
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
81
|
+
await fs.mkdir(".");
|
|
95
82
|
}
|
|
96
83
|
async onPrune(data) {
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
packageName: data.snapshot.packageName,
|
|
84
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
85
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, {
|
|
86
|
+
name: data.snapshot.packageName,
|
|
101
87
|
});
|
|
102
|
-
const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
103
88
|
if (data.options.verbose)
|
|
104
|
-
(0, cli_1.logExec)(`Deleting ${
|
|
105
|
-
if (await
|
|
106
|
-
await (
|
|
107
|
-
recursive: true,
|
|
108
|
-
});
|
|
89
|
+
(0, cli_1.logExec)(`Deleting ${fs.resolvePath(snapshotName)}`);
|
|
90
|
+
if (await fs.existsDir(snapshotName))
|
|
91
|
+
await fs.rmAll(snapshotName);
|
|
109
92
|
}
|
|
110
93
|
async onSnapshots(data) {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
94
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
95
|
+
if (!(await fs.existsDir(".")))
|
|
96
|
+
throw new Error(`Repository (${this.repository.name}) out path does not exist: ${fs.resolvePath(".")}`);
|
|
114
97
|
const snapshots = [];
|
|
98
|
+
const snapshotNames = await fs.readdir(".");
|
|
115
99
|
const packagePatterns = (0, string_1.makePathPatterns)(data.options.packageNames);
|
|
116
100
|
const taskPatterns = (0, string_1.makePathPatterns)(data.options.packageTaskNames);
|
|
117
101
|
for (const snapshotName of snapshotNames) {
|
|
@@ -124,8 +108,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
124
108
|
if (data.options.ids &&
|
|
125
109
|
!data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
|
|
126
110
|
continue;
|
|
127
|
-
const
|
|
128
|
-
const meta = await DatatruckRepository.parseMetaData(
|
|
111
|
+
const metaData = await fs.readFileIfExists(`${snapshotName}/meta.json`);
|
|
112
|
+
const meta = !!metaData && (await DatatruckRepository.parseMetaData(metaData));
|
|
129
113
|
if (!meta)
|
|
130
114
|
continue;
|
|
131
115
|
if (taskPatterns && !(0, string_1.checkMatch)(meta.task, taskPatterns))
|
|
@@ -149,16 +133,15 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
149
133
|
return snapshots;
|
|
150
134
|
}
|
|
151
135
|
async onBackup(data) {
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
const outPath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
|
|
136
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
137
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
138
|
+
const outPath = fs.isLocal()
|
|
139
|
+
? fs.resolvePath(snapshotName)
|
|
140
|
+
: await this.mkTmpDir("datatruck-backup");
|
|
158
141
|
const pkg = data.package;
|
|
159
142
|
const sourcePath = data.targetPath ?? pkg.path;
|
|
160
143
|
(0, assert_1.ok)(sourcePath);
|
|
161
|
-
await
|
|
144
|
+
await fs.mkdir(snapshotName);
|
|
162
145
|
const backupPathsOptions = {
|
|
163
146
|
package: data.package,
|
|
164
147
|
snapshot: data.snapshot,
|
|
@@ -224,21 +207,28 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
224
207
|
if (includeList) {
|
|
225
208
|
tarStats[packBasename] = {
|
|
226
209
|
files: stream.lines(packIndex),
|
|
210
|
+
size: 0,
|
|
227
211
|
};
|
|
212
|
+
const tarPath = (0, path_1.join)(outPath, packBasename);
|
|
228
213
|
await (0, tar_1.createTar)({
|
|
229
214
|
compress: pack.compress,
|
|
230
215
|
verbose: data.options.verbose,
|
|
231
216
|
includeList,
|
|
232
217
|
path: sourcePath,
|
|
233
|
-
output:
|
|
218
|
+
output: tarPath,
|
|
234
219
|
onEntry: async (data) => await scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
|
|
235
220
|
});
|
|
221
|
+
tarStats[packBasename].size = (await (0, promises_1.stat)(tarPath)).size;
|
|
222
|
+
if (!fs.isLocal()) {
|
|
223
|
+
await fs.upload(tarPath, `${snapshotName}/${packBasename}`);
|
|
224
|
+
await (0, promises_1.rm)(tarPath);
|
|
225
|
+
}
|
|
236
226
|
}
|
|
237
227
|
packIndex++;
|
|
238
228
|
}
|
|
239
229
|
await scanner.end();
|
|
240
230
|
// Meta
|
|
241
|
-
const metaPath = `${
|
|
231
|
+
const metaPath = `${snapshotName}/meta.json`;
|
|
242
232
|
const nodePkg = (0, fs_1.parsePackageFile)();
|
|
243
233
|
const meta = {
|
|
244
234
|
id: data.snapshot.id,
|
|
@@ -247,47 +237,42 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
247
237
|
package: data.package.name,
|
|
248
238
|
task: data.package.task?.name,
|
|
249
239
|
version: nodePkg.version,
|
|
250
|
-
size:
|
|
240
|
+
size: Object.values(tarStats).reduce((total, { size }) => total + size, 0),
|
|
251
241
|
tarStats,
|
|
252
242
|
};
|
|
253
243
|
if (data.options.verbose)
|
|
254
|
-
(0, cli_1.logExec)(`Writing metadata into ${metaPath}`);
|
|
255
|
-
await
|
|
244
|
+
(0, cli_1.logExec)(`Writing metadata into ${fs.resolvePath(metaPath)}`);
|
|
245
|
+
await fs.writeFile(`${snapshotName}/meta.json`, JSON.stringify(meta));
|
|
256
246
|
}
|
|
257
247
|
async onCopyBackup(data) {
|
|
258
|
-
const
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
packageName: data.package.name,
|
|
262
|
-
});
|
|
263
|
-
const sourcePath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
|
|
264
|
-
const targetPath = (0, path_1.resolve)((0, path_1.join)(data.mirrorRepositoryConfig.outPath, snapshotName));
|
|
248
|
+
const sourceFs = (0, client_1.createFs)(this.config.backend);
|
|
249
|
+
const targetFs = (0, client_1.createFs)(data.mirrorRepositoryConfig.backend);
|
|
250
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
265
251
|
if (data.options.verbose)
|
|
266
|
-
(0, cli_1.logExec)(`Copying backup files to ${
|
|
267
|
-
await
|
|
268
|
-
await
|
|
269
|
-
const
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
await (0, promises_1.cp)(sourceFile, targetFile);
|
|
252
|
+
(0, cli_1.logExec)(`Copying backup files to ${data.mirrorRepositoryConfig.backend}`);
|
|
253
|
+
await targetFs.mkdir(snapshotName);
|
|
254
|
+
await targetFs.ensureEmptyDir(snapshotName);
|
|
255
|
+
const entries = await sourceFs.readdir(snapshotName);
|
|
256
|
+
for (const entry of entries) {
|
|
257
|
+
const sourceEntry = `${snapshotName}/${entry}`;
|
|
258
|
+
if (targetFs.isLocal()) {
|
|
259
|
+
await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
|
|
260
|
+
}
|
|
261
|
+
else {
|
|
262
|
+
const tempDir = await this.mkTmpDir("remote-copy", entry);
|
|
263
|
+
const tempFile = (0, path_1.join)(tempDir, entry);
|
|
264
|
+
try {
|
|
265
|
+
await sourceFs.download(sourceEntry, tempFile);
|
|
266
|
+
await targetFs.upload(tempFile, sourceEntry);
|
|
267
|
+
}
|
|
268
|
+
finally {
|
|
269
|
+
await (0, fs_1.tryRm)(tempFile);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
287
272
|
}
|
|
288
|
-
await scanner.end();
|
|
289
273
|
}
|
|
290
274
|
async onRestore(data) {
|
|
275
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
291
276
|
const relRestorePath = data.targetPath ?? data.package.restorePath;
|
|
292
277
|
(0, assert_1.ok)(relRestorePath);
|
|
293
278
|
const restorePath = (0, path_1.resolve)(relRestorePath);
|
|
@@ -298,61 +283,41 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
298
283
|
});
|
|
299
284
|
if (!snapshot)
|
|
300
285
|
throw new AppError_1.AppError("Snapshot not found");
|
|
301
|
-
const snapshotName = DatatruckRepository.buildSnapshotName(
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
307
|
-
const metaPath = (0, path_1.join)(sourcePath, "meta.json");
|
|
308
|
-
const meta = await DatatruckRepository.parseMetaData(metaPath);
|
|
309
|
-
const scanner = await (0, fs_1.createFileScanner)({
|
|
310
|
-
onProgress: data.onProgress,
|
|
311
|
-
glob: {
|
|
312
|
-
cwd: sourcePath,
|
|
313
|
-
include: ["**/*"],
|
|
314
|
-
},
|
|
315
|
-
});
|
|
316
|
-
const tarFiles = [];
|
|
286
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(snapshot, data.package);
|
|
287
|
+
const meta = await DatatruckRepository.parseMetaData(await fs.readFile(`${snapshotName}/meta.json`));
|
|
288
|
+
const progress = (0, fs_1.createProgress)({ onProgress: data.onProgress });
|
|
289
|
+
await progress.update("Scanning files");
|
|
290
|
+
const entries = (await fs.readdir(snapshotName)).filter((v) => v.endsWith(".tar") || v.endsWith(".tar.gz"));
|
|
317
291
|
const tarStats = meta?.tarStats || {};
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
const isTarGz = entry.name.endsWith(".tar.gz");
|
|
322
|
-
if (isTar || isTarGz) {
|
|
323
|
-
tarFiles.push(path);
|
|
324
|
-
if (typeof tarStats[entry.name]?.files === "number") {
|
|
325
|
-
scanner.total += tarStats[entry.name].files;
|
|
326
|
-
}
|
|
327
|
-
else {
|
|
328
|
-
scanner.progress("Scanning", entry.name, false);
|
|
329
|
-
const selfTarStats = (tarStats[entry.name] = { files: 0 });
|
|
330
|
-
await (0, tar_1.listTar)({
|
|
331
|
-
input: path,
|
|
332
|
-
verbose: data.options.verbose,
|
|
333
|
-
onEntry: () => {
|
|
334
|
-
scanner.total++;
|
|
335
|
-
selfTarStats.files++;
|
|
336
|
-
},
|
|
337
|
-
});
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
return false;
|
|
341
|
-
});
|
|
292
|
+
for (const file in tarStats)
|
|
293
|
+
progress.total += tarStats[file].files;
|
|
294
|
+
await progress.update(`Scanned files: ${progress.total}`);
|
|
342
295
|
if (data.options.verbose)
|
|
343
296
|
(0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
|
|
344
|
-
for (const
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
297
|
+
for (const entry of entries) {
|
|
298
|
+
let tempEntry;
|
|
299
|
+
try {
|
|
300
|
+
const sourceEntry = `${snapshotName}/${entry}`;
|
|
301
|
+
if (!fs.isLocal()) {
|
|
302
|
+
const tempDir = await this.mkTmpDir("remote-restore", entry);
|
|
303
|
+
tempEntry = `${tempDir}/${entry}`;
|
|
304
|
+
await fs.download(sourceEntry, tempEntry);
|
|
305
|
+
}
|
|
306
|
+
await (0, tar_1.extractTar)({
|
|
307
|
+
total: tarStats[entry].files,
|
|
308
|
+
input: tempEntry ?? fs.resolvePath(sourceEntry),
|
|
309
|
+
output: restorePath,
|
|
310
|
+
decompress: entry.endsWith(".tar.gz"),
|
|
311
|
+
verbose: data.options.verbose,
|
|
312
|
+
onEntry: async (data) => await progress.update(entry.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
finally {
|
|
316
|
+
if (tempEntry)
|
|
317
|
+
await (0, fs_1.tryRm)(tempEntry);
|
|
318
|
+
}
|
|
354
319
|
}
|
|
355
|
-
await
|
|
320
|
+
await progress.update("Finished");
|
|
356
321
|
}
|
|
357
322
|
}
|
|
358
323
|
exports.DatatruckRepository = DatatruckRepository;
|
package/Task/MysqlDumpTask.js
CHANGED
|
@@ -163,7 +163,7 @@ class MysqlDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
|
163
163
|
const database = {
|
|
164
164
|
name: (0, config_1.resolveDatabaseName)(this.config.database, params),
|
|
165
165
|
};
|
|
166
|
-
if (this.config.targetDatabase &&
|
|
166
|
+
if (this.config.targetDatabase && data.options.restorePath)
|
|
167
167
|
database.name = (0, config_1.resolveDatabaseName)(this.config.targetDatabase.name, {
|
|
168
168
|
...params,
|
|
169
169
|
database: database.name,
|
|
@@ -192,7 +192,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
|
|
|
192
192
|
database: undefined,
|
|
193
193
|
}),
|
|
194
194
|
};
|
|
195
|
-
if (this.config.targetDatabase &&
|
|
195
|
+
if (this.config.targetDatabase && data.options.restorePath) {
|
|
196
196
|
database.name = (0, config_1.resolveDatabaseName)(this.config.targetDatabase.name, {
|
|
197
197
|
packageName: data.package.name,
|
|
198
198
|
snapshotId: data.options.snapshotId,
|
package/cli.js
CHANGED
|
@@ -83,6 +83,7 @@ program.option("-c,--config <path>", "Config path", process.env["DATATRUCK_CONFI
|
|
|
83
83
|
program.option("--progress <value>", "Progress type (auto, plain, tty)", "auto");
|
|
84
84
|
program.option("--progress-interval <ms>", "Progress interval", Number, 1000);
|
|
85
85
|
program.option("-o,--output-format <format>", "Output format (json, pjson, yaml, table, custom=$, tpl=name)", "table");
|
|
86
|
+
makeCommand(CommandFactory_1.CommandEnum.startServer).alias("start");
|
|
86
87
|
makeCommand(CommandFactory_1.CommandEnum.config).alias("c");
|
|
87
88
|
makeCommand(CommandFactory_1.CommandEnum.init).alias("i");
|
|
88
89
|
makeCommand(CommandFactory_1.CommandEnum.snapshots).alias("s");
|
package/config.schema.json
CHANGED
|
@@ -444,11 +444,11 @@
|
|
|
444
444
|
"datatruck-repository": {
|
|
445
445
|
"type": "object",
|
|
446
446
|
"required": [
|
|
447
|
-
"
|
|
447
|
+
"backend"
|
|
448
448
|
],
|
|
449
449
|
"additionalProperties": false,
|
|
450
450
|
"properties": {
|
|
451
|
-
"
|
|
451
|
+
"backend": {
|
|
452
452
|
"type": "string"
|
|
453
453
|
},
|
|
454
454
|
"compress": {
|
|
@@ -996,6 +996,42 @@
|
|
|
996
996
|
"items": {
|
|
997
997
|
"$ref": "#/definitions/package"
|
|
998
998
|
}
|
|
999
|
+
},
|
|
1000
|
+
"server": {
|
|
1001
|
+
"type": "object",
|
|
1002
|
+
"additionalProperties": false,
|
|
1003
|
+
"properties": {
|
|
1004
|
+
"path": {
|
|
1005
|
+
"type": "string"
|
|
1006
|
+
},
|
|
1007
|
+
"users": {
|
|
1008
|
+
"type": "array",
|
|
1009
|
+
"items": {
|
|
1010
|
+
"type": "object",
|
|
1011
|
+
"additionalProperties": false,
|
|
1012
|
+
"properties": {
|
|
1013
|
+
"name": {
|
|
1014
|
+
"type": "string"
|
|
1015
|
+
},
|
|
1016
|
+
"password": {
|
|
1017
|
+
"type": "string"
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
},
|
|
1022
|
+
"listen": {
|
|
1023
|
+
"type": "object",
|
|
1024
|
+
"additionalProperties": false,
|
|
1025
|
+
"properties": {
|
|
1026
|
+
"port": {
|
|
1027
|
+
"type": "integer"
|
|
1028
|
+
},
|
|
1029
|
+
"address": {
|
|
1030
|
+
"type": "string"
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
999
1035
|
}
|
|
1000
1036
|
}
|
|
1001
1037
|
},
|