@datatruck/cli 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/BackupAction.d.ts +1 -1
- package/Action/BackupAction.js +14 -14
- package/Action/BackupSessionsAction.d.ts +1 -1
- package/Action/CleanCacheAction.d.ts +1 -1
- package/Action/CleanCacheAction.js +3 -3
- package/Action/ConfigAction.d.ts +1 -1
- package/Action/ConfigAction.js +5 -5
- package/Action/InitAction.d.ts +1 -1
- package/Action/InitAction.js +2 -2
- package/Action/PruneAction.d.ts +1 -1
- package/Action/PruneAction.js +4 -4
- package/Action/RestoreAction.d.ts +1 -1
- package/Action/RestoreAction.js +18 -18
- package/Action/RestoreSessionsAction.d.ts +1 -1
- package/Action/SnapshotsAction.d.ts +1 -1
- package/Action/SnapshotsAction.js +4 -4
- package/Command/BackupCommand.d.ts +2 -2
- package/Command/BackupCommand.js +6 -6
- package/Command/BackupSessionsCommand.d.ts +2 -2
- package/Command/BackupSessionsCommand.js +9 -9
- package/Command/CleanCacheCommand.d.ts +1 -1
- package/Command/CleanCacheCommand.js +2 -2
- package/Command/CommandAbstract.d.ts +3 -3
- package/Command/CommandAbstract.js +2 -2
- package/Command/ConfigCommand.d.ts +2 -2
- package/Command/ConfigCommand.js +8 -8
- package/Command/InitCommand.d.ts +2 -2
- package/Command/InitCommand.js +9 -9
- package/Command/PruneCommand.d.ts +2 -2
- package/Command/PruneCommand.js +10 -10
- package/Command/RestoreCommand.d.ts +2 -2
- package/Command/RestoreCommand.js +6 -6
- package/Command/RestoreSessionsCommand.d.ts +2 -2
- package/Command/RestoreSessionsCommand.js +9 -9
- package/Command/SnapshotsCommand.d.ts +2 -2
- package/Command/SnapshotsCommand.js +9 -9
- package/Entity/StateEntityAbstract.d.ts +1 -1
- package/Repository/DatatruckRepository.d.ts +1 -2
- package/Repository/DatatruckRepository.js +56 -118
- package/Repository/GitRepository.js +23 -23
- package/Repository/RepositoryAbstract.d.ts +1 -1
- package/Repository/RepositoryAbstract.js +2 -2
- package/Repository/ResticRepository.d.ts +1 -1
- package/Repository/ResticRepository.js +27 -27
- package/SessionDriver/ConsoleSessionDriver.d.ts +1 -1
- package/SessionDriver/ConsoleSessionDriver.js +10 -10
- package/SessionDriver/SqliteSessionDriver.js +12 -12
- package/SessionManager/BackupSessionManager.d.ts +2 -2
- package/SessionManager/BackupSessionManager.js +1 -1
- package/SessionManager/RestoreSessionManager.d.ts +2 -2
- package/SessionManager/RestoreSessionManager.js +1 -1
- package/Task/GitTask.js +21 -21
- package/Task/MariadbTask.d.ts +11 -0
- package/Task/MariadbTask.js +240 -58
- package/Task/MssqlTask.js +8 -8
- package/Task/MysqlDumpTask.d.ts +1 -1
- package/Task/MysqlDumpTask.js +13 -13
- package/Task/PostgresqlDumpTask.d.ts +1 -1
- package/Task/PostgresqlDumpTask.js +4 -4
- package/Task/ScriptTask.js +7 -7
- package/Task/SqlDumpTaskAbstract.d.ts +1 -1
- package/Task/SqlDumpTaskAbstract.js +12 -12
- package/Task/TaskAbstract.d.ts +1 -1
- package/Task/TaskAbstract.js +2 -2
- package/cli.js +10 -10
- package/config.schema.json +44 -0
- package/package.json +3 -3
- package/{util → utils}/DataFormat.d.ts +0 -0
- package/{util → utils}/DataFormat.js +0 -0
- package/{util/GitUtil.d.ts → utils/Git.d.ts} +8 -8
- package/{util/GitUtil.js → utils/Git.js} +8 -8
- package/{util → utils}/ObjectVault.d.ts +0 -0
- package/{util → utils}/ObjectVault.js +0 -0
- package/{util/ResticUtil.d.ts → utils/Restic.d.ts} +3 -3
- package/{util/ResticUtil.js → utils/Restic.js} +9 -9
- package/{util/cli-util.d.ts → utils/cli.d.ts} +0 -0
- package/{util/cli-util.js → utils/cli.js} +0 -0
- package/{util/datatruck/config-util.d.ts → utils/datatruck/config.d.ts} +1 -1
- package/{util/datatruck/config-util.js → utils/datatruck/config.js} +10 -10
- package/{util/datatruck/paths-util.d.ts → utils/datatruck/paths.d.ts} +0 -0
- package/{util/datatruck/paths-util.js → utils/datatruck/paths.js} +2 -2
- package/{util/datatruck/snapshot-util.d.ts → utils/datatruck/snapshot.d.ts} +1 -1
- package/{util/datatruck/snapshot-util.js → utils/datatruck/snapshot.js} +4 -4
- package/{util/date-util.d.ts → utils/date.d.ts} +0 -0
- package/{util/date-util.js → utils/date.js} +2 -2
- package/{util/entity-util.d.ts → utils/entity.d.ts} +0 -0
- package/{util/entity-util.js → utils/entity.js} +0 -0
- package/{util/fs-util.d.ts → utils/fs.d.ts} +23 -2
- package/{util/fs-util.js → utils/fs.js} +83 -24
- package/{util/math-util.d.ts → utils/math.d.ts} +0 -0
- package/{util/math-util.js → utils/math.js} +0 -0
- package/{util/object-util.d.ts → utils/object.d.ts} +0 -0
- package/{util/object-util.js → utils/object.js} +0 -0
- package/{util/path-util.d.ts → utils/path.d.ts} +0 -0
- package/{util/path-util.js → utils/path.js} +0 -0
- package/utils/process.d.ts +107 -0
- package/{util/process-util.js → utils/process.js} +168 -28
- package/{util → utils}/progress.d.ts +0 -0
- package/{util → utils}/progress.js +0 -0
- package/{util/string-util.d.ts → utils/string.d.ts} +0 -0
- package/{util/string-util.js → utils/string.js} +0 -0
- package/{util/zip-util.d.ts → utils/zip.d.ts} +0 -0
- package/{util/zip-util.js → utils/zip.js} +5 -5
- package/util/process-util.d.ts +0 -55
package/Task/MariadbTask.js
CHANGED
|
@@ -2,15 +2,17 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.MariadbTask = exports.mariadbTaskDefinition = exports.mariadbTaskName = void 0;
|
|
4
4
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
const
|
|
5
|
+
const cli_1 = require("../utils/cli");
|
|
6
|
+
const fs_1 = require("../utils/fs");
|
|
7
|
+
const math_1 = require("../utils/math");
|
|
8
|
+
const process_1 = require("../utils/process");
|
|
9
|
+
const zip_1 = require("../utils/zip");
|
|
9
10
|
const TaskAbstract_1 = require("./TaskAbstract");
|
|
10
11
|
const assert_1 = require("assert");
|
|
12
|
+
const fs_2 = require("fs");
|
|
11
13
|
const promises_1 = require("fs/promises");
|
|
14
|
+
const os_1 = require("os");
|
|
12
15
|
const path_1 = require("path");
|
|
13
|
-
const posix_1 = require("path/posix");
|
|
14
16
|
exports.mariadbTaskName = "mariadb";
|
|
15
17
|
exports.mariadbTaskDefinition = {
|
|
16
18
|
type: "object",
|
|
@@ -39,8 +41,65 @@ exports.mariadbTaskDefinition = {
|
|
|
39
41
|
excludeTables: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
|
|
40
42
|
includeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
|
|
41
43
|
excludeDatabases: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
|
|
44
|
+
parallel: {
|
|
45
|
+
anyOf: [{ type: "integer", minimum: 1 }, { enum: ["auto"] }],
|
|
46
|
+
},
|
|
47
|
+
compress: {
|
|
48
|
+
anyOf: [
|
|
49
|
+
{
|
|
50
|
+
type: "object",
|
|
51
|
+
additionalProperties: false,
|
|
52
|
+
properties: {
|
|
53
|
+
command: { type: "string" },
|
|
54
|
+
args: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
type: "object",
|
|
59
|
+
additionalProperties: false,
|
|
60
|
+
properties: {
|
|
61
|
+
type: { enum: ["gzip", "pigz"] },
|
|
62
|
+
args: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
],
|
|
66
|
+
},
|
|
42
67
|
},
|
|
43
68
|
};
|
|
69
|
+
function normalizeConfig(input) {
|
|
70
|
+
let parallel = input.parallel ?? "auto";
|
|
71
|
+
let cores = (0, os_1.cpus)().length;
|
|
72
|
+
if (parallel === "auto") {
|
|
73
|
+
parallel = input.compress ? Math.round(cores / 2) : cores;
|
|
74
|
+
cores = Math.max(1, cores - parallel);
|
|
75
|
+
}
|
|
76
|
+
let compress;
|
|
77
|
+
if (input.compress && "type" in input.compress) {
|
|
78
|
+
if (input.compress.type === "pigz") {
|
|
79
|
+
compress = { command: "pigz", args: [] };
|
|
80
|
+
if (!input.compress.args?.includes("-p")) {
|
|
81
|
+
compress.args.push("-p", cores.toString());
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
else if (input.compress.type === "gzip") {
|
|
85
|
+
compress = { command: "gzip" };
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
compress = input.compress;
|
|
90
|
+
}
|
|
91
|
+
return { parallel, compress };
|
|
92
|
+
}
|
|
93
|
+
const parseLineRegex = /^(?:\[(?<step>\d+)\] )?(?<dateTime>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})?\s+(?<text>.+)\s*$/;
|
|
94
|
+
function parseLine(line) {
|
|
95
|
+
const result = parseLineRegex.exec(line);
|
|
96
|
+
if (result) {
|
|
97
|
+
return result.groups;
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
return { text: line };
|
|
101
|
+
}
|
|
102
|
+
}
|
|
44
103
|
class MariadbTask extends TaskAbstract_1.TaskAbstract {
|
|
45
104
|
get command() {
|
|
46
105
|
return this.config.command ?? "mariabackup";
|
|
@@ -58,18 +117,25 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
|
|
|
58
117
|
const targetPath = data.targetPath;
|
|
59
118
|
(0, assert_1.ok)(typeof sourcePath === "string");
|
|
60
119
|
(0, assert_1.ok)(typeof targetPath === "string");
|
|
120
|
+
const { parallel, compress } = normalizeConfig(config);
|
|
61
121
|
const args = [
|
|
62
122
|
`--backup`,
|
|
63
123
|
`--datadir=${sourcePath}`,
|
|
64
|
-
`--target-dir=${targetPath}`,
|
|
65
124
|
`--host=${config.hostname}`,
|
|
66
125
|
`--user=${config.username}`,
|
|
126
|
+
`--parallel=${parallel}`,
|
|
67
127
|
`--password=${typeof config.password === "string"
|
|
68
128
|
? config.password
|
|
69
129
|
: config.password
|
|
70
130
|
? (await (0, promises_1.readFile)(config.password.path)).toString()
|
|
71
131
|
: ""}`,
|
|
72
132
|
];
|
|
133
|
+
if (compress) {
|
|
134
|
+
args.push(`--stream=xbstream`);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
args.push(`--target-dir=${targetPath}`);
|
|
138
|
+
}
|
|
73
139
|
if (config.includeDatabases)
|
|
74
140
|
args.push(`--databases=${config.includeDatabases.join(" ")}`);
|
|
75
141
|
if (config.excludeDatabases)
|
|
@@ -80,76 +146,192 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
|
|
|
80
146
|
args.push(`--tables-exclude=^(${config.excludeTables.join("|")})$`);
|
|
81
147
|
let total = 0;
|
|
82
148
|
let current = 0;
|
|
83
|
-
await (0,
|
|
149
|
+
await (0, fs_1.forEachFile)(sourcePath, () => {
|
|
84
150
|
total++;
|
|
85
151
|
});
|
|
86
|
-
let
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
fatalError = true;
|
|
96
|
-
}
|
|
97
|
-
else {
|
|
98
|
-
const matches = pathRegex.exec(line);
|
|
99
|
-
if (matches) {
|
|
100
|
-
current++;
|
|
101
|
-
paths.push(matches[1]);
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
if (fatalError) {
|
|
106
|
-
childProcess.kill();
|
|
152
|
+
let p1;
|
|
153
|
+
let lastLineText;
|
|
154
|
+
const pathRegex = /((Copying|Streaming) .+) to/;
|
|
155
|
+
const onData = async (line) => {
|
|
156
|
+
const { text } = parseLine(line);
|
|
157
|
+
lastLineText = text;
|
|
158
|
+
if (line.includes("[ERROR] InnoDB: Unsupported redo log format.") ||
|
|
159
|
+
line.includes("Error: cannot read redo log header")) {
|
|
160
|
+
p1.kill();
|
|
107
161
|
}
|
|
108
|
-
else
|
|
109
|
-
const
|
|
162
|
+
else {
|
|
163
|
+
const matches = pathRegex.exec(text);
|
|
164
|
+
if (matches)
|
|
165
|
+
current++;
|
|
110
166
|
await data.onProgress({
|
|
111
167
|
relative: {
|
|
112
|
-
|
|
113
|
-
payload: path,
|
|
168
|
+
payload: matches ? matches[1] : text,
|
|
114
169
|
},
|
|
115
170
|
absolute: {
|
|
116
171
|
current,
|
|
117
|
-
percent: (0,
|
|
172
|
+
percent: (0, math_1.progressPercent)(total, current),
|
|
118
173
|
total,
|
|
119
174
|
},
|
|
120
175
|
});
|
|
121
176
|
}
|
|
122
177
|
};
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
178
|
+
const stats = { xbFiles: total };
|
|
179
|
+
await (0, promises_1.writeFile)((0, path_1.join)(targetPath, "stats.dtt.json"), JSON.stringify(stats));
|
|
180
|
+
if (compress) {
|
|
181
|
+
const p0 = (0, fs_2.createWriteStream)((0, path_1.join)(targetPath, "db.xb.gz"));
|
|
182
|
+
p1 = (0, process_1.createProcess)(command, args, {
|
|
183
|
+
$log: {
|
|
184
|
+
exec: this.verbose,
|
|
185
|
+
stderr: this.verbose,
|
|
186
|
+
},
|
|
187
|
+
$stderr: {
|
|
188
|
+
parseLines: true,
|
|
189
|
+
onData,
|
|
190
|
+
},
|
|
191
|
+
$onExitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
|
|
192
|
+
});
|
|
193
|
+
const p2 = (0, process_1.createProcess)(compress.command, compress.args);
|
|
194
|
+
p1.stdout.pipe(p2.stdin, { end: true });
|
|
195
|
+
p2.stdout.pipe(p0, { end: true });
|
|
196
|
+
await Promise.all([p1, p2, (0, fs_1.waitForClose)(p0)]);
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
p1 = (0, process_1.createProcess)(command, args, {
|
|
200
|
+
$log: this.verbose,
|
|
201
|
+
$stdout: {
|
|
202
|
+
parseLines: true,
|
|
203
|
+
onData,
|
|
204
|
+
},
|
|
205
|
+
$stderr: {
|
|
206
|
+
parseLines: true,
|
|
207
|
+
onData,
|
|
208
|
+
},
|
|
209
|
+
$onExitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
|
|
210
|
+
});
|
|
211
|
+
await p1;
|
|
212
|
+
await (0, process_1.exec)(command, [`--prepare`, `--target-dir=${targetPath}`], undefined, {
|
|
213
|
+
log: this.verbose,
|
|
214
|
+
stderr: { onData: () => { } },
|
|
215
|
+
});
|
|
216
|
+
}
|
|
139
217
|
}
|
|
140
218
|
async onRestore(data) {
|
|
141
219
|
this.verbose = data.options.verbose;
|
|
142
220
|
const restorePath = data.package.restorePath;
|
|
143
221
|
(0, assert_1.ok)(typeof restorePath === "string");
|
|
144
|
-
await (0,
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
222
|
+
await (0, fs_1.mkdirIfNotExists)(restorePath);
|
|
223
|
+
const removeFiles = [];
|
|
224
|
+
let files = [];
|
|
225
|
+
const reloadFiles = async (data = {}) => {
|
|
226
|
+
if (data.removeFile)
|
|
227
|
+
removeFiles.push(data.removeFile);
|
|
228
|
+
return (files = (await (0, fs_1.readDir)(restorePath)).filter((v) => !removeFiles.includes(v)));
|
|
229
|
+
};
|
|
230
|
+
await reloadFiles();
|
|
231
|
+
const absolute = {
|
|
232
|
+
format: "amount",
|
|
233
|
+
total: 3,
|
|
234
|
+
current: 0,
|
|
235
|
+
description: undefined,
|
|
236
|
+
payload: undefined,
|
|
237
|
+
percent: undefined,
|
|
238
|
+
};
|
|
239
|
+
// Stats
|
|
240
|
+
const statsFile = files.find((file) => file === "stats.dtt.json");
|
|
241
|
+
let stats;
|
|
242
|
+
if (statsFile) {
|
|
243
|
+
const statsFilePath = (0, path_1.join)(restorePath, statsFile);
|
|
244
|
+
const statsBuffer = await (0, promises_1.readFile)(statsFilePath);
|
|
245
|
+
stats = JSON.parse(statsBuffer.toString());
|
|
246
|
+
await reloadFiles({ removeFile: statsFile });
|
|
247
|
+
}
|
|
248
|
+
const zipFile = files.find((file) => file.endsWith(".gz"));
|
|
249
|
+
absolute.current++;
|
|
250
|
+
absolute.percent = (0, math_1.progressPercent)(absolute.total, absolute.current);
|
|
251
|
+
// Extract
|
|
252
|
+
if (files.length === 1 && zipFile) {
|
|
253
|
+
absolute.description = "Extracting";
|
|
254
|
+
absolute.payload = zipFile;
|
|
255
|
+
await data.onProgress({
|
|
256
|
+
absolute,
|
|
257
|
+
});
|
|
258
|
+
await (0, zip_1.unzip)({
|
|
259
|
+
input: (0, path_1.join)(restorePath, zipFile),
|
|
260
|
+
output: restorePath,
|
|
261
|
+
verbose: this.verbose,
|
|
262
|
+
async onProgress(item) {
|
|
263
|
+
await data.onProgress({
|
|
264
|
+
absolute,
|
|
265
|
+
relative: {
|
|
266
|
+
payload: item.path,
|
|
267
|
+
format: "amount",
|
|
268
|
+
percent: item.percent,
|
|
269
|
+
},
|
|
270
|
+
});
|
|
271
|
+
},
|
|
272
|
+
});
|
|
273
|
+
await reloadFiles({ removeFile: zipFile });
|
|
274
|
+
}
|
|
275
|
+
// Extract stream
|
|
276
|
+
const xbFile = files.find((file) => file.endsWith(".xb"));
|
|
277
|
+
absolute.current++;
|
|
278
|
+
absolute.percent = (0, math_1.progressPercent)(absolute.total, absolute.current);
|
|
279
|
+
if (files.length === 1 && xbFile) {
|
|
280
|
+
const xbFilePath = (0, path_1.join)(restorePath, xbFile);
|
|
281
|
+
const xbStream = (0, fs_2.createReadStream)(xbFilePath);
|
|
282
|
+
removeFiles.push(xbFile);
|
|
283
|
+
absolute.description = "Extracting stream";
|
|
284
|
+
absolute.payload = xbFile;
|
|
285
|
+
await data.onProgress({
|
|
286
|
+
absolute,
|
|
287
|
+
});
|
|
288
|
+
let currentXbFiles = 0;
|
|
289
|
+
const { parallel } = normalizeConfig({ parallel: this.config.parallel });
|
|
290
|
+
const p1 = (0, process_1.createProcess)("mbstream", ["-x", "-C", restorePath, "-v", "-p", parallel], {
|
|
291
|
+
$log: this.verbose,
|
|
292
|
+
$stderr: {
|
|
293
|
+
parseLines: true,
|
|
294
|
+
async onData(line) {
|
|
295
|
+
const { text: path } = parseLine(line);
|
|
296
|
+
await data.onProgress({
|
|
297
|
+
absolute,
|
|
298
|
+
relative: {
|
|
299
|
+
payload: path,
|
|
300
|
+
format: "amount",
|
|
301
|
+
current: ++currentXbFiles,
|
|
302
|
+
total: stats?.xbFiles,
|
|
303
|
+
percent: stats?.xbFiles
|
|
304
|
+
? (0, math_1.progressPercent)(stats.xbFiles, currentXbFiles)
|
|
305
|
+
: undefined,
|
|
306
|
+
},
|
|
307
|
+
});
|
|
308
|
+
},
|
|
309
|
+
},
|
|
310
|
+
});
|
|
311
|
+
xbStream.pipe(p1.stdin, { end: true });
|
|
312
|
+
await Promise.all([(0, fs_1.waitForClose)(xbStream), p1]);
|
|
313
|
+
}
|
|
314
|
+
// Prepare
|
|
315
|
+
absolute.current++;
|
|
316
|
+
absolute.percent = (0, math_1.progressPercent)(absolute.total, absolute.current);
|
|
317
|
+
if (files.length === 1 && xbFile) {
|
|
318
|
+
absolute.description = "Preparing";
|
|
319
|
+
await data.onProgress({
|
|
320
|
+
absolute,
|
|
321
|
+
});
|
|
322
|
+
await (0, process_1.exec)(this.command, [`--prepare`, `--target-dir=${restorePath}`], undefined, {
|
|
323
|
+
log: this.verbose,
|
|
324
|
+
stderr: { onData: () => { } },
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
await reloadFiles();
|
|
328
|
+
removeFiles.push(...files.filter((file) => file.startsWith("ib_logfile")));
|
|
329
|
+
// Remove files
|
|
330
|
+
for (const file of removeFiles) {
|
|
331
|
+
const filePath = (0, path_1.join)(restorePath, file);
|
|
332
|
+
if (this.verbose)
|
|
333
|
+
(0, cli_1.logExec)("rm", [filePath]);
|
|
334
|
+
await (0, promises_1.rm)(filePath);
|
|
153
335
|
}
|
|
154
336
|
}
|
|
155
337
|
}
|
package/Task/MssqlTask.js
CHANGED
|
@@ -3,9 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.MssqlTask = exports.mssqlTaskDefinition = exports.mssqlTaskName = void 0;
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
const
|
|
6
|
+
const config_1 = require("../utils/datatruck/config");
|
|
7
|
+
const fs_1 = require("../utils/fs");
|
|
8
|
+
const process_1 = require("../utils/process");
|
|
9
9
|
const TaskAbstract_1 = require("./TaskAbstract");
|
|
10
10
|
const assert_1 = require("assert");
|
|
11
11
|
const promises_1 = require("fs/promises");
|
|
@@ -29,7 +29,7 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
|
|
|
29
29
|
return this.config.command ?? "sqlcmd";
|
|
30
30
|
}
|
|
31
31
|
async exec(query) {
|
|
32
|
-
const result = await (0,
|
|
32
|
+
const result = await (0, process_1.exec)(this.command, [
|
|
33
33
|
...(this.config.hostname ? ["-S", this.config.hostname] : []),
|
|
34
34
|
...(this.config.username ? ["-U", this.config.username] : []),
|
|
35
35
|
...(this.config.passwordFile
|
|
@@ -73,7 +73,7 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
|
|
|
73
73
|
(0, micromatch_1.isMatch)(databaseName, this.config.includeDatabases)) &&
|
|
74
74
|
(!this.config.excludeDatabases ||
|
|
75
75
|
!(0, micromatch_1.isMatch)(databaseName, this.config.excludeDatabases)));
|
|
76
|
-
await (0,
|
|
76
|
+
await (0, fs_1.mkdirIfNotExists)(targetPath);
|
|
77
77
|
for (const databaseName of databaseNames) {
|
|
78
78
|
const databasePath = (0, path_1.join)(targetPath, `${databaseName}${MssqlTask.SUFFIX}`);
|
|
79
79
|
await this.exec(`BACKUP DATABASE [${databaseName}] TO DISK='${databasePath}' WITH FORMAT`);
|
|
@@ -83,14 +83,14 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
|
|
|
83
83
|
this.verbose = data.options.verbose;
|
|
84
84
|
const restorePath = data.package.restorePath;
|
|
85
85
|
(0, assert_1.ok)(typeof restorePath === "string");
|
|
86
|
-
await (0,
|
|
87
|
-
const files = await (0,
|
|
86
|
+
await (0, fs_1.mkdirIfNotExists)(restorePath);
|
|
87
|
+
const files = await (0, fs_1.readDir)(restorePath);
|
|
88
88
|
for (const file of files) {
|
|
89
89
|
if (!file.endsWith(MssqlTask.SUFFIX))
|
|
90
90
|
continue;
|
|
91
91
|
let databaseName = file.slice(0, MssqlTask.SUFFIX.length * -1);
|
|
92
92
|
if (this.config.targetDatabase)
|
|
93
|
-
databaseName = (0,
|
|
93
|
+
databaseName = (0, config_1.resolveDatabaseName)(this.config.targetDatabase, {
|
|
94
94
|
action: "restore",
|
|
95
95
|
database: databaseName,
|
|
96
96
|
packageName: data.package.name,
|
package/Task/MysqlDumpTask.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ export declare class MysqlDumpTask extends SqlDumpTaskAbstract<MysqlDumpTaskConf
|
|
|
7
7
|
buildConnectionArgs(database?: boolean): Promise<string[]>;
|
|
8
8
|
onDatabaseIsEmpty(name: string): Promise<boolean>;
|
|
9
9
|
onCreateDatabase(database: TargetDatabaseType): Promise<void>;
|
|
10
|
-
onExecQuery(query: string): Promise<import("../
|
|
10
|
+
onExecQuery(query: string): Promise<import("../utils/process").ExecResultType>;
|
|
11
11
|
onFetchTableNames(database: string): Promise<string[]>;
|
|
12
12
|
onExportTables(tableNames: string[], output: string, onProgress: (progress: {
|
|
13
13
|
totalBytes: number;
|
package/Task/MysqlDumpTask.js
CHANGED
|
@@ -3,11 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.MysqlDumpTask = exports.mysqlDumpTaskDefinition = exports.mysqlDumpTaskName = void 0;
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
6
|
-
const
|
|
7
|
-
const
|
|
6
|
+
const fs_1 = require("../utils/fs");
|
|
7
|
+
const process_1 = require("../utils/process");
|
|
8
8
|
const SqlDumpTaskAbstract_1 = require("./SqlDumpTaskAbstract");
|
|
9
|
-
const fs_1 = require("fs");
|
|
10
9
|
const fs_2 = require("fs");
|
|
10
|
+
const fs_3 = require("fs");
|
|
11
11
|
exports.mysqlDumpTaskName = "mysql-dump";
|
|
12
12
|
exports.mysqlDumpTaskDefinition = {
|
|
13
13
|
allOf: [(0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.sqlDumpTask)],
|
|
@@ -43,7 +43,7 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
43
43
|
await this.onExecQuery(query);
|
|
44
44
|
}
|
|
45
45
|
async onExecQuery(query) {
|
|
46
|
-
return await (0,
|
|
46
|
+
return await (0, process_1.exec)("mysql", [
|
|
47
47
|
...(await this.buildConnectionArgs()),
|
|
48
48
|
"-e",
|
|
49
49
|
query.replace(/\s{1,}/g, " "),
|
|
@@ -71,13 +71,13 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
71
71
|
`);
|
|
72
72
|
}
|
|
73
73
|
async onExportTables(tableNames, output, onProgress) {
|
|
74
|
-
const stream = (0,
|
|
74
|
+
const stream = (0, fs_2.createWriteStream)(output);
|
|
75
75
|
await Promise.all([
|
|
76
76
|
new Promise((resolve, reject) => {
|
|
77
77
|
stream.on("close", resolve);
|
|
78
78
|
stream.on("error", reject);
|
|
79
79
|
}),
|
|
80
|
-
await (0,
|
|
80
|
+
await (0, process_1.exec)("mysqldump", [
|
|
81
81
|
...(await this.buildConnectionArgs()),
|
|
82
82
|
"--lock-tables=false",
|
|
83
83
|
"--skip-add-drop-table=false",
|
|
@@ -97,8 +97,8 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
97
97
|
},
|
|
98
98
|
}),
|
|
99
99
|
]);
|
|
100
|
-
const headerContents = await (0,
|
|
101
|
-
const footerContents = await (0,
|
|
100
|
+
const headerContents = await (0, fs_1.readPartialFile)(output, [0, 100]);
|
|
101
|
+
const footerContents = await (0, fs_1.readPartialFile)(output, [-100]);
|
|
102
102
|
const successHeader = headerContents.split(/\r?\n/).some((line) => {
|
|
103
103
|
const firstLine = line.trim().toLowerCase();
|
|
104
104
|
return (firstLine.startsWith("-- mysql dump") ||
|
|
@@ -113,13 +113,13 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
113
113
|
throw new AppError_1.AppError("No end line found (incomplete backup)");
|
|
114
114
|
}
|
|
115
115
|
async onExportStoredPrograms(output) {
|
|
116
|
-
const stream = (0,
|
|
116
|
+
const stream = (0, fs_2.createWriteStream)(output);
|
|
117
117
|
await Promise.all([
|
|
118
118
|
new Promise((resolve, reject) => {
|
|
119
119
|
stream.on("close", resolve);
|
|
120
120
|
stream.on("error", reject);
|
|
121
121
|
}),
|
|
122
|
-
await (0,
|
|
122
|
+
await (0, process_1.exec)("mysqldump", [
|
|
123
123
|
...(await this.buildConnectionArgs()),
|
|
124
124
|
"--lock-tables=false",
|
|
125
125
|
"--routines",
|
|
@@ -143,12 +143,12 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
143
143
|
]);
|
|
144
144
|
}
|
|
145
145
|
async onImport(path, database) {
|
|
146
|
-
await (0,
|
|
146
|
+
await (0, process_1.exec)("mysql", [...(await this.buildConnectionArgs(false)), database], null, {
|
|
147
147
|
pipe: {
|
|
148
|
-
stream: (0,
|
|
148
|
+
stream: (0, fs_3.createReadStream)(path),
|
|
149
149
|
onReadProgress: (data) => {
|
|
150
150
|
if (this.verbose)
|
|
151
|
-
(0,
|
|
151
|
+
(0, process_1.logExecStdout)({
|
|
152
152
|
data: JSON.stringify(data),
|
|
153
153
|
colorize: true,
|
|
154
154
|
stderr: true,
|
|
@@ -7,7 +7,7 @@ export declare class PostgresqlDumpTask extends SqlDumpTaskAbstract<PostgresqlDu
|
|
|
7
7
|
buildConnectionArgs(database?: string): Promise<string[]>;
|
|
8
8
|
onDatabaseIsEmpty(name: string): Promise<boolean>;
|
|
9
9
|
onCreateDatabase(database: TargetDatabaseType): Promise<void>;
|
|
10
|
-
onExecQuery(query: string): Promise<import("../
|
|
10
|
+
onExecQuery(query: string): Promise<import("../utils/process").ExecResultType>;
|
|
11
11
|
onFetchTableNames(database: string): Promise<string[]>;
|
|
12
12
|
onExportTables(tableNames: string[], output: string, onProgress: (progress: {
|
|
13
13
|
totalBytes: number;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.PostgresqlDumpTask = exports.postgresqlDumpTaskDefinition = exports.postgresqlDumpTaskName = void 0;
|
|
4
4
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
5
|
-
const
|
|
5
|
+
const process_1 = require("../utils/process");
|
|
6
6
|
const SqlDumpTaskAbstract_1 = require("./SqlDumpTaskAbstract");
|
|
7
7
|
const fs_1 = require("fs");
|
|
8
8
|
const path_1 = require("path");
|
|
@@ -45,7 +45,7 @@ class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
45
45
|
await this.onExecQuery(query);
|
|
46
46
|
}
|
|
47
47
|
async onExecQuery(query) {
|
|
48
|
-
return await (0,
|
|
48
|
+
return await (0, process_1.exec)("psql", [
|
|
49
49
|
...(await this.buildConnectionArgs()),
|
|
50
50
|
"-t",
|
|
51
51
|
"-c",
|
|
@@ -80,7 +80,7 @@ class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
80
80
|
stream.on("close", resolve);
|
|
81
81
|
stream.on("error", reject);
|
|
82
82
|
}),
|
|
83
|
-
(0,
|
|
83
|
+
(0, process_1.exec)("pg_dump", [
|
|
84
84
|
...(await this.buildConnectionArgs()),
|
|
85
85
|
...(tableNames?.flatMap((v) => ["-t", v]) ?? []),
|
|
86
86
|
], null, {
|
|
@@ -96,7 +96,7 @@ class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
96
96
|
throw new Error(`Method not implemented: onExportStoredPrograms`);
|
|
97
97
|
}
|
|
98
98
|
async onImport(path, database) {
|
|
99
|
-
await (0,
|
|
99
|
+
await (0, process_1.exec)("psql", [...(await this.buildConnectionArgs(database)), "-f", (0, path_1.normalize)(path)], undefined, {
|
|
100
100
|
log: this.verbose,
|
|
101
101
|
});
|
|
102
102
|
}
|
package/Task/ScriptTask.js
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ScriptTask = exports.scriptTaskDefinition = exports.scriptTaskName = void 0;
|
|
4
4
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
const
|
|
5
|
+
const fs_1 = require("../utils/fs");
|
|
6
|
+
const process_1 = require("../utils/process");
|
|
7
|
+
const string_1 = require("../utils/string");
|
|
8
8
|
const TaskAbstract_1 = require("./TaskAbstract");
|
|
9
9
|
const assert_1 = require("assert");
|
|
10
10
|
const promises_1 = require("fs/promises");
|
|
@@ -110,7 +110,7 @@ class ScriptTask extends TaskAbstract_1.TaskAbstract {
|
|
|
110
110
|
const steps = Array.isArray(input) ? input : [input];
|
|
111
111
|
for (const step of steps) {
|
|
112
112
|
if (step.type === "process") {
|
|
113
|
-
await (0,
|
|
113
|
+
await (0, process_1.exec)(step.config.command, (step.config.args || []).map((v) => (0, string_1.render)(v, options.vars)), {
|
|
114
114
|
env: {
|
|
115
115
|
...process.env,
|
|
116
116
|
...options.vars,
|
|
@@ -127,7 +127,7 @@ class ScriptTask extends TaskAbstract_1.TaskAbstract {
|
|
|
127
127
|
await (0, promises_1.writeFile)(scriptPath, Array.isArray(step.config.code)
|
|
128
128
|
? step.config.code.join("\n")
|
|
129
129
|
: step.config.code);
|
|
130
|
-
await (0,
|
|
130
|
+
await (0, process_1.exec)("node", [scriptPath], {
|
|
131
131
|
env: {
|
|
132
132
|
...process.env,
|
|
133
133
|
...options.vars,
|
|
@@ -168,8 +168,8 @@ class ScriptTask extends TaskAbstract_1.TaskAbstract {
|
|
|
168
168
|
const targetPath = data.targetPath;
|
|
169
169
|
(0, assert_1.ok)(typeof restorePath === "string");
|
|
170
170
|
(0, assert_1.ok)(typeof targetPath === "string");
|
|
171
|
-
await (0,
|
|
172
|
-
await (0,
|
|
171
|
+
await (0, fs_1.mkdirIfNotExists)(restorePath);
|
|
172
|
+
await (0, fs_1.ensureEmptyDir)(restorePath);
|
|
173
173
|
await this.processSteps(config.restoreSteps, {
|
|
174
174
|
env: config.env,
|
|
175
175
|
vars: this.getVars(data),
|