@datatruck/cli 0.13.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/BackupAction.js +7 -19
- package/Action/RestoreAction.js +4 -12
- package/Command/BackupCommand.js +2 -1
- package/Command/BackupSessionsCommand.js +1 -0
- package/Command/RestoreCommand.js +1 -1
- package/Command/RestoreSessionsCommand.js +1 -0
- package/Entity/StateEntityAbstract.d.ts +2 -5
- package/Error/AppError.d.ts +1 -0
- package/Error/AppError.js +4 -0
- package/Repository/DatatruckRepository.d.ts +2 -0
- package/Repository/DatatruckRepository.js +234 -123
- package/Repository/RepositoryAbstract.d.ts +4 -10
- package/Repository/ResticRepository.js +34 -17
- package/SessionDriver/ConsoleSessionDriver.d.ts +2 -7
- package/SessionDriver/ConsoleSessionDriver.js +51 -24
- package/SessionDriver/SqliteSessionDriver.js +5 -0
- package/SessionManager/BackupSessionManager.d.ts +12 -11
- package/SessionManager/BackupSessionManager.js +20 -5
- package/SessionManager/RestoreSessionManager.d.ts +14 -11
- package/SessionManager/RestoreSessionManager.js +20 -5
- package/SessionManager/SessionManagerAbstract.d.ts +18 -0
- package/SessionManager/SessionManagerAbstract.js +32 -0
- package/Task/GitTask.js +22 -14
- package/Task/MariadbTask.js +9 -4
- package/Task/MysqlDumpTask.d.ts +3 -1
- package/Task/MysqlDumpTask.js +5 -2
- package/Task/PostgresqlDumpTask.d.ts +3 -1
- package/Task/PostgresqlDumpTask.js +2 -2
- package/Task/SqlDumpTaskAbstract.d.ts +3 -1
- package/Task/SqlDumpTaskAbstract.js +55 -13
- package/Task/TaskAbstract.d.ts +3 -9
- package/cli.js +1 -1
- package/config.schema.json +3 -0
- package/migrations/001-initial.sql +6 -30
- package/package.json +1 -1
- package/util/cli-util.d.ts +1 -1
- package/util/cli-util.js +17 -2
- package/util/fs-util.d.ts +25 -21
- package/util/fs-util.js +60 -93
- package/util/math-util.js +2 -0
- package/util/process-util.d.ts +4 -0
- package/util/process-util.js +31 -4
- package/util/progress.d.ts +12 -0
- package/util/progress.js +2 -0
- package/util/string-util.d.ts +1 -0
- package/util/string-util.js +8 -1
- package/util/zip-util.d.ts +64 -20
- package/util/zip-util.js +153 -59
package/Action/BackupAction.js
CHANGED
|
@@ -15,7 +15,7 @@ class BackupAction {
|
|
|
15
15
|
}
|
|
16
16
|
async init(session) {
|
|
17
17
|
const snapshot = {
|
|
18
|
-
id: (0, crypto_1.
|
|
18
|
+
id: (0, crypto_1.randomUUID)().replaceAll("-", ""),
|
|
19
19
|
date: this.options.date ?? new Date().toISOString(),
|
|
20
20
|
};
|
|
21
21
|
await session.initDrivers();
|
|
@@ -74,14 +74,10 @@ class BackupAction {
|
|
|
74
74
|
options: this.options,
|
|
75
75
|
snapshot,
|
|
76
76
|
targetPath,
|
|
77
|
-
onProgress: async (
|
|
77
|
+
onProgress: async (progress) => {
|
|
78
78
|
await session.progressTask({
|
|
79
79
|
id: taskId,
|
|
80
|
-
|
|
81
|
-
progressPercent: data.percent,
|
|
82
|
-
progressStep: data.step,
|
|
83
|
-
progressStepPercent: data.stepPercent,
|
|
84
|
-
progressTotal: data.total,
|
|
80
|
+
progress,
|
|
85
81
|
});
|
|
86
82
|
},
|
|
87
83
|
});
|
|
@@ -120,14 +116,10 @@ class BackupAction {
|
|
|
120
116
|
(!config.names || config.names.includes(repo.name)))?.config,
|
|
121
117
|
options: this.options,
|
|
122
118
|
snapshot: snapshot,
|
|
123
|
-
onProgress: async (
|
|
119
|
+
onProgress: async (progress) => {
|
|
124
120
|
await session.progressRepository({
|
|
125
121
|
id: repositoryId,
|
|
126
|
-
|
|
127
|
-
progressPercent: data.percent,
|
|
128
|
-
progressStep: data.step,
|
|
129
|
-
progressStepPercent: data.stepPercent,
|
|
130
|
-
progressTotal: data.total,
|
|
122
|
+
progress,
|
|
131
123
|
});
|
|
132
124
|
},
|
|
133
125
|
});
|
|
@@ -164,14 +156,10 @@ class BackupAction {
|
|
|
164
156
|
package: pkg,
|
|
165
157
|
snapshot,
|
|
166
158
|
mirrorRepositoryConfig: mirrorRepo.config,
|
|
167
|
-
onProgress: async (
|
|
159
|
+
onProgress: async (progress) => {
|
|
168
160
|
await session.progressRepository({
|
|
169
161
|
id: repositoryId,
|
|
170
|
-
|
|
171
|
-
progressPercent: data.percent,
|
|
172
|
-
progressStep: data.step,
|
|
173
|
-
progressStepPercent: data.stepPercent,
|
|
174
|
-
progressTotal: data.total,
|
|
162
|
+
progress,
|
|
175
163
|
});
|
|
176
164
|
},
|
|
177
165
|
});
|
package/Action/RestoreAction.js
CHANGED
|
@@ -111,14 +111,10 @@ class RestoreAction {
|
|
|
111
111
|
options: this.options,
|
|
112
112
|
snapshot,
|
|
113
113
|
targetPath,
|
|
114
|
-
onProgress: async (
|
|
114
|
+
onProgress: async (progress) => {
|
|
115
115
|
await session.progressTask({
|
|
116
116
|
id: taskId,
|
|
117
|
-
|
|
118
|
-
progressPercent: data.percent,
|
|
119
|
-
progressStep: data.step,
|
|
120
|
-
progressStepPercent: data.stepPercent,
|
|
121
|
-
progressTotal: data.total,
|
|
117
|
+
progress,
|
|
122
118
|
});
|
|
123
119
|
},
|
|
124
120
|
});
|
|
@@ -160,14 +156,10 @@ class RestoreAction {
|
|
|
160
156
|
(!config.names || config.names.includes(repo.name)))?.config,
|
|
161
157
|
options: this.options,
|
|
162
158
|
snapshot: snapshot,
|
|
163
|
-
onProgress: async (
|
|
159
|
+
onProgress: async (progress) => {
|
|
164
160
|
await session.progressRepository({
|
|
165
161
|
id: repositoryId,
|
|
166
|
-
|
|
167
|
-
progressPercent: data.percent,
|
|
168
|
-
progressStep: data.step,
|
|
169
|
-
progressStepPercent: data.stepPercent,
|
|
170
|
-
progressTotal: data.total,
|
|
162
|
+
progress,
|
|
171
163
|
});
|
|
172
164
|
},
|
|
173
165
|
});
|
package/Command/BackupCommand.js
CHANGED
|
@@ -68,9 +68,10 @@ class BackupCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
68
68
|
new ConsoleSessionDriver_1.ConsoleSessionDriver({
|
|
69
69
|
verbose: verbose > 0,
|
|
70
70
|
progress: this.globalOptions.progress,
|
|
71
|
-
progressInterval: this.globalOptions.progressInterval,
|
|
72
71
|
}),
|
|
73
72
|
],
|
|
73
|
+
verbose: verbose > 1,
|
|
74
|
+
progressInterval: this.globalOptions.progressInterval,
|
|
74
75
|
});
|
|
75
76
|
const result = await backup.exec(sessionManager);
|
|
76
77
|
if (result.errors) {
|
|
@@ -50,6 +50,7 @@ class BackupSessionsCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
50
50
|
verbose: verbose > 1,
|
|
51
51
|
}),
|
|
52
52
|
verbose: verbose > 1,
|
|
53
|
+
progressInterval: this.globalOptions.progressInterval,
|
|
53
54
|
});
|
|
54
55
|
const items = await action.exec(manager);
|
|
55
56
|
const dataFormat = new DataFormat_1.DataFormat({
|
|
@@ -68,10 +68,10 @@ class RestoreCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
68
68
|
new ConsoleSessionDriver_1.ConsoleSessionDriver({
|
|
69
69
|
verbose: verbose > 0,
|
|
70
70
|
progress: this.globalOptions.progress,
|
|
71
|
-
progressInterval: this.globalOptions.progressInterval,
|
|
72
71
|
}),
|
|
73
72
|
],
|
|
74
73
|
verbose: verbose > 1,
|
|
74
|
+
progressInterval: this.globalOptions.progressInterval,
|
|
75
75
|
});
|
|
76
76
|
const result = await restore.exec(sessionManager);
|
|
77
77
|
return result ? 0 : 1;
|
|
@@ -49,6 +49,7 @@ class RestoreSessionsCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
49
49
|
driver: new SqliteSessionDriver_1.SqliteSessionDriver({
|
|
50
50
|
verbose: verbose > 1,
|
|
51
51
|
}),
|
|
52
|
+
progressInterval: this.globalOptions.progressInterval,
|
|
52
53
|
});
|
|
53
54
|
const items = await action.exec(manager);
|
|
54
55
|
const dataFormat = new DataFormat_1.DataFormat({
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
+
import { Progress } from "../util/progress";
|
|
1
2
|
import { CrudEntityAbstract } from "./CrudEntityAbstract";
|
|
2
3
|
export declare abstract class StateEntityAbstract extends CrudEntityAbstract {
|
|
3
4
|
state: "started" | "ended" | null;
|
|
4
5
|
error?: string | null;
|
|
5
6
|
startDate?: string | null;
|
|
6
7
|
endDate?: string | null;
|
|
7
|
-
|
|
8
|
-
progressCurrent?: number | null;
|
|
9
|
-
progressPercent?: number | null;
|
|
10
|
-
progressStep?: string | null;
|
|
11
|
-
progressStepPercent?: number | null;
|
|
8
|
+
progress?: Progress;
|
|
12
9
|
}
|
package/Error/AppError.d.ts
CHANGED
package/Error/AppError.js
CHANGED
|
@@ -19,6 +19,7 @@ export declare type DatatruckRepositoryConfigType = {
|
|
|
19
19
|
};
|
|
20
20
|
declare type CompressObjectType = {
|
|
21
21
|
packs?: {
|
|
22
|
+
name?: string;
|
|
22
23
|
include: string[];
|
|
23
24
|
exclude?: string[];
|
|
24
25
|
onePackByResult?: boolean;
|
|
@@ -48,6 +49,7 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
|
|
|
48
49
|
static stringifyMetaData(data: MetaDataType): string;
|
|
49
50
|
onGetSource(): string;
|
|
50
51
|
onInit(data: InitDataType): Promise<void>;
|
|
52
|
+
private createFileScanner;
|
|
51
53
|
onPrune(data: PruneDataType): Promise<void>;
|
|
52
54
|
onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
|
|
53
55
|
private normalizeCompressConfig;
|
|
@@ -19,6 +19,7 @@ const fs_1 = require("fs");
|
|
|
19
19
|
const promises_1 = require("fs/promises");
|
|
20
20
|
const micromatch_1 = require("micromatch");
|
|
21
21
|
const path_1 = require("path");
|
|
22
|
+
const perf_hooks_1 = require("perf_hooks");
|
|
22
23
|
const readline_1 = require("readline");
|
|
23
24
|
exports.datatruckRepositoryName = "datatruck";
|
|
24
25
|
exports.datatruckRepositoryDefinition = {
|
|
@@ -50,6 +51,7 @@ exports.datatruckPackageRepositoryDefinition = {
|
|
|
50
51
|
additionalProperties: false,
|
|
51
52
|
required: ["include"],
|
|
52
53
|
properties: {
|
|
54
|
+
name: { type: "string" },
|
|
53
55
|
include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
|
|
54
56
|
exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
|
|
55
57
|
onePackByResult: { type: "boolean" },
|
|
@@ -74,8 +76,6 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
74
76
|
return `${date}_${pkgName}_${snapshotShortId}`;
|
|
75
77
|
}
|
|
76
78
|
static parseSnapshotName(name) {
|
|
77
|
-
if (!name.endsWith(".json"))
|
|
78
|
-
return null;
|
|
79
79
|
name = name.replace(/\.json$/, "");
|
|
80
80
|
const nameParts = name.split("_");
|
|
81
81
|
if (nameParts.length !== 3)
|
|
@@ -102,6 +102,67 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
102
102
|
async onInit(data) {
|
|
103
103
|
await (0, fs_util_1.mkdirIfNotExists)(this.config.outPath);
|
|
104
104
|
}
|
|
105
|
+
async createFileScanner(options) {
|
|
106
|
+
const object = {
|
|
107
|
+
total: 0,
|
|
108
|
+
current: 0,
|
|
109
|
+
progress: async (description, data) => {
|
|
110
|
+
await options.onProgress({
|
|
111
|
+
relative: {
|
|
112
|
+
description,
|
|
113
|
+
payload: data.path,
|
|
114
|
+
percent: data.percent,
|
|
115
|
+
},
|
|
116
|
+
absolute: {
|
|
117
|
+
total: object.total,
|
|
118
|
+
current: object.current + data.current,
|
|
119
|
+
percent: (0, math_util_1.progressPercent)(object.total, object.current + data.current),
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
if (data.type === "end") {
|
|
123
|
+
object.current += data.current;
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
start: async (cb) => {
|
|
127
|
+
for await (const entry of (0, fs_util_1.pathIterator)(stream)) {
|
|
128
|
+
if (!options.disableCounting)
|
|
129
|
+
object.total++;
|
|
130
|
+
const currentTime = perf_hooks_1.performance.now();
|
|
131
|
+
const diff = currentTime - lastTime;
|
|
132
|
+
if (diff > 1000) {
|
|
133
|
+
await options.onProgress({
|
|
134
|
+
relative: {
|
|
135
|
+
description: "Scanning files",
|
|
136
|
+
payload: object.total.toString(),
|
|
137
|
+
},
|
|
138
|
+
});
|
|
139
|
+
lastTime = currentTime;
|
|
140
|
+
}
|
|
141
|
+
if (cb)
|
|
142
|
+
await cb(entry);
|
|
143
|
+
}
|
|
144
|
+
await options.onProgress({
|
|
145
|
+
relative: {
|
|
146
|
+
description: "Scanned files",
|
|
147
|
+
payload: object.total.toString(),
|
|
148
|
+
},
|
|
149
|
+
});
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
await options.onProgress({
|
|
153
|
+
relative: {
|
|
154
|
+
description: "Scanning files",
|
|
155
|
+
},
|
|
156
|
+
});
|
|
157
|
+
const stream = fast_glob_1.default.stream(options.glob.include, {
|
|
158
|
+
dot: true,
|
|
159
|
+
markDirectories: true,
|
|
160
|
+
stats: true,
|
|
161
|
+
...options.glob,
|
|
162
|
+
});
|
|
163
|
+
let lastTime = perf_hooks_1.performance.now();
|
|
164
|
+
return object;
|
|
165
|
+
}
|
|
105
166
|
async onPrune(data) {
|
|
106
167
|
const snapshotName = DatatruckRepository.buildSnapshotName({
|
|
107
168
|
snapshotId: data.snapshot.id,
|
|
@@ -109,15 +170,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
109
170
|
packageName: data.snapshot.packageName,
|
|
110
171
|
});
|
|
111
172
|
const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
112
|
-
const metaPath = `${snapshotPath}.json`;
|
|
113
173
|
if (data.options.verbose)
|
|
114
174
|
(0, cli_util_1.logExec)(`Deleting ${snapshotPath}`);
|
|
115
175
|
if (await (0, fs_util_1.checkDir)(snapshotPath))
|
|
116
176
|
await (0, promises_1.rm)(snapshotPath, {
|
|
117
177
|
recursive: true,
|
|
118
178
|
});
|
|
119
|
-
if (await (0, fs_util_1.checkFile)(metaPath))
|
|
120
|
-
await (0, promises_1.rm)(metaPath);
|
|
121
179
|
}
|
|
122
180
|
async onSnapshots(data) {
|
|
123
181
|
if (!(await (0, fs_util_1.checkDir)(this.config.outPath)))
|
|
@@ -136,7 +194,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
136
194
|
if (data.options.ids &&
|
|
137
195
|
!data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
|
|
138
196
|
continue;
|
|
139
|
-
const metaPath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
197
|
+
const metaPath = (0, path_1.join)(this.config.outPath, snapshotName, "meta.json");
|
|
140
198
|
const meta = await DatatruckRepository.parseMetaData(metaPath);
|
|
141
199
|
if (taskPatterns && !(0, string_util_1.checkMatch)(meta.task, taskPatterns))
|
|
142
200
|
continue;
|
|
@@ -160,7 +218,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
160
218
|
}
|
|
161
219
|
normalizeCompressConfig(packageConfig) {
|
|
162
220
|
let compress = packageConfig?.compress ?? this.config.compress;
|
|
163
|
-
if (compress === true) {
|
|
221
|
+
if (compress === true || (compress && !Array.isArray(compress.packs))) {
|
|
164
222
|
return {
|
|
165
223
|
packs: [
|
|
166
224
|
{
|
|
@@ -198,96 +256,124 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
198
256
|
verbose: data.options.verbose,
|
|
199
257
|
})
|
|
200
258
|
: undefined;
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
});
|
|
259
|
+
const packs = compress?.packs || [];
|
|
260
|
+
const tmpDir = await (0, fs_util_1.mkTmpDir)("path-lists");
|
|
261
|
+
const unpackedStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "unpacked.txt"));
|
|
262
|
+
const singlePackStream = (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, "single-pack.txt"));
|
|
263
|
+
const packStreams = Array.from({ length: packs.length }).map((v, i) => (0, fs_1.createWriteStream)((0, path_1.join)(tmpDir, `pack-${i}.txt`)));
|
|
264
|
+
const streams = [unpackedStream, singlePackStream, ...packStreams];
|
|
208
265
|
if (data.options.verbose)
|
|
209
|
-
(0, cli_util_1.logExec)(`Writing
|
|
210
|
-
const
|
|
211
|
-
|
|
212
|
-
|
|
266
|
+
(0, cli_util_1.logExec)(`Writing file lists in ${tmpDir}`);
|
|
267
|
+
const scanner = await this.createFileScanner({
|
|
268
|
+
glob: {
|
|
269
|
+
include,
|
|
270
|
+
cwd: sourcePath,
|
|
271
|
+
ignore: exclude,
|
|
272
|
+
onlyFiles: false,
|
|
273
|
+
},
|
|
274
|
+
onProgress: data.onProgress,
|
|
275
|
+
disableCounting: true,
|
|
213
276
|
});
|
|
214
|
-
|
|
215
|
-
(0,
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
path: pkg.path,
|
|
232
|
-
output: target,
|
|
233
|
-
filter: [{ patterns: [packPath] }],
|
|
234
|
-
excludeList: pathLists.excludedPackPaths[packIndex],
|
|
235
|
-
verbose: data.options.verbose,
|
|
236
|
-
onStream: async (stream) => await data.onProgress({
|
|
237
|
-
total: pathLists.total.all,
|
|
238
|
-
current: currentFiles + stream.data.files,
|
|
239
|
-
percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
|
|
240
|
-
step: stream.type === "progress" ? stream.data.path : "",
|
|
241
|
-
stepPercent: stream.type === "progress" ? stream.data.progress : null,
|
|
242
|
-
}),
|
|
243
|
-
});
|
|
244
|
-
currentFiles += stats.files;
|
|
277
|
+
await Promise.all([
|
|
278
|
+
...streams.map((p) => (0, fs_util_1.waitForClose)(p)),
|
|
279
|
+
(async () => {
|
|
280
|
+
await scanner.start(async (entry) => {
|
|
281
|
+
const pathSubject = entry.stats.isDirectory()
|
|
282
|
+
? entry.path.slice(0, -1)
|
|
283
|
+
: entry.path;
|
|
284
|
+
let stream = unpackedStream;
|
|
285
|
+
let successPackIndex;
|
|
286
|
+
for (const [packIndex, pack] of packs.entries()) {
|
|
287
|
+
if ((0, string_util_1.checkPath)(pathSubject, pack.include, pack.exclude)) {
|
|
288
|
+
stream = pack.onePackByResult
|
|
289
|
+
? singlePackStream
|
|
290
|
+
: packStreams[packIndex];
|
|
291
|
+
successPackIndex = packIndex;
|
|
292
|
+
break;
|
|
293
|
+
}
|
|
245
294
|
}
|
|
295
|
+
const isUnpackedStream = stream === unpackedStream;
|
|
296
|
+
const isPackStream = stream !== unpackedStream;
|
|
297
|
+
const isSinglePackStream = stream === singlePackStream;
|
|
298
|
+
const include = isPackStream
|
|
299
|
+
? entry.stats.isDirectory()
|
|
300
|
+
? await (0, fs_util_1.isEmptyDir)((0, path_1.join)(sourcePath, entry.path))
|
|
301
|
+
: true
|
|
302
|
+
: true;
|
|
303
|
+
if (include) {
|
|
304
|
+
let value = entry.path;
|
|
305
|
+
if (isUnpackedStream) {
|
|
306
|
+
value += `:${entry.stats.uid}:${entry.stats.gid}:${entry.stats.mode}`;
|
|
307
|
+
}
|
|
308
|
+
else if (isSinglePackStream) {
|
|
309
|
+
value += `:${successPackIndex}`;
|
|
310
|
+
}
|
|
311
|
+
if (!entry.stats.isDirectory())
|
|
312
|
+
scanner.total++;
|
|
313
|
+
stream.write(`${value}\n`);
|
|
314
|
+
}
|
|
315
|
+
});
|
|
316
|
+
for (const stream of streams) {
|
|
317
|
+
stream.end();
|
|
246
318
|
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
excludeList: pathLists.excludedPackPaths[packIndex],
|
|
254
|
-
verbose: data.options.verbose,
|
|
255
|
-
onStream: async (stream) => await data.onProgress({
|
|
256
|
-
total: pathLists.total.all,
|
|
257
|
-
current: currentFiles + stream.data.files,
|
|
258
|
-
percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
|
|
259
|
-
step: stream.type === "progress" ? stream.data.path : "",
|
|
260
|
-
}),
|
|
261
|
-
});
|
|
262
|
-
currentFiles += stats.files;
|
|
263
|
-
}
|
|
264
|
-
packIndex++;
|
|
265
|
-
}
|
|
266
|
-
}
|
|
319
|
+
})(),
|
|
320
|
+
]);
|
|
321
|
+
const unpackedPath = (0, path_1.join)(outPath, "unpacked");
|
|
322
|
+
await (0, promises_1.mkdir)(unpackedPath);
|
|
323
|
+
await (0, promises_1.copyFile)(unpackedStream.path, (0, path_1.join)(outPath, "permissions.txt"));
|
|
324
|
+
// Non pack
|
|
267
325
|
if (data.options.verbose)
|
|
268
|
-
(0, cli_util_1.logExec)(`Copying files to ${
|
|
326
|
+
(0, cli_util_1.logExec)(`Copying files from ${unpackedStream.path.toString()} to ${unpackedPath}`);
|
|
269
327
|
await (0, fs_util_1.cpy)({
|
|
270
328
|
input: {
|
|
271
329
|
type: "pathList",
|
|
272
|
-
path:
|
|
330
|
+
path: unpackedStream.path.toString(),
|
|
273
331
|
basePath: sourcePath,
|
|
274
332
|
},
|
|
275
|
-
targetPath:
|
|
333
|
+
targetPath: unpackedPath,
|
|
276
334
|
skipNotFoundError: true,
|
|
277
335
|
concurrency: this.config.fileCopyConcurrency,
|
|
278
|
-
async
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
total: pathLists.total.all,
|
|
284
|
-
current: currentFiles,
|
|
285
|
-
percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles),
|
|
286
|
-
step: entryPath,
|
|
287
|
-
});
|
|
288
|
-
},
|
|
336
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
|
|
337
|
+
});
|
|
338
|
+
// Single pack
|
|
339
|
+
const singleReader = (0, readline_1.createInterface)({
|
|
340
|
+
input: (0, fs_1.createReadStream)(singlePackStream.path),
|
|
289
341
|
});
|
|
290
|
-
const
|
|
342
|
+
for await (const line of singleReader) {
|
|
343
|
+
let [packPath, packIndex] = line.split(":");
|
|
344
|
+
const pack = packs[packIndex];
|
|
345
|
+
if (packPath.endsWith("/"))
|
|
346
|
+
packPath = packPath.slice(0, -1);
|
|
347
|
+
const outBasename = (`pack${pack.name ? `-${encodeURIComponent(pack.name)}` : ""}` +
|
|
348
|
+
`-${encodeURIComponent(packPath.replace(/[\\/]/g, "-"))}` +
|
|
349
|
+
`.zip`).slice(0, 255);
|
|
350
|
+
const target = (0, path_1.join)(outPath, outBasename);
|
|
351
|
+
await (0, zip_util_1.zip)({
|
|
352
|
+
path: pkg.path,
|
|
353
|
+
output: target,
|
|
354
|
+
filter: [{ patterns: [packPath] }],
|
|
355
|
+
verbose: data.options.verbose,
|
|
356
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "start"
|
|
357
|
+
? "Starting compressing"
|
|
358
|
+
: "Compressing file", progress),
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
// Packs
|
|
362
|
+
for (const [packIndex, packStream] of packStreams.entries()) {
|
|
363
|
+
const pack = packs[packIndex];
|
|
364
|
+
const target = (0, path_1.join)(outPath, `pack-${packIndex}${pack.name ? `-${pack.name}` : ""}.zip`);
|
|
365
|
+
await (0, zip_util_1.zip)({
|
|
366
|
+
path: sourcePath,
|
|
367
|
+
output: target,
|
|
368
|
+
includeList: packStream.path.toString(),
|
|
369
|
+
verbose: data.options.verbose,
|
|
370
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "start"
|
|
371
|
+
? "Starting compressing"
|
|
372
|
+
: "Compressing file", progress),
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
// Meta
|
|
376
|
+
const metaPath = `${outPath}/meta.json`;
|
|
291
377
|
const nodePkg = (0, fs_util_1.parsePackageFile)();
|
|
292
378
|
const meta = {
|
|
293
379
|
id: data.snapshot.id,
|
|
@@ -310,19 +396,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
310
396
|
});
|
|
311
397
|
const sourcePath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
|
|
312
398
|
const targetPath = (0, path_1.resolve)((0, path_1.join)(data.mirrorRepositoryConfig.outPath, snapshotName));
|
|
313
|
-
const sourceMetaPath = `${sourcePath}.json`;
|
|
314
|
-
const targetMetaPath = `${targetPath}.json`;
|
|
315
399
|
if (data.options.verbose)
|
|
316
|
-
(0, cli_util_1.logExec)(`Copying files to ${targetPath}`);
|
|
400
|
+
(0, cli_util_1.logExec)(`Copying backup files to ${targetPath}`);
|
|
317
401
|
await (0, promises_1.mkdir)(targetPath);
|
|
402
|
+
const scanner = await this.createFileScanner({
|
|
403
|
+
glob: {
|
|
404
|
+
include: ["**/*"],
|
|
405
|
+
cwd: sourcePath,
|
|
406
|
+
},
|
|
407
|
+
onProgress: data.onProgress,
|
|
408
|
+
});
|
|
409
|
+
await scanner.start();
|
|
318
410
|
await (0, fs_util_1.cpy)({
|
|
319
411
|
input: {
|
|
320
412
|
type: "glob",
|
|
321
413
|
sourcePath,
|
|
322
414
|
},
|
|
323
415
|
targetPath,
|
|
416
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
|
|
324
417
|
});
|
|
325
|
-
await (0, promises_1.copyFile)(sourceMetaPath, targetMetaPath);
|
|
326
418
|
}
|
|
327
419
|
async onRestore(data) {
|
|
328
420
|
const relRestorePath = data.targetPath ?? data.package.restorePath;
|
|
@@ -341,50 +433,69 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
341
433
|
packageName: data.package.name,
|
|
342
434
|
});
|
|
343
435
|
const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
436
|
+
const scanner = await this.createFileScanner({
|
|
437
|
+
glob: {
|
|
438
|
+
include: ["unpacked/**/*"],
|
|
439
|
+
cwd: sourcePath,
|
|
440
|
+
},
|
|
441
|
+
onProgress: data.onProgress,
|
|
442
|
+
});
|
|
443
|
+
await scanner.start();
|
|
444
|
+
const it = await (0, promises_1.opendir)(sourcePath);
|
|
445
|
+
for await (const dirent of it) {
|
|
446
|
+
const path = (0, path_1.join)(sourcePath, dirent.name);
|
|
447
|
+
if (dirent.name === "permissions.txt") {
|
|
448
|
+
scanner.total++;
|
|
449
|
+
}
|
|
450
|
+
else if (dirent.name.endsWith(".zip")) {
|
|
451
|
+
await (0, zip_util_1.listZip)({
|
|
452
|
+
path,
|
|
453
|
+
verbose: data.options.verbose,
|
|
454
|
+
onStream: async (item) => {
|
|
455
|
+
const isDir = item.Folder === "+";
|
|
456
|
+
if (!isDir)
|
|
457
|
+
scanner.total++;
|
|
458
|
+
},
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
}
|
|
349
462
|
if (data.options.verbose)
|
|
350
463
|
(0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
|
|
351
464
|
await (0, fs_util_1.cpy)({
|
|
352
465
|
input: {
|
|
353
466
|
type: "glob",
|
|
354
|
-
sourcePath,
|
|
467
|
+
sourcePath: (0, path_1.join)(sourcePath, "unpacked"),
|
|
355
468
|
},
|
|
356
469
|
targetPath: restorePath,
|
|
357
470
|
concurrency: this.config.fileCopyConcurrency,
|
|
358
|
-
|
|
359
|
-
const isRootFile = (0, path_1.basename)(entryPath) === entryPath;
|
|
360
|
-
const isZipFile = isRootFile &&
|
|
361
|
-
entryPath.startsWith(".") &&
|
|
362
|
-
entryPath.endsWith(".dd.zip");
|
|
363
|
-
await data.onProgress({
|
|
364
|
-
total: totalFiles,
|
|
365
|
-
current: Math.max(currentFiles, 0),
|
|
366
|
-
percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
|
|
367
|
-
step: entryPath,
|
|
368
|
-
});
|
|
369
|
-
if (isZipFile) {
|
|
370
|
-
await (0, zip_util_1.unzip)({
|
|
371
|
-
input: entrySourcePath,
|
|
372
|
-
output: restorePath,
|
|
373
|
-
verbose: data.options.verbose,
|
|
374
|
-
onStream: async (stream) => await data.onProgress({
|
|
375
|
-
total: totalFiles,
|
|
376
|
-
current: currentFiles + 1,
|
|
377
|
-
percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + 1),
|
|
378
|
-
step: stream.type === "progress"
|
|
379
|
-
? `Extracting ${stream.data.path}`
|
|
380
|
-
: "",
|
|
381
|
-
}),
|
|
382
|
-
});
|
|
383
|
-
}
|
|
384
|
-
currentFiles++;
|
|
385
|
-
return isZipFile ? false : true;
|
|
386
|
-
},
|
|
471
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "end" ? "Files copied" : "Copying file", progress),
|
|
387
472
|
});
|
|
473
|
+
const it2 = await (0, promises_1.opendir)(sourcePath);
|
|
474
|
+
for await (const dirent of it2) {
|
|
475
|
+
const path = (0, path_1.join)(sourcePath, dirent.name);
|
|
476
|
+
if (dirent.name === "permissions.txt") {
|
|
477
|
+
if (data.options.verbose)
|
|
478
|
+
(0, cli_util_1.logExec)(`Applying permissions (${path})`);
|
|
479
|
+
await scanner.progress("Applying permissions", {
|
|
480
|
+
current: 0,
|
|
481
|
+
});
|
|
482
|
+
await (0, fs_util_1.applyPermissions)(restorePath, path);
|
|
483
|
+
await scanner.progress("Permissions applied", {
|
|
484
|
+
current: 1,
|
|
485
|
+
type: "end",
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
else if (dirent.name.endsWith(".zip")) {
|
|
489
|
+
await (0, zip_util_1.unzip)({
|
|
490
|
+
input: path,
|
|
491
|
+
output: restorePath,
|
|
492
|
+
verbose: data.options.verbose,
|
|
493
|
+
onProgress: async (progress) => await scanner.progress(progress.type === "start"
|
|
494
|
+
? "Starting extracting"
|
|
495
|
+
: "Extracting file", progress),
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
}
|
|
388
499
|
}
|
|
389
500
|
}
|
|
390
501
|
exports.DatatruckRepository = DatatruckRepository;
|