@datatruck/cli 0.27.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/BackupAction.d.ts +75 -34
- package/Action/BackupAction.js +302 -246
- package/Action/CleanCacheAction.d.ts +8 -4
- package/Action/CleanCacheAction.js +8 -5
- package/Action/ConfigAction.d.ts +12 -5
- package/Action/ConfigAction.js +14 -18
- package/Action/CopyAction.d.ts +51 -0
- package/Action/CopyAction.js +165 -0
- package/Action/InitAction.d.ts +3 -3
- package/Action/InitAction.js +9 -9
- package/Action/PruneAction.d.ts +10 -16
- package/Action/PruneAction.js +37 -34
- package/Action/RestoreAction.d.ts +49 -24
- package/Action/RestoreAction.js +164 -195
- package/Action/SnapshotsAction.d.ts +8 -8
- package/Action/SnapshotsAction.js +9 -9
- package/CHANGELOG.md +513 -0
- package/Command/BackupCommand.d.ts +7 -4
- package/Command/BackupCommand.js +14 -26
- package/Command/CleanCacheCommand.d.ts +4 -4
- package/Command/CleanCacheCommand.js +26 -5
- package/Command/CommandAbstract.d.ts +10 -7
- package/Command/CommandAbstract.js +4 -1
- package/Command/ConfigCommand.d.ts +6 -9
- package/Command/ConfigCommand.js +13 -8
- package/Command/CopyCommand.d.ts +16 -0
- package/Command/CopyCommand.js +66 -0
- package/Command/InitCommand.d.ts +4 -4
- package/Command/InitCommand.js +13 -17
- package/Command/PruneCommand.d.ts +4 -10
- package/Command/PruneCommand.js +13 -12
- package/Command/RestoreCommand.d.ts +1 -1
- package/Command/RestoreCommand.js +13 -21
- package/Command/SnapshotsCommand.d.ts +4 -4
- package/Command/SnapshotsCommand.js +16 -15
- package/Command/StartServerCommand.d.ts +3 -3
- package/Config/Config.d.ts +11 -0
- package/Config/Config.js +18 -0
- package/Config/PrunePolicyConfig.d.ts +2 -2
- package/Factory/CommandFactory.d.ts +27 -34
- package/Factory/CommandFactory.js +27 -54
- package/Factory/RepositoryFactory.d.ts +1 -1
- package/Factory/RepositoryFactory.js +3 -3
- package/Factory/TaskFactory.d.ts +1 -1
- package/Factory/TaskFactory.js +3 -3
- package/Repository/DatatruckRepository.d.ts +10 -8
- package/Repository/DatatruckRepository.js +47 -25
- package/Repository/GitRepository.d.ts +9 -8
- package/Repository/GitRepository.js +22 -25
- package/Repository/RepositoryAbstract.d.ts +39 -37
- package/Repository/RepositoryAbstract.js +4 -5
- package/Repository/ResticRepository.d.ts +9 -8
- package/Repository/ResticRepository.js +30 -28
- package/Task/GitTask.d.ts +6 -7
- package/Task/GitTask.js +24 -30
- package/Task/MariadbTask.d.ts +4 -5
- package/Task/MariadbTask.js +26 -32
- package/Task/MssqlTask.d.ts +5 -3
- package/Task/MssqlTask.js +11 -12
- package/Task/MysqlDumpTask.d.ts +10 -3
- package/Task/MysqlDumpTask.js +102 -32
- package/Task/ScriptTask.d.ts +23 -18
- package/Task/ScriptTask.js +34 -24
- package/Task/SqlDumpTaskAbstract.d.ts +8 -3
- package/Task/SqlDumpTaskAbstract.js +32 -20
- package/Task/TaskAbstract.d.ts +24 -25
- package/Task/TaskAbstract.js +6 -10
- package/cli.js +13 -5
- package/config.schema.json +89 -1
- package/package.json +4 -5
- package/utils/DataFormat.d.ts +23 -12
- package/utils/DataFormat.js +36 -14
- package/utils/cli.d.ts +3 -9
- package/utils/cli.js +19 -55
- package/utils/crypto.d.ts +1 -0
- package/utils/crypto.js +15 -0
- package/utils/datatruck/client.d.ts +2 -0
- package/utils/datatruck/client.js +3 -0
- package/utils/datatruck/config.d.ts +2 -0
- package/utils/datatruck/config.js +18 -3
- package/utils/datatruck/paths.d.ts +5 -9
- package/utils/datatruck/paths.js +2 -2
- package/utils/datatruck/snapshot.d.ts +5 -2
- package/utils/datatruck/snapshot.js +12 -22
- package/utils/date.d.ts +21 -4
- package/utils/date.js +46 -24
- package/utils/fs.d.ts +16 -11
- package/utils/fs.js +81 -48
- package/utils/list.d.ts +64 -0
- package/utils/list.js +145 -0
- package/utils/mysql.d.ts +2 -0
- package/utils/mysql.js +21 -2
- package/utils/process.d.ts +1 -0
- package/utils/process.js +24 -31
- package/utils/progress.d.ts +33 -0
- package/utils/progress.js +113 -0
- package/utils/steps.d.ts +11 -0
- package/utils/steps.js +22 -10
- package/utils/stream.d.ts +7 -0
- package/utils/stream.js +10 -0
- package/utils/string.d.ts +0 -1
- package/utils/string.js +1 -13
- package/utils/tar.d.ts +10 -3
- package/utils/tar.js +73 -45
- package/utils/temp.d.ts +26 -0
- package/utils/temp.js +133 -0
- package/utils/virtual-fs.d.ts +6 -2
- package/utils/virtual-fs.js +6 -0
- package/Action/BackupSessionsAction.d.ts +0 -13
- package/Action/BackupSessionsAction.js +0 -18
- package/Action/RestoreSessionsAction.d.ts +0 -13
- package/Action/RestoreSessionsAction.js +0 -18
- package/Command/BackupSessionsCommand.d.ts +0 -12
- package/Command/BackupSessionsCommand.js +0 -92
- package/Command/RestoreSessionsCommand.d.ts +0 -12
- package/Command/RestoreSessionsCommand.js +0 -91
- package/Decorator/EntityDecorator.d.ts +0 -11
- package/Decorator/EntityDecorator.js +0 -17
- package/Entity/BackupSessionEntity.d.ts +0 -6
- package/Entity/BackupSessionEntity.js +0 -25
- package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
- package/Entity/BackupSessionRepositoryEntity.js +0 -25
- package/Entity/BackupSessionTaskEntity.d.ts +0 -5
- package/Entity/BackupSessionTaskEntity.js +0 -24
- package/Entity/CrudEntityAbstract.d.ts +0 -5
- package/Entity/CrudEntityAbstract.js +0 -9
- package/Entity/RestoreSessionEntity.d.ts +0 -5
- package/Entity/RestoreSessionEntity.js +0 -24
- package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
- package/Entity/RestoreSessionRepositoryEntity.js +0 -25
- package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
- package/Entity/RestoreSessionTaskEntity.js +0 -24
- package/Entity/StateEntityAbstract.d.ts +0 -9
- package/Entity/StateEntityAbstract.js +0 -12
- package/Factory/EntityFactory.d.ts +0 -6
- package/Factory/EntityFactory.js +0 -40
- package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
- package/SessionDriver/ConsoleSessionDriver.js +0 -208
- package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
- package/SessionDriver/SessionDriverAbstract.js +0 -28
- package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
- package/SessionDriver/SqliteSessionDriver.js +0 -173
- package/SessionManager/BackupSessionManager.d.ts +0 -45
- package/SessionManager/BackupSessionManager.js +0 -218
- package/SessionManager/RestoreSessionManager.d.ts +0 -47
- package/SessionManager/RestoreSessionManager.js +0 -218
- package/SessionManager/SessionManagerAbstract.d.ts +0 -18
- package/SessionManager/SessionManagerAbstract.js +0 -36
- package/migrations/001-initial.sql +0 -98
- package/utils/entity.d.ts +0 -4
- package/utils/entity.js +0 -10
|
@@ -4,14 +4,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.GitRepository = exports.gitPackageRepositoryDefinition = exports.gitRepositoryDefinition = exports.gitRepositoryName = void 0;
|
|
7
|
-
const AppError_1 = require("../Error/AppError");
|
|
8
7
|
const Git_1 = require("../utils/Git");
|
|
9
8
|
const cli_1 = require("../utils/cli");
|
|
10
9
|
const paths_1 = require("../utils/datatruck/paths");
|
|
11
10
|
const fs_1 = require("../utils/fs");
|
|
12
11
|
const string_1 = require("../utils/string");
|
|
12
|
+
const temp_1 = require("../utils/temp");
|
|
13
13
|
const RepositoryAbstract_1 = require("./RepositoryAbstract");
|
|
14
|
-
const assert_1 = require("assert");
|
|
15
14
|
const fast_glob_1 = __importDefault(require("fast-glob"));
|
|
16
15
|
const promises_1 = require("fs/promises");
|
|
17
16
|
const micromatch_1 = require("micromatch");
|
|
@@ -33,9 +32,13 @@ exports.gitPackageRepositoryDefinition = {
|
|
|
33
32
|
};
|
|
34
33
|
class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
35
34
|
static refPrefix = "dt";
|
|
36
|
-
|
|
35
|
+
getSource() {
|
|
37
36
|
return this.config.repo;
|
|
38
37
|
}
|
|
38
|
+
async fetchDiskStats(config) {
|
|
39
|
+
if ((0, fs_1.isLocalDir)(config.repo))
|
|
40
|
+
return await (0, fs_1.fetchDiskStats)(config.repo);
|
|
41
|
+
}
|
|
39
42
|
static buildSnapshotTagName(tag) {
|
|
40
43
|
return `${GitRepository.refPrefix}/${tag.package}/${tag.id}`;
|
|
41
44
|
}
|
|
@@ -56,9 +59,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
56
59
|
static buildBranchName(packageName) {
|
|
57
60
|
return `${GitRepository.refPrefix}/${packageName}`;
|
|
58
61
|
}
|
|
59
|
-
async
|
|
62
|
+
async init(data) {
|
|
60
63
|
const git = new Git_1.Git({
|
|
61
|
-
dir: (0,
|
|
64
|
+
dir: (0, temp_1.tmpDir)(exports.gitRepositoryName, "repository", "init"),
|
|
62
65
|
log: data.options.verbose,
|
|
63
66
|
});
|
|
64
67
|
if (await git.canBeInit(this.config.repo)) {
|
|
@@ -83,9 +86,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
83
86
|
await git.push({ branchName });
|
|
84
87
|
}
|
|
85
88
|
}
|
|
86
|
-
async
|
|
89
|
+
async prune(data) {
|
|
87
90
|
const git = new Git_1.Git({
|
|
88
|
-
dir: await
|
|
91
|
+
dir: await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "prune"),
|
|
89
92
|
log: data.options.verbose,
|
|
90
93
|
});
|
|
91
94
|
const branchName = GitRepository.buildBranchName(data.snapshot.packageName);
|
|
@@ -106,9 +109,9 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
106
109
|
await git.exec(["push", "origin", branchName, "--force-with-lease"]);
|
|
107
110
|
await git.exec(["push", "--delete", "origin", data.snapshot.originalId]);
|
|
108
111
|
}
|
|
109
|
-
async
|
|
112
|
+
async fetchSnapshots(data) {
|
|
110
113
|
const git = new Git_1.Git({
|
|
111
|
-
dir: await
|
|
114
|
+
dir: await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "snapshots"),
|
|
112
115
|
log: data.options.verbose,
|
|
113
116
|
});
|
|
114
117
|
const pkgPatterns = (0, string_1.makePathPatterns)(data.options.packageNames);
|
|
@@ -143,13 +146,10 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
143
146
|
}, [])
|
|
144
147
|
.sort((a, b) => a.date.localeCompare(b.date));
|
|
145
148
|
}
|
|
146
|
-
async
|
|
149
|
+
async backup(data) {
|
|
147
150
|
const pkg = data.package;
|
|
148
|
-
const
|
|
149
|
-
(0,
|
|
150
|
-
if (!(await (0, fs_1.existsDir)(sourcePath)))
|
|
151
|
-
throw new AppError_1.AppError(`Package path not exists: ${sourcePath}`);
|
|
152
|
-
const tmpPath = await this.mkTmpDir(GitRepository.name + "-backup");
|
|
151
|
+
const path = pkg.path;
|
|
152
|
+
const tmpPath = await (0, temp_1.mkTmpDir)(exports.gitRepositoryName, "repo", "backup");
|
|
153
153
|
const branchName = GitRepository.buildBranchName(data.package.name);
|
|
154
154
|
const git = new Git_1.Git({
|
|
155
155
|
dir: tmpPath,
|
|
@@ -169,7 +169,7 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
169
169
|
const backupPathsOptions = {
|
|
170
170
|
package: data.package,
|
|
171
171
|
snapshot: data.snapshot,
|
|
172
|
-
|
|
172
|
+
path: path,
|
|
173
173
|
verbose: data.options.verbose,
|
|
174
174
|
};
|
|
175
175
|
const include = await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions);
|
|
@@ -177,13 +177,13 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
177
177
|
? await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions)
|
|
178
178
|
: undefined;
|
|
179
179
|
const stream = await (0, fast_glob_1.default)(include, {
|
|
180
|
-
cwd:
|
|
180
|
+
cwd: path,
|
|
181
181
|
ignore: exclude,
|
|
182
182
|
dot: true,
|
|
183
183
|
});
|
|
184
184
|
let files = 0;
|
|
185
185
|
for await (const entry of stream) {
|
|
186
|
-
const source = (0, path_1.join)(
|
|
186
|
+
const source = (0, path_1.join)(path, entry);
|
|
187
187
|
const target = (0, path_1.join)(tmpPath, entry);
|
|
188
188
|
const dir = (0, path_1.dirname)(target);
|
|
189
189
|
if (!createdPaths.includes(dir)) {
|
|
@@ -215,16 +215,13 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
215
215
|
await git.addTag(meta.name, meta.message);
|
|
216
216
|
await git.push({ branchName });
|
|
217
217
|
await git.pushTags();
|
|
218
|
-
await (0, promises_1.rm)(tmpPath, {
|
|
219
|
-
recursive: true,
|
|
220
|
-
});
|
|
218
|
+
await (0, promises_1.rm)(tmpPath, { recursive: true });
|
|
221
219
|
}
|
|
222
|
-
|
|
220
|
+
copy(data) {
|
|
223
221
|
throw new Error("Method not implemented.");
|
|
224
222
|
}
|
|
225
|
-
async
|
|
226
|
-
const restorePath = data.
|
|
227
|
-
(0, assert_1.ok)(restorePath);
|
|
223
|
+
async restore(data) {
|
|
224
|
+
const restorePath = data.snapshotPath;
|
|
228
225
|
const tagName = GitRepository.buildSnapshotTagName({
|
|
229
226
|
id: data.snapshot.id,
|
|
230
227
|
package: data.package.name,
|
|
@@ -1,52 +1,54 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
3
|
-
import type {
|
|
4
|
-
import type {
|
|
1
|
+
import type { BackupActionOptions } from "../Action/BackupAction";
|
|
2
|
+
import type { InitActionOptions } from "../Action/InitAction";
|
|
3
|
+
import type { RestoreActionOptions } from "../Action/RestoreAction";
|
|
4
|
+
import type { ExtendedSnapshot, SnapshotsActionOptions } from "../Action/SnapshotsAction";
|
|
5
5
|
import type { PackageConfigType } from "../Config/PackageConfig";
|
|
6
6
|
import type { RepositoryConfigType } from "../Config/RepositoryConfig";
|
|
7
|
-
import {
|
|
8
|
-
|
|
7
|
+
import { type DiskStats } from "../utils/fs";
|
|
8
|
+
import type { Progress } from "../utils/progress";
|
|
9
|
+
export type PreSnapshot = {
|
|
9
10
|
id: string;
|
|
10
11
|
date: string;
|
|
11
12
|
};
|
|
12
|
-
export type
|
|
13
|
+
export type Snapshot = PreSnapshot & {
|
|
13
14
|
originalId: string;
|
|
14
15
|
packageName: string;
|
|
15
16
|
packageTaskName: string | undefined;
|
|
16
17
|
tags: string[];
|
|
17
18
|
size: number;
|
|
18
19
|
};
|
|
19
|
-
export type
|
|
20
|
-
options:
|
|
20
|
+
export type RepoInitData = {
|
|
21
|
+
options: InitActionOptions;
|
|
21
22
|
};
|
|
22
|
-
export type
|
|
23
|
-
options: Pick<
|
|
23
|
+
export type RepoFetchSnapshotsData = {
|
|
24
|
+
options: Pick<SnapshotsActionOptions, "ids" | "packageNames" | "packageTaskNames" | "verbose" | "tags">;
|
|
24
25
|
};
|
|
25
|
-
export type
|
|
26
|
-
options:
|
|
27
|
-
snapshot:
|
|
26
|
+
export type RepoCopyData<TRepositoryConfig> = {
|
|
27
|
+
options: BackupActionOptions;
|
|
28
|
+
snapshot: PreSnapshot;
|
|
28
29
|
package: PackageConfigType;
|
|
29
30
|
mirrorRepositoryConfig: TRepositoryConfig;
|
|
30
|
-
onProgress: (data: Progress) =>
|
|
31
|
+
onProgress: (data: Progress) => void;
|
|
31
32
|
};
|
|
32
|
-
export type
|
|
33
|
-
options:
|
|
34
|
-
snapshot:
|
|
35
|
-
package: PackageConfigType
|
|
36
|
-
|
|
33
|
+
export type RepoBackupData<TPackageConfig> = {
|
|
34
|
+
options: BackupActionOptions;
|
|
35
|
+
snapshot: PreSnapshot;
|
|
36
|
+
package: Omit<PackageConfigType, "path"> & {
|
|
37
|
+
path: string;
|
|
38
|
+
};
|
|
37
39
|
packageConfig: TPackageConfig | undefined;
|
|
38
|
-
onProgress: (data: Progress) =>
|
|
40
|
+
onProgress: (data: Progress) => void;
|
|
39
41
|
};
|
|
40
|
-
export type
|
|
41
|
-
options:
|
|
42
|
-
snapshot:
|
|
42
|
+
export type RepoRestoreData<TPackageConfig> = {
|
|
43
|
+
options: RestoreActionOptions;
|
|
44
|
+
snapshot: PreSnapshot;
|
|
43
45
|
package: PackageConfigType;
|
|
44
|
-
|
|
46
|
+
snapshotPath: string;
|
|
45
47
|
packageConfig: TPackageConfig;
|
|
46
|
-
onProgress: (data: Progress) =>
|
|
48
|
+
onProgress: (data: Progress) => void;
|
|
47
49
|
};
|
|
48
|
-
export type
|
|
49
|
-
snapshot:
|
|
50
|
+
export type RepoPruneData = {
|
|
51
|
+
snapshot: ExtendedSnapshot;
|
|
50
52
|
options: {
|
|
51
53
|
verbose?: boolean;
|
|
52
54
|
};
|
|
@@ -74,14 +76,14 @@ export type SnapshotTagObjectType = {
|
|
|
74
76
|
export declare abstract class RepositoryAbstract<TConfig> {
|
|
75
77
|
readonly repository: RepositoryConfigType;
|
|
76
78
|
readonly config: TConfig;
|
|
77
|
-
readonly tmpDirs: string[];
|
|
78
79
|
constructor(repository: RepositoryConfigType);
|
|
79
|
-
|
|
80
|
-
abstract
|
|
81
|
-
|
|
82
|
-
abstract
|
|
83
|
-
abstract
|
|
84
|
-
abstract
|
|
85
|
-
abstract
|
|
86
|
-
abstract
|
|
80
|
+
abstract getSource(): string;
|
|
81
|
+
abstract fetchDiskStats(config: TConfig): Promise<DiskStats | undefined>;
|
|
82
|
+
ensureFreeDiskSpace(config: TConfig, minFreeDiskSpace: number | string): Promise<void>;
|
|
83
|
+
abstract init(data: RepoInitData): Promise<void>;
|
|
84
|
+
abstract prune(data: RepoPruneData): Promise<void>;
|
|
85
|
+
abstract fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
86
|
+
abstract copy(data: RepoCopyData<TConfig>): Promise<void>;
|
|
87
|
+
abstract backup(data: RepoBackupData<unknown>): Promise<void>;
|
|
88
|
+
abstract restore(data: RepoRestoreData<unknown>): Promise<void>;
|
|
87
89
|
}
|
|
@@ -16,15 +16,14 @@ var SnapshotTagEnum;
|
|
|
16
16
|
class RepositoryAbstract {
|
|
17
17
|
repository;
|
|
18
18
|
config;
|
|
19
|
-
tmpDirs = [];
|
|
20
19
|
constructor(repository) {
|
|
21
20
|
this.repository = repository;
|
|
22
21
|
this.config = repository.config;
|
|
23
22
|
}
|
|
24
|
-
async
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
23
|
+
async ensureFreeDiskSpace(config, minFreeDiskSpace) {
|
|
24
|
+
const diskStats = await this.fetchDiskStats(config);
|
|
25
|
+
if (diskStats)
|
|
26
|
+
await (0, fs_1.ensureFreeDiskSpace)(diskStats, minFreeDiskSpace);
|
|
28
27
|
}
|
|
29
28
|
}
|
|
30
29
|
exports.RepositoryAbstract = RepositoryAbstract;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { RepositoryType } from "../utils/Restic";
|
|
2
|
-
import { RepositoryAbstract,
|
|
2
|
+
import { RepositoryAbstract, RepoBackupData, RepoInitData, RepoRestoreData, RepoFetchSnapshotsData, Snapshot, SnapshotTagObjectType, SnapshotTagEnum, RepoPruneData, RepoCopyData } from "./RepositoryAbstract";
|
|
3
3
|
import { JSONSchema7 } from "json-schema";
|
|
4
4
|
export type ResticRepositoryConfigType = {
|
|
5
5
|
password: string | {
|
|
@@ -31,11 +31,12 @@ export declare class ResticRepository extends RepositoryAbstract<ResticRepositor
|
|
|
31
31
|
static parseSnapshotTags(tags: string[]): SnapshotTagObjectType & {
|
|
32
32
|
tags: string[];
|
|
33
33
|
};
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
34
|
+
getSource(): string;
|
|
35
|
+
fetchDiskStats(config: ResticRepositoryConfigType): Promise<import("../utils/fs").DiskStats | undefined>;
|
|
36
|
+
init(data: RepoInitData): Promise<void>;
|
|
37
|
+
fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
|
|
38
|
+
prune(data: RepoPruneData): Promise<void>;
|
|
39
|
+
backup(data: RepoBackupData<ResticPackageRepositoryConfigType>): Promise<void>;
|
|
40
|
+
copy(data: RepoCopyData<ResticRepositoryConfigType>): Promise<void>;
|
|
41
|
+
restore(data: RepoRestoreData<ResticPackageRepositoryConfigType>): Promise<void>;
|
|
41
42
|
}
|
|
@@ -11,8 +11,8 @@ const paths_1 = require("../utils/datatruck/paths");
|
|
|
11
11
|
const fs_1 = require("../utils/fs");
|
|
12
12
|
const math_1 = require("../utils/math");
|
|
13
13
|
const string_1 = require("../utils/string");
|
|
14
|
+
const temp_1 = require("../utils/temp");
|
|
14
15
|
const RepositoryAbstract_1 = require("./RepositoryAbstract");
|
|
15
|
-
const assert_1 = require("assert");
|
|
16
16
|
const fast_glob_1 = __importDefault(require("fast-glob"));
|
|
17
17
|
const promises_1 = require("fs/promises");
|
|
18
18
|
const micromatch_1 = require("micromatch");
|
|
@@ -123,10 +123,14 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
123
123
|
}
|
|
124
124
|
return result;
|
|
125
125
|
}
|
|
126
|
-
|
|
126
|
+
getSource() {
|
|
127
127
|
return (0, string_1.formatUri)({ ...this.config.repository, password: undefined });
|
|
128
128
|
}
|
|
129
|
-
async
|
|
129
|
+
async fetchDiskStats(config) {
|
|
130
|
+
if (config.repository.backend === "local" && config.repository.path)
|
|
131
|
+
return (0, fs_1.fetchDiskStats)(config.repository.path);
|
|
132
|
+
}
|
|
133
|
+
async init(data) {
|
|
130
134
|
const restic = new Restic_1.Restic({
|
|
131
135
|
env: await this.buildEnv(),
|
|
132
136
|
log: data.options.verbose,
|
|
@@ -136,7 +140,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
136
140
|
if (!(await restic.checkRepository()))
|
|
137
141
|
await restic.exec(["init"]);
|
|
138
142
|
}
|
|
139
|
-
async
|
|
143
|
+
async fetchSnapshots(data) {
|
|
140
144
|
const restic = new Restic_1.Restic({
|
|
141
145
|
env: await this.buildEnv(),
|
|
142
146
|
log: data.options.verbose,
|
|
@@ -172,7 +176,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
172
176
|
return items;
|
|
173
177
|
}, []);
|
|
174
178
|
}
|
|
175
|
-
async
|
|
179
|
+
async prune(data) {
|
|
176
180
|
const restic = new Restic_1.Restic({
|
|
177
181
|
env: await this.buildEnv(),
|
|
178
182
|
log: data.options.verbose,
|
|
@@ -182,30 +186,29 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
182
186
|
prune: true,
|
|
183
187
|
});
|
|
184
188
|
}
|
|
185
|
-
async
|
|
189
|
+
async backup(data) {
|
|
186
190
|
const restic = new Restic_1.Restic({
|
|
187
191
|
env: await this.buildEnv(),
|
|
188
192
|
log: data.options.verbose,
|
|
189
193
|
});
|
|
190
194
|
const pkg = data.package;
|
|
191
|
-
const
|
|
192
|
-
(0, assert_1.ok)(sourcePath);
|
|
195
|
+
const path = data.package.path;
|
|
193
196
|
let gitignorePath;
|
|
194
197
|
const backupPathsOptions = {
|
|
195
198
|
package: data.package,
|
|
196
199
|
snapshot: data.snapshot,
|
|
197
|
-
|
|
200
|
+
path,
|
|
198
201
|
verbose: data.options.verbose,
|
|
199
202
|
};
|
|
200
203
|
if (!pkg.include && pkg.exclude) {
|
|
201
204
|
const exclude = await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions);
|
|
202
|
-
|
|
205
|
+
data.onProgress({
|
|
203
206
|
relative: {
|
|
204
207
|
description: "Writing excluded paths list",
|
|
205
208
|
},
|
|
206
209
|
});
|
|
207
|
-
const tmpDir = await
|
|
208
|
-
const ignoredContents = (0, fs_1.fastglobToGitIgnore)(exclude,
|
|
210
|
+
const tmpDir = await (0, temp_1.mkTmpDir)(exports.resticRepositoryName, "repo", "backup", "exclude");
|
|
211
|
+
const ignoredContents = (0, fs_1.fastglobToGitIgnore)(exclude, path).join("\n");
|
|
209
212
|
gitignorePath = (0, path_1.join)(tmpDir, "ignored.txt");
|
|
210
213
|
await (0, promises_1.writeFile)(gitignorePath, ignoredContents);
|
|
211
214
|
}
|
|
@@ -215,7 +218,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
215
218
|
? await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions)
|
|
216
219
|
: undefined;
|
|
217
220
|
const stream = fast_glob_1.default.stream(include, {
|
|
218
|
-
cwd:
|
|
221
|
+
cwd: path,
|
|
219
222
|
ignore: exclude,
|
|
220
223
|
dot: true,
|
|
221
224
|
onlyFiles: true,
|
|
@@ -223,20 +226,20 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
223
226
|
});
|
|
224
227
|
if (data.options.verbose)
|
|
225
228
|
(0, cli_1.logExec)(`Writing paths lists`);
|
|
226
|
-
|
|
229
|
+
data.onProgress({
|
|
227
230
|
relative: {
|
|
228
231
|
description: "Writing excluded paths list",
|
|
229
232
|
},
|
|
230
233
|
});
|
|
231
234
|
gitignorePath = await (0, fs_1.writeGitIgnoreList)({
|
|
232
235
|
paths: stream,
|
|
233
|
-
outDir: await
|
|
236
|
+
outDir: await (0, temp_1.mkTmpDir)(exports.resticRepositoryName, "repo", "backup", "gitignore"),
|
|
234
237
|
});
|
|
235
238
|
}
|
|
236
239
|
if (data.options.tags?.some((tag) => tag.startsWith(ResticRepository.refPrefix)))
|
|
237
240
|
throw new AppError_1.AppError(`Tag prefix is not allowed`);
|
|
238
241
|
const packageTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.PACKAGE, data.package.name);
|
|
239
|
-
|
|
242
|
+
data.onProgress({
|
|
240
243
|
relative: {
|
|
241
244
|
description: "Fetching last snapshot",
|
|
242
245
|
},
|
|
@@ -250,7 +253,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
250
253
|
let lastProgress;
|
|
251
254
|
let totalFilesChanges = 0;
|
|
252
255
|
const totalFilesChangesLimit = 10;
|
|
253
|
-
|
|
256
|
+
data.onProgress({
|
|
254
257
|
relative: {
|
|
255
258
|
description: "Executing backup action",
|
|
256
259
|
},
|
|
@@ -258,7 +261,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
258
261
|
let resticSnapshotId;
|
|
259
262
|
let resticTotalBytes;
|
|
260
263
|
await restic.backup({
|
|
261
|
-
cwd:
|
|
264
|
+
cwd: path,
|
|
262
265
|
paths: ["."],
|
|
263
266
|
allowEmptySnapshot: true,
|
|
264
267
|
excludeFile: gitignorePath ? [gitignorePath] : undefined,
|
|
@@ -280,7 +283,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
280
283
|
: []),
|
|
281
284
|
...(data.options.tags ?? []),
|
|
282
285
|
],
|
|
283
|
-
createEmptyDir: async () => await
|
|
286
|
+
createEmptyDir: async () => await (0, temp_1.mkTmpDir)(exports.resticRepositoryName, "repo", "backup", "empty-dir"),
|
|
284
287
|
onStream: async (streamData) => {
|
|
285
288
|
if (streamData.message_type === "status") {
|
|
286
289
|
let showProgressBar = false;
|
|
@@ -293,7 +296,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
293
296
|
else {
|
|
294
297
|
totalFilesChanges++;
|
|
295
298
|
}
|
|
296
|
-
|
|
299
|
+
data.onProgress((lastProgress = {
|
|
297
300
|
relative: {
|
|
298
301
|
description: "Copying file",
|
|
299
302
|
payload: streamData.current_files?.join(", ") ?? "-",
|
|
@@ -319,7 +322,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
319
322
|
throw new AppError_1.AppError(`Restic snapshot total bytes is not defined`);
|
|
320
323
|
const sizeTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.SIZE, resticTotalBytes.toString());
|
|
321
324
|
await restic.exec(["tag", "--add", sizeTag, resticSnapshotId]);
|
|
322
|
-
|
|
325
|
+
data.onProgress({
|
|
323
326
|
absolute: {
|
|
324
327
|
total: lastProgress?.absolute?.total || 0,
|
|
325
328
|
current: lastProgress?.absolute?.total || 0,
|
|
@@ -327,9 +330,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
327
330
|
},
|
|
328
331
|
});
|
|
329
332
|
}
|
|
330
|
-
async
|
|
333
|
+
async copy(data) {
|
|
331
334
|
const config = data.mirrorRepositoryConfig;
|
|
332
|
-
const [snapshot] = await this.
|
|
335
|
+
const [snapshot] = await this.fetchSnapshots({
|
|
333
336
|
options: {
|
|
334
337
|
ids: [data.snapshot.id],
|
|
335
338
|
packageNames: [data.package.name],
|
|
@@ -351,14 +354,13 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
351
354
|
id: snapshot.originalId,
|
|
352
355
|
});
|
|
353
356
|
}
|
|
354
|
-
async
|
|
355
|
-
const restorePath = data.
|
|
356
|
-
(0, assert_1.ok)(restorePath);
|
|
357
|
+
async restore(data) {
|
|
358
|
+
const restorePath = data.snapshotPath;
|
|
357
359
|
const restic = new Restic_1.Restic({
|
|
358
360
|
env: await this.buildEnv(),
|
|
359
361
|
log: data.options.verbose,
|
|
360
362
|
});
|
|
361
|
-
const [snapshot] = await this.
|
|
363
|
+
const [snapshot] = await this.fetchSnapshots({
|
|
362
364
|
options: {
|
|
363
365
|
ids: [data.snapshot.id],
|
|
364
366
|
packageNames: [data.package.name],
|
|
@@ -372,7 +374,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
372
374
|
onStream: async (streamData) => {
|
|
373
375
|
if (streamData.message_type === "restore-status") {
|
|
374
376
|
const current = Math.min(streamData.total_bytes, snapshot.size);
|
|
375
|
-
|
|
377
|
+
data.onProgress({
|
|
376
378
|
absolute: {
|
|
377
379
|
total: snapshot.size,
|
|
378
380
|
current,
|
package/Task/GitTask.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TaskBackupData, TaskRestoreData, TaskAbstract } from "./TaskAbstract";
|
|
2
2
|
import { JSONSchema7 } from "json-schema";
|
|
3
3
|
export type GitTaskConfigType = {
|
|
4
4
|
command?: string;
|
|
@@ -28,12 +28,11 @@ export declare const gitTaskDefinition: JSONSchema7;
|
|
|
28
28
|
export declare class GitTask extends TaskAbstract<GitTaskConfigType> {
|
|
29
29
|
protected verbose?: boolean;
|
|
30
30
|
private get command();
|
|
31
|
-
|
|
32
|
-
|
|
31
|
+
backup(data: TaskBackupData): Promise<{
|
|
32
|
+
snapshotPath: string;
|
|
33
33
|
}>;
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
targetPath: string;
|
|
34
|
+
prepareRestore(): Promise<{
|
|
35
|
+
snapshotPath: string;
|
|
37
36
|
}>;
|
|
38
|
-
|
|
37
|
+
restore(data: TaskRestoreData): Promise<void>;
|
|
39
38
|
}
|
package/Task/GitTask.js
CHANGED
|
@@ -6,6 +6,7 @@ const cli_1 = require("../utils/cli");
|
|
|
6
6
|
const fs_1 = require("../utils/fs");
|
|
7
7
|
const math_1 = require("../utils/math");
|
|
8
8
|
const process_1 = require("../utils/process");
|
|
9
|
+
const temp_1 = require("../utils/temp");
|
|
9
10
|
const TaskAbstract_1 = require("./TaskAbstract");
|
|
10
11
|
const assert_1 = require("assert");
|
|
11
12
|
const fs_2 = require("fs");
|
|
@@ -59,21 +60,17 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
59
60
|
get command() {
|
|
60
61
|
return this.config.command ?? "git";
|
|
61
62
|
}
|
|
62
|
-
async
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
}
|
|
67
|
-
async onBackup(data) {
|
|
63
|
+
async backup(data) {
|
|
64
|
+
if (!data.package.path)
|
|
65
|
+
throw new Error(`Path is required`);
|
|
66
|
+
const snapshotPath = await (0, temp_1.mkTmpDir)(exports.gitTaskName, "task", "backup", "snapshot");
|
|
68
67
|
this.verbose = data.options.verbose;
|
|
69
68
|
const config = this.config;
|
|
70
69
|
const path = data.package.path;
|
|
71
|
-
const targetPath = data.targetPath;
|
|
72
70
|
(0, assert_1.ok)(typeof path === "string");
|
|
73
|
-
(0, assert_1.ok)(typeof targetPath === "string");
|
|
74
71
|
// Bundle
|
|
75
|
-
const bundlePath = (0, path_1.join)(
|
|
76
|
-
|
|
72
|
+
const bundlePath = (0, path_1.join)(snapshotPath, "repo.bundle");
|
|
73
|
+
data.onProgress({
|
|
77
74
|
relative: {
|
|
78
75
|
description: "Creating bundle",
|
|
79
76
|
},
|
|
@@ -85,7 +82,7 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
85
82
|
});
|
|
86
83
|
// Config
|
|
87
84
|
if (this.config.includeConfig ?? true) {
|
|
88
|
-
const configPath = (0, path_1.join)(
|
|
85
|
+
const configPath = (0, path_1.join)(snapshotPath, "repo.config");
|
|
89
86
|
await (0, promises_1.copyFile)((0, path_1.join)(path, ".git", "config"), configPath);
|
|
90
87
|
}
|
|
91
88
|
// git ls-files
|
|
@@ -112,7 +109,7 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
112
109
|
for (const option of lsFilesConfig) {
|
|
113
110
|
if (!option.include)
|
|
114
111
|
continue;
|
|
115
|
-
option.pathsPath = (0, path_1.join)(
|
|
112
|
+
option.pathsPath = (0, path_1.join)(snapshotPath, `repo.${option.name}-paths.txt`);
|
|
116
113
|
const stream = (0, fs_2.createWriteStream)(option.pathsPath);
|
|
117
114
|
let streamError;
|
|
118
115
|
stream.on("error", (e) => (streamError = e));
|
|
@@ -161,7 +158,7 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
161
158
|
for (const option of lsFilesConfig) {
|
|
162
159
|
if (!option.include)
|
|
163
160
|
continue;
|
|
164
|
-
const outPath = (0, path_1.join)(
|
|
161
|
+
const outPath = (0, path_1.join)(snapshotPath, `repo.${option.name}`);
|
|
165
162
|
await (0, fs_1.mkdirIfNotExists)(outPath);
|
|
166
163
|
if (data.options.verbose)
|
|
167
164
|
(0, cli_1.logExec)(`Copying ${option.name} files to ${outPath}`);
|
|
@@ -171,12 +168,12 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
171
168
|
path: option.pathsPath,
|
|
172
169
|
basePath: path,
|
|
173
170
|
},
|
|
174
|
-
|
|
171
|
+
outPath: outPath,
|
|
175
172
|
skipNotFoundError: true,
|
|
176
173
|
concurrency: this.config.fileCopyConcurrency,
|
|
177
174
|
onPath: async ({ entryPath }) => {
|
|
178
175
|
currentFiles++;
|
|
179
|
-
|
|
176
|
+
data.onProgress({
|
|
180
177
|
relative: {
|
|
181
178
|
description: "Copying file",
|
|
182
179
|
payload: entryPath,
|
|
@@ -191,26 +188,23 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
191
188
|
});
|
|
192
189
|
await (0, promises_1.rm)(option.pathsPath);
|
|
193
190
|
}
|
|
191
|
+
return { snapshotPath };
|
|
194
192
|
}
|
|
195
|
-
async
|
|
193
|
+
async prepareRestore() {
|
|
196
194
|
return {
|
|
197
|
-
|
|
195
|
+
snapshotPath: await (0, temp_1.mkTmpDir)(exports.gitTaskName, "task", "restore", "snapshot"),
|
|
198
196
|
};
|
|
199
197
|
}
|
|
200
|
-
async
|
|
198
|
+
async restore(data) {
|
|
201
199
|
this.verbose = data.options.verbose;
|
|
202
|
-
const
|
|
203
|
-
const
|
|
204
|
-
(0, assert_1.ok)(typeof restorePath === "string");
|
|
205
|
-
(0, assert_1.ok)(typeof targetPath === "string");
|
|
206
|
-
await (0, fs_1.mkdirIfNotExists)(restorePath);
|
|
207
|
-
await (0, fs_1.ensureEmptyDir)(restorePath);
|
|
200
|
+
const snapshotPath = data.snapshotPath;
|
|
201
|
+
const restorePath = await (0, fs_1.initEmptyDir)(data.package.restorePath ?? data.package.path);
|
|
208
202
|
// Stats
|
|
209
203
|
let totalFiles = 0;
|
|
210
204
|
let currentFiles = 0;
|
|
211
|
-
await (0, fs_1.forEachFile)(
|
|
205
|
+
await (0, fs_1.forEachFile)(snapshotPath, () => totalFiles++, true);
|
|
212
206
|
const incrementProgress = async (description, item, count = true) => {
|
|
213
|
-
|
|
207
|
+
data.onProgress({
|
|
214
208
|
absolute: {
|
|
215
209
|
total: totalFiles,
|
|
216
210
|
current: Math.max(currentFiles, 0),
|
|
@@ -222,7 +216,7 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
222
216
|
currentFiles++;
|
|
223
217
|
};
|
|
224
218
|
// Bundle
|
|
225
|
-
const bundlePath = (0, path_1.join)(
|
|
219
|
+
const bundlePath = (0, path_1.join)(snapshotPath, "repo.bundle");
|
|
226
220
|
await (0, process_1.exec)(this.command, ["clone", bundlePath, "."], {
|
|
227
221
|
cwd: restorePath,
|
|
228
222
|
}, {
|
|
@@ -230,14 +224,14 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
230
224
|
});
|
|
231
225
|
await incrementProgress();
|
|
232
226
|
// Config
|
|
233
|
-
const configPath = (0, path_1.join)(
|
|
227
|
+
const configPath = (0, path_1.join)(snapshotPath, "repo.config");
|
|
234
228
|
if (await (0, fs_1.existsFile)(configPath)) {
|
|
235
229
|
await (0, promises_1.copyFile)(configPath, (0, path_1.join)(restorePath, ".git", "config"));
|
|
236
230
|
await incrementProgress();
|
|
237
231
|
}
|
|
238
232
|
// ls-files
|
|
239
233
|
for (const name of ["untracked", "modified", "ignored"]) {
|
|
240
|
-
const sourcePath = (0, path_1.join)(
|
|
234
|
+
const sourcePath = (0, path_1.join)(snapshotPath, `repo.${name}`);
|
|
241
235
|
if (await (0, fs_1.existsDir)(sourcePath)) {
|
|
242
236
|
if (data.options.verbose)
|
|
243
237
|
(0, cli_1.logExec)(`Copying ${name} files to ${restorePath}`);
|
|
@@ -246,7 +240,7 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
|
|
|
246
240
|
type: "glob",
|
|
247
241
|
sourcePath,
|
|
248
242
|
},
|
|
249
|
-
|
|
243
|
+
outPath: restorePath,
|
|
250
244
|
concurrency: this.config.fileCopyConcurrency,
|
|
251
245
|
onProgress: async (progress) => await incrementProgress(progress.type === "end" ? "Files copied" : "Copying file", progress.path, !progress.type),
|
|
252
246
|
});
|