@datatruck/cli 0.27.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/BackupAction.d.ts +75 -34
- package/Action/BackupAction.js +302 -246
- package/Action/CleanCacheAction.d.ts +8 -4
- package/Action/CleanCacheAction.js +8 -5
- package/Action/ConfigAction.d.ts +12 -5
- package/Action/ConfigAction.js +14 -18
- package/Action/CopyAction.d.ts +51 -0
- package/Action/CopyAction.js +165 -0
- package/Action/InitAction.d.ts +3 -3
- package/Action/InitAction.js +9 -9
- package/Action/PruneAction.d.ts +10 -16
- package/Action/PruneAction.js +37 -34
- package/Action/RestoreAction.d.ts +49 -24
- package/Action/RestoreAction.js +164 -195
- package/Action/SnapshotsAction.d.ts +8 -8
- package/Action/SnapshotsAction.js +9 -9
- package/CHANGELOG.md +513 -0
- package/Command/BackupCommand.d.ts +7 -4
- package/Command/BackupCommand.js +14 -26
- package/Command/CleanCacheCommand.d.ts +4 -4
- package/Command/CleanCacheCommand.js +26 -5
- package/Command/CommandAbstract.d.ts +10 -7
- package/Command/CommandAbstract.js +4 -1
- package/Command/ConfigCommand.d.ts +6 -9
- package/Command/ConfigCommand.js +13 -8
- package/Command/CopyCommand.d.ts +16 -0
- package/Command/CopyCommand.js +66 -0
- package/Command/InitCommand.d.ts +4 -4
- package/Command/InitCommand.js +13 -17
- package/Command/PruneCommand.d.ts +4 -10
- package/Command/PruneCommand.js +13 -12
- package/Command/RestoreCommand.d.ts +1 -1
- package/Command/RestoreCommand.js +13 -21
- package/Command/SnapshotsCommand.d.ts +4 -4
- package/Command/SnapshotsCommand.js +16 -15
- package/Command/StartServerCommand.d.ts +3 -3
- package/Config/Config.d.ts +11 -0
- package/Config/Config.js +18 -0
- package/Config/PrunePolicyConfig.d.ts +2 -2
- package/Factory/CommandFactory.d.ts +27 -34
- package/Factory/CommandFactory.js +27 -54
- package/Factory/RepositoryFactory.d.ts +1 -1
- package/Factory/RepositoryFactory.js +3 -3
- package/Factory/TaskFactory.d.ts +1 -1
- package/Factory/TaskFactory.js +3 -3
- package/Repository/DatatruckRepository.d.ts +10 -8
- package/Repository/DatatruckRepository.js +47 -25
- package/Repository/GitRepository.d.ts +9 -8
- package/Repository/GitRepository.js +22 -25
- package/Repository/RepositoryAbstract.d.ts +39 -37
- package/Repository/RepositoryAbstract.js +4 -5
- package/Repository/ResticRepository.d.ts +9 -8
- package/Repository/ResticRepository.js +30 -28
- package/Task/GitTask.d.ts +6 -7
- package/Task/GitTask.js +24 -30
- package/Task/MariadbTask.d.ts +4 -5
- package/Task/MariadbTask.js +26 -32
- package/Task/MssqlTask.d.ts +5 -3
- package/Task/MssqlTask.js +11 -12
- package/Task/MysqlDumpTask.d.ts +10 -3
- package/Task/MysqlDumpTask.js +102 -32
- package/Task/ScriptTask.d.ts +23 -18
- package/Task/ScriptTask.js +34 -24
- package/Task/SqlDumpTaskAbstract.d.ts +8 -3
- package/Task/SqlDumpTaskAbstract.js +32 -20
- package/Task/TaskAbstract.d.ts +24 -25
- package/Task/TaskAbstract.js +6 -10
- package/cli.js +13 -5
- package/config.schema.json +89 -1
- package/package.json +4 -5
- package/utils/DataFormat.d.ts +23 -12
- package/utils/DataFormat.js +36 -14
- package/utils/cli.d.ts +3 -9
- package/utils/cli.js +19 -55
- package/utils/crypto.d.ts +1 -0
- package/utils/crypto.js +15 -0
- package/utils/datatruck/client.d.ts +2 -0
- package/utils/datatruck/client.js +3 -0
- package/utils/datatruck/config.d.ts +2 -0
- package/utils/datatruck/config.js +18 -3
- package/utils/datatruck/paths.d.ts +5 -9
- package/utils/datatruck/paths.js +2 -2
- package/utils/datatruck/snapshot.d.ts +5 -2
- package/utils/datatruck/snapshot.js +12 -22
- package/utils/date.d.ts +21 -4
- package/utils/date.js +46 -24
- package/utils/fs.d.ts +16 -11
- package/utils/fs.js +81 -48
- package/utils/list.d.ts +64 -0
- package/utils/list.js +145 -0
- package/utils/mysql.d.ts +2 -0
- package/utils/mysql.js +21 -2
- package/utils/process.d.ts +1 -0
- package/utils/process.js +24 -31
- package/utils/progress.d.ts +33 -0
- package/utils/progress.js +113 -0
- package/utils/steps.d.ts +11 -0
- package/utils/steps.js +22 -10
- package/utils/stream.d.ts +7 -0
- package/utils/stream.js +10 -0
- package/utils/string.d.ts +0 -1
- package/utils/string.js +1 -13
- package/utils/tar.d.ts +10 -3
- package/utils/tar.js +73 -45
- package/utils/temp.d.ts +26 -0
- package/utils/temp.js +133 -0
- package/utils/virtual-fs.d.ts +6 -2
- package/utils/virtual-fs.js +6 -0
- package/Action/BackupSessionsAction.d.ts +0 -13
- package/Action/BackupSessionsAction.js +0 -18
- package/Action/RestoreSessionsAction.d.ts +0 -13
- package/Action/RestoreSessionsAction.js +0 -18
- package/Command/BackupSessionsCommand.d.ts +0 -12
- package/Command/BackupSessionsCommand.js +0 -92
- package/Command/RestoreSessionsCommand.d.ts +0 -12
- package/Command/RestoreSessionsCommand.js +0 -91
- package/Decorator/EntityDecorator.d.ts +0 -11
- package/Decorator/EntityDecorator.js +0 -17
- package/Entity/BackupSessionEntity.d.ts +0 -6
- package/Entity/BackupSessionEntity.js +0 -25
- package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
- package/Entity/BackupSessionRepositoryEntity.js +0 -25
- package/Entity/BackupSessionTaskEntity.d.ts +0 -5
- package/Entity/BackupSessionTaskEntity.js +0 -24
- package/Entity/CrudEntityAbstract.d.ts +0 -5
- package/Entity/CrudEntityAbstract.js +0 -9
- package/Entity/RestoreSessionEntity.d.ts +0 -5
- package/Entity/RestoreSessionEntity.js +0 -24
- package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
- package/Entity/RestoreSessionRepositoryEntity.js +0 -25
- package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
- package/Entity/RestoreSessionTaskEntity.js +0 -24
- package/Entity/StateEntityAbstract.d.ts +0 -9
- package/Entity/StateEntityAbstract.js +0 -12
- package/Factory/EntityFactory.d.ts +0 -6
- package/Factory/EntityFactory.js +0 -40
- package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
- package/SessionDriver/ConsoleSessionDriver.js +0 -208
- package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
- package/SessionDriver/SessionDriverAbstract.js +0 -28
- package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
- package/SessionDriver/SqliteSessionDriver.js +0 -173
- package/SessionManager/BackupSessionManager.d.ts +0 -45
- package/SessionManager/BackupSessionManager.js +0 -218
- package/SessionManager/RestoreSessionManager.d.ts +0 -47
- package/SessionManager/RestoreSessionManager.js +0 -218
- package/SessionManager/SessionManagerAbstract.d.ts +0 -18
- package/SessionManager/SessionManagerAbstract.js +0 -36
- package/migrations/001-initial.sql +0 -98
- package/utils/entity.d.ts +0 -4
- package/utils/entity.js +0 -10
package/utils/progress.js
CHANGED
|
@@ -1,2 +1,115 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
2
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.renderProgressStats = exports.renderProgress = exports.ProgressManager = void 0;
|
|
7
|
+
const cli_1 = require("./cli");
|
|
8
|
+
const date_1 = require("./date");
|
|
9
|
+
const bytes_1 = __importDefault(require("bytes"));
|
|
10
|
+
const chalk_1 = require("chalk");
|
|
11
|
+
const readline_1 = require("readline");
|
|
12
|
+
class ProgressManager {
|
|
13
|
+
options;
|
|
14
|
+
timer = (0, date_1.createTimer)();
|
|
15
|
+
interval = (0, date_1.createTimer)();
|
|
16
|
+
keydownListener;
|
|
17
|
+
tty;
|
|
18
|
+
enabled;
|
|
19
|
+
constructor(options) {
|
|
20
|
+
this.options = options;
|
|
21
|
+
this.tty =
|
|
22
|
+
options.tty === "auto"
|
|
23
|
+
? options.verbose
|
|
24
|
+
? false
|
|
25
|
+
: process.stdout.isTTY
|
|
26
|
+
: !!options.tty;
|
|
27
|
+
this.enabled =
|
|
28
|
+
options.enabled === "auto"
|
|
29
|
+
? this.tty
|
|
30
|
+
? true
|
|
31
|
+
: "interval"
|
|
32
|
+
: !!options.enabled;
|
|
33
|
+
}
|
|
34
|
+
elapsed() {
|
|
35
|
+
return this.timer.elapsed();
|
|
36
|
+
}
|
|
37
|
+
start() {
|
|
38
|
+
this.timer.reset();
|
|
39
|
+
(0, readline_1.emitKeypressEvents)(process.stdin);
|
|
40
|
+
process.stdin?.setRawMode?.(true);
|
|
41
|
+
process.stdin?.resume();
|
|
42
|
+
process.stdin?.setEncoding("utf8");
|
|
43
|
+
process.stdin?.on("keypress", (this.keydownListener = (inKey) => {
|
|
44
|
+
const key = inKey.toString();
|
|
45
|
+
if (key === "\u0003") {
|
|
46
|
+
process.stdin.setRawMode?.(false);
|
|
47
|
+
process.emit("SIGINT");
|
|
48
|
+
}
|
|
49
|
+
else if (/^(\r\n)|\r|\n$/.test(key)) {
|
|
50
|
+
this.interval = undefined;
|
|
51
|
+
}
|
|
52
|
+
}));
|
|
53
|
+
}
|
|
54
|
+
dispose() {
|
|
55
|
+
this.timer.stop();
|
|
56
|
+
if (this.keydownListener) {
|
|
57
|
+
process.stdin?.off("keypress", this.keydownListener);
|
|
58
|
+
this.keydownListener = undefined;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
update(progress, cb) {
|
|
62
|
+
if (!this.enabled)
|
|
63
|
+
return;
|
|
64
|
+
if (this.enabled === "interval") {
|
|
65
|
+
if (this.interval) {
|
|
66
|
+
if (!this.interval.reset(this.options.interval ?? 5000))
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
this.interval = (0, date_1.createTimer)();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
cb(renderProgress(progress, this.tty).join("\n"));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
exports.ProgressManager = ProgressManager;
|
|
77
|
+
function renderProgress(progress, bar) {
|
|
78
|
+
return [
|
|
79
|
+
progress.absolute && renderProgressStats(progress.absolute, bar),
|
|
80
|
+
progress.relative && renderProgressStats(progress.relative, bar),
|
|
81
|
+
].filter((v) => !!v);
|
|
82
|
+
}
|
|
83
|
+
exports.renderProgress = renderProgress;
|
|
84
|
+
function renderProgressStats(stats, progressBar) {
|
|
85
|
+
const text = [];
|
|
86
|
+
if (typeof stats.percent === "number") {
|
|
87
|
+
if (progressBar)
|
|
88
|
+
text.push((0, cli_1.renderProgressBar)(stats.percent));
|
|
89
|
+
text.push(`${stats.percent.toFixed(2)}%`);
|
|
90
|
+
}
|
|
91
|
+
if (typeof stats.current === "number" || typeof stats.total === "number") {
|
|
92
|
+
const format = (value) => stats.format === "size" ? (0, bytes_1.default)(value) : value;
|
|
93
|
+
if (typeof stats.current === "number" && typeof stats.total === "number") {
|
|
94
|
+
text.push(`${format(stats.current)}/${format(stats.total)}`);
|
|
95
|
+
}
|
|
96
|
+
else if (typeof stats.current === "number") {
|
|
97
|
+
text.push(`${format(stats.current)}`);
|
|
98
|
+
}
|
|
99
|
+
else if (typeof stats.total === "number") {
|
|
100
|
+
text.push(`?/${format(stats.total)}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (stats.description && stats.payload) {
|
|
104
|
+
text.push(`${stats.description}: ${stats.payload}`);
|
|
105
|
+
}
|
|
106
|
+
else if (stats.description) {
|
|
107
|
+
text.push(stats.description);
|
|
108
|
+
}
|
|
109
|
+
else if (stats.payload) {
|
|
110
|
+
text.push(stats.payload);
|
|
111
|
+
}
|
|
112
|
+
const sep = (0, chalk_1.grey)(`|`);
|
|
113
|
+
return text.join(` ${sep} `);
|
|
114
|
+
}
|
|
115
|
+
exports.renderProgressStats = renderProgressStats;
|
package/utils/steps.d.ts
CHANGED
|
@@ -9,12 +9,20 @@ export type NodeStepConfig = {
|
|
|
9
9
|
vars?: Record<string, any>;
|
|
10
10
|
code: string | string[];
|
|
11
11
|
};
|
|
12
|
+
export type TelegramMessageStepConfig = {
|
|
13
|
+
bot: string;
|
|
14
|
+
chatId: number;
|
|
15
|
+
text?: string;
|
|
16
|
+
};
|
|
12
17
|
export type Step = {
|
|
13
18
|
type: "process";
|
|
14
19
|
config: ProcessStepConfig;
|
|
15
20
|
} | {
|
|
16
21
|
type: "node";
|
|
17
22
|
config: NodeStepConfig;
|
|
23
|
+
} | {
|
|
24
|
+
type: "telegram-message";
|
|
25
|
+
config: TelegramMessageStepConfig;
|
|
18
26
|
};
|
|
19
27
|
export type StepOptions = {
|
|
20
28
|
env?: Record<string, string | undefined>;
|
|
@@ -25,6 +33,9 @@ export type StepOptions = {
|
|
|
25
33
|
vars?: Record<string, any>;
|
|
26
34
|
tempDir?: () => Promise<string>;
|
|
27
35
|
};
|
|
36
|
+
telegram?: {
|
|
37
|
+
vars?: Record<string, string>;
|
|
38
|
+
};
|
|
28
39
|
cwd?: string;
|
|
29
40
|
onLine?: (p: string) => any;
|
|
30
41
|
verbose?: boolean;
|
package/utils/steps.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.runSteps = void 0;
|
|
4
|
-
const
|
|
4
|
+
const http_1 = require("./http");
|
|
5
5
|
const process_1 = require("./process");
|
|
6
6
|
const string_1 = require("./string");
|
|
7
|
+
const temp_1 = require("./temp");
|
|
7
8
|
const promises_1 = require("fs/promises");
|
|
8
9
|
const path_1 = require("path");
|
|
9
10
|
async function runSteps(input, options) {
|
|
@@ -33,19 +34,19 @@ async function runSteps(input, options) {
|
|
|
33
34
|
tempDir = await options.node.tempDir();
|
|
34
35
|
}
|
|
35
36
|
else {
|
|
36
|
-
tempDir = await (0,
|
|
37
|
+
tempDir = await (0, temp_1.mkTmpDir)("node-step");
|
|
37
38
|
}
|
|
38
39
|
const scriptPath = (0, path_1.join)(tempDir, "script.js");
|
|
39
|
-
const vars =
|
|
40
|
+
const vars = {
|
|
40
41
|
...step.config.vars,
|
|
41
42
|
...options.node?.vars,
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
43
|
+
};
|
|
44
|
+
const varKeys = Object.keys(vars);
|
|
45
|
+
const varJson = JSON.stringify(vars);
|
|
46
|
+
const code = Array.isArray(step.config.code)
|
|
47
|
+
? [...step.config.code].join(";\n")
|
|
48
|
+
: step.config.code;
|
|
49
|
+
await (0, promises_1.writeFile)(scriptPath, `(async function({ ${varKeys} }) {\n${code};\n})(${varJson});`);
|
|
49
50
|
await (0, process_1.exec)("node", [scriptPath], {
|
|
50
51
|
cwd: options.cwd,
|
|
51
52
|
env: {
|
|
@@ -63,6 +64,17 @@ async function runSteps(input, options) {
|
|
|
63
64
|
}),
|
|
64
65
|
});
|
|
65
66
|
}
|
|
67
|
+
else if (step.type === "telegram-message") {
|
|
68
|
+
await (0, http_1.post)(`https://api.telegram.org/bot${step.config.bot}/sendMessage`, JSON.stringify({
|
|
69
|
+
chat_id: step.config.chatId.toString(),
|
|
70
|
+
text: (0, string_1.render)(step.config.text ?? `{TEXT}`, options.telegram?.vars || {}),
|
|
71
|
+
disable_notification: true,
|
|
72
|
+
}), {
|
|
73
|
+
headers: {
|
|
74
|
+
"Content-Type": "application/json",
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
}
|
|
66
78
|
else {
|
|
67
79
|
throw new Error(`Invalid step type: ${step.type}`);
|
|
68
80
|
}
|
package/utils/stream.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createStreams = void 0;
|
|
4
|
+
function createStreams(options = {}) {
|
|
5
|
+
return {
|
|
6
|
+
stdout: options.stdout ?? process.stdout,
|
|
7
|
+
stderr: options.stderr ?? process.stderr,
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
exports.createStreams = createStreams;
|
package/utils/string.d.ts
CHANGED
|
@@ -19,6 +19,5 @@ export declare function endsWith(input: string, patterns: string[]): boolean;
|
|
|
19
19
|
export declare function createMatchFilter(include?: string[], exclude?: string[]): (input: string) => boolean;
|
|
20
20
|
export declare function checkMatch(subject: string | undefined, patterns: string[]): boolean;
|
|
21
21
|
export declare function formatDateTime(datetime: string): string;
|
|
22
|
-
export declare function splitLines(input: string, satinize?: boolean): string[];
|
|
23
22
|
export declare function undefIfEmpty(input: string): string | undefined;
|
|
24
23
|
export {};
|
package/utils/string.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.undefIfEmpty = exports.
|
|
3
|
+
exports.undefIfEmpty = exports.formatDateTime = exports.checkMatch = exports.createMatchFilter = exports.endsWith = exports.match = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const micromatch_1 = require("micromatch");
|
|
6
6
|
function serialize(message, data) {
|
|
@@ -121,18 +121,6 @@ function formatDateTime(datetime) {
|
|
|
121
121
|
return result;
|
|
122
122
|
}
|
|
123
123
|
exports.formatDateTime = formatDateTime;
|
|
124
|
-
function splitLines(input, satinize = true) {
|
|
125
|
-
const lines = input.split(/\r?\n/);
|
|
126
|
-
return satinize
|
|
127
|
-
? input.split(/\r?\n/).reduce((result, value) => {
|
|
128
|
-
value = value.trim();
|
|
129
|
-
if (value.length)
|
|
130
|
-
result.push(value);
|
|
131
|
-
return result;
|
|
132
|
-
}, [])
|
|
133
|
-
: lines;
|
|
134
|
-
}
|
|
135
|
-
exports.splitLines = splitLines;
|
|
136
124
|
function undefIfEmpty(input) {
|
|
137
125
|
return input.length ? input : undefined;
|
|
138
126
|
}
|
package/utils/tar.d.ts
CHANGED
|
@@ -24,14 +24,17 @@ export type DecompressOptions = {
|
|
|
24
24
|
*/
|
|
25
25
|
cores?: CoresOptions;
|
|
26
26
|
};
|
|
27
|
-
export
|
|
27
|
+
export type CreateTarOptions = {
|
|
28
28
|
path: string;
|
|
29
29
|
verbose?: boolean;
|
|
30
30
|
output: string;
|
|
31
|
-
includeList: string;
|
|
32
31
|
compress?: boolean | CompressOptions;
|
|
33
32
|
onEntry?: (entry: TarEntry) => void;
|
|
34
|
-
}
|
|
33
|
+
} & ({
|
|
34
|
+
includeList: string;
|
|
35
|
+
} | {
|
|
36
|
+
include: string[];
|
|
37
|
+
});
|
|
35
38
|
export interface ExtractOptions {
|
|
36
39
|
input: string;
|
|
37
40
|
output: string;
|
|
@@ -51,4 +54,8 @@ export type ListTarOptions = {
|
|
|
51
54
|
export declare function listTar(options: ListTarOptions): Promise<number>;
|
|
52
55
|
export declare function checkPigzLib(cache?: boolean): Promise<boolean>;
|
|
53
56
|
export declare function createTar(options: CreateTarOptions): Promise<void>;
|
|
57
|
+
/**
|
|
58
|
+
* bsdtar (only windows?) fails if path ends with slash
|
|
59
|
+
*/
|
|
60
|
+
export declare function normalizeTarPath(path: string): string;
|
|
54
61
|
export declare function extractTar(options: ExtractOptions): Promise<void>;
|
package/utils/tar.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.extractTar = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
|
|
3
|
+
exports.extractTar = exports.normalizeTarPath = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
|
|
4
4
|
const cli_1 = require("./cli");
|
|
5
5
|
const fs_1 = require("./fs");
|
|
6
6
|
const math_1 = require("./math");
|
|
@@ -52,13 +52,18 @@ async function getTarVendor(cache = true, log = false) {
|
|
|
52
52
|
}
|
|
53
53
|
exports.getTarVendor = getTarVendor;
|
|
54
54
|
async function listTar(options) {
|
|
55
|
+
const vendor = await getTarVendor(true, options.verbose);
|
|
55
56
|
let total = 0;
|
|
56
|
-
await (0, process_1.exec)("tar", [
|
|
57
|
+
await (0, process_1.exec)("tar", [
|
|
58
|
+
"-tf",
|
|
59
|
+
toLocalPath(options.input),
|
|
60
|
+
...(vendor === "bsdtar" ? [] : ["--force-local"]),
|
|
61
|
+
], {}, {
|
|
57
62
|
log: options.verbose,
|
|
58
63
|
stdout: {
|
|
59
64
|
parseLines: "skip-empty",
|
|
60
65
|
onData: (path) => {
|
|
61
|
-
options.onEntry?.({ path });
|
|
66
|
+
options.onEntry?.({ path: normalizeTarPath(path) });
|
|
62
67
|
total++;
|
|
63
68
|
},
|
|
64
69
|
},
|
|
@@ -91,7 +96,9 @@ async function ifX(input, cb) {
|
|
|
91
96
|
}
|
|
92
97
|
async function createTar(options) {
|
|
93
98
|
const vendor = await getTarVendor(true, options.verbose);
|
|
94
|
-
const total =
|
|
99
|
+
const total = "include" in options
|
|
100
|
+
? options.include.length
|
|
101
|
+
: await (0, fs_1.countFileLines)(options.includeList);
|
|
95
102
|
const compress = await ifX(options.compress, async (compress) => ({
|
|
96
103
|
...compress,
|
|
97
104
|
cores: await resolveCores(compress.cores),
|
|
@@ -103,16 +110,27 @@ async function createTar(options) {
|
|
|
103
110
|
GZIP_OPT: compress.level.toString(),
|
|
104
111
|
}),
|
|
105
112
|
};
|
|
113
|
+
const onData = (line) => {
|
|
114
|
+
current++;
|
|
115
|
+
let path = vendor === "bsdtar" ? line.slice(2) : line;
|
|
116
|
+
options.onEntry?.({
|
|
117
|
+
path: normalizeTarPath(path),
|
|
118
|
+
progress: {
|
|
119
|
+
total,
|
|
120
|
+
current,
|
|
121
|
+
percent: (0, math_1.progressPercent)(total, current),
|
|
122
|
+
},
|
|
123
|
+
});
|
|
124
|
+
};
|
|
106
125
|
await (0, process_1.exec)("tar", [
|
|
107
126
|
"--no-recursion",
|
|
108
127
|
"-C",
|
|
109
128
|
toLocalPath(options.path),
|
|
110
129
|
compress?.cores === 1 ? "-czvf" : "-cvf",
|
|
111
130
|
toLocalPath(options.output),
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
"--
|
|
115
|
-
"--force-local",
|
|
131
|
+
// https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=172293
|
|
132
|
+
...(vendor === "bsdtar" ? [] : ["--ignore-failed-read"]),
|
|
133
|
+
...(vendor === "bsdtar" ? [] : ["--force-local"]),
|
|
116
134
|
...(compress && compress.cores > 1
|
|
117
135
|
? [
|
|
118
136
|
"-I",
|
|
@@ -126,6 +144,9 @@ async function createTar(options) {
|
|
|
126
144
|
.join(" ")}"`,
|
|
127
145
|
]
|
|
128
146
|
: []),
|
|
147
|
+
...("includeList" in options
|
|
148
|
+
? ["-T", toLocalPath(options.includeList)]
|
|
149
|
+
: ["--", ...options.include]),
|
|
129
150
|
], {
|
|
130
151
|
...(compress &&
|
|
131
152
|
compress.cores > 1 && {
|
|
@@ -136,25 +157,19 @@ async function createTar(options) {
|
|
|
136
157
|
...env,
|
|
137
158
|
},
|
|
138
159
|
}, {
|
|
139
|
-
log: options.verbose
|
|
140
|
-
|
|
141
|
-
|
|
160
|
+
log: options.verbose
|
|
161
|
+
? { envNames: Object.keys(env), exec: true, stderr: true, stdout: true }
|
|
162
|
+
: false,
|
|
163
|
+
...(vendor === "bsdtar"
|
|
142
164
|
? {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
const path = vendor === "bsdtar" ? line.slice(2) : line;
|
|
147
|
-
options.onEntry?.({
|
|
148
|
-
path,
|
|
149
|
-
progress: {
|
|
150
|
-
total,
|
|
151
|
-
current,
|
|
152
|
-
percent: (0, math_1.progressPercent)(total, current),
|
|
153
|
-
},
|
|
154
|
-
});
|
|
155
|
-
},
|
|
165
|
+
stderr: options.onEntry
|
|
166
|
+
? { toExitCode: true, parseLines: "skip-empty", onData }
|
|
167
|
+
: { toExitCode: true },
|
|
156
168
|
}
|
|
157
|
-
:
|
|
169
|
+
: {
|
|
170
|
+
stderr: { toExitCode: true },
|
|
171
|
+
stdout: { parseLines: "skip-empty", onData },
|
|
172
|
+
}),
|
|
158
173
|
});
|
|
159
174
|
}
|
|
160
175
|
exports.createTar = createTar;
|
|
@@ -164,9 +179,18 @@ exports.createTar = createTar;
|
|
|
164
179
|
function toLocalPath(path) {
|
|
165
180
|
return (0, os_1.platform)() === "win32" ? path.replace(/\\/g, "/") : path;
|
|
166
181
|
}
|
|
182
|
+
/**
|
|
183
|
+
* bsdtar (only windows?) fails if path ends with slash
|
|
184
|
+
*/
|
|
185
|
+
function normalizeTarPath(path) {
|
|
186
|
+
return path.endsWith("/") ? path.slice(0, -1) : path;
|
|
187
|
+
}
|
|
188
|
+
exports.normalizeTarPath = normalizeTarPath;
|
|
167
189
|
async function extractTar(options) {
|
|
168
|
-
let total = options.
|
|
169
|
-
|
|
190
|
+
let total = options.onEntry
|
|
191
|
+
? options.total ??
|
|
192
|
+
(await listTar({ input: options.input, verbose: options.verbose }))
|
|
193
|
+
: undefined;
|
|
170
194
|
if (!(await (0, fs_1.existsDir)(options.output))) {
|
|
171
195
|
if (options.verbose)
|
|
172
196
|
(0, cli_1.logExec)("mkdir", ["-p", options.output]);
|
|
@@ -177,12 +201,25 @@ async function extractTar(options) {
|
|
|
177
201
|
cores: await resolveCores(decompress.cores),
|
|
178
202
|
}));
|
|
179
203
|
let current = 0;
|
|
204
|
+
const vendor = await getTarVendor(true, options.verbose);
|
|
205
|
+
const onData = (line) => {
|
|
206
|
+
const path = vendor === "bsdtar" ? line.slice(2) : line;
|
|
207
|
+
current++;
|
|
208
|
+
options.onEntry?.({
|
|
209
|
+
path: normalizeTarPath(path),
|
|
210
|
+
progress: {
|
|
211
|
+
total: total,
|
|
212
|
+
current,
|
|
213
|
+
percent: (0, math_1.progressPercent)(total, current),
|
|
214
|
+
},
|
|
215
|
+
});
|
|
216
|
+
};
|
|
180
217
|
await (0, process_1.exec)("tar", [
|
|
181
218
|
decompress?.cores === 1 ? "-xzvpf" : "-xvpf",
|
|
182
219
|
toLocalPath(options.input),
|
|
183
220
|
"-C",
|
|
184
221
|
toLocalPath(options.output),
|
|
185
|
-
"--force-local",
|
|
222
|
+
...(vendor === "bsdtar" ? [] : ["--force-local"]),
|
|
186
223
|
...(decompress && decompress.cores > 1
|
|
187
224
|
? ["-I", `"pigz -p ${decompress.cores}"`]
|
|
188
225
|
: []),
|
|
@@ -193,25 +230,16 @@ async function extractTar(options) {
|
|
|
193
230
|
}),
|
|
194
231
|
}, {
|
|
195
232
|
log: options.verbose,
|
|
196
|
-
|
|
197
|
-
toExitCode: true,
|
|
198
|
-
},
|
|
199
|
-
stdout: options.onEntry
|
|
233
|
+
...(vendor === "bsdtar"
|
|
200
234
|
? {
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
options.onEntry?.({
|
|
205
|
-
path,
|
|
206
|
-
progress: {
|
|
207
|
-
total,
|
|
208
|
-
current,
|
|
209
|
-
percent: (0, math_1.progressPercent)(total, current),
|
|
210
|
-
},
|
|
211
|
-
});
|
|
212
|
-
},
|
|
235
|
+
stderr: options.onEntry
|
|
236
|
+
? { toExitCode: true, parseLines: "skip-empty", onData }
|
|
237
|
+
: { toExitCode: true },
|
|
213
238
|
}
|
|
214
|
-
:
|
|
239
|
+
: {
|
|
240
|
+
stderr: { toExitCode: true },
|
|
241
|
+
stdout: { parseLines: "skip-empty", onData },
|
|
242
|
+
}),
|
|
215
243
|
});
|
|
216
244
|
}
|
|
217
245
|
exports.extractTar = extractTar;
|
package/utils/temp.d.ts
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
export declare function parentTmpDir(): string;
|
|
3
|
+
export declare function sessionTmpDir(): string;
|
|
4
|
+
export declare function ensureFreeDiskTempSpace(size: number | string): Promise<void>;
|
|
5
|
+
export declare function isTmpDir(path: string): boolean;
|
|
6
|
+
export declare function rmTmpDir(input: string | string[]): Promise<void>;
|
|
7
|
+
export declare function tmpDir(...keys: [string, ...string[]]): string;
|
|
8
|
+
export declare function mkTmpDir(...keys: [string, ...string[]]): Promise<string>;
|
|
9
|
+
export declare function useTempDir(...keys: [string, ...string[]]): Promise<AsyncDisposable & {
|
|
10
|
+
path: string;
|
|
11
|
+
}>;
|
|
12
|
+
export declare function useTempFile(path: string): AsyncDisposable & {
|
|
13
|
+
path: string;
|
|
14
|
+
};
|
|
15
|
+
export declare class CleanupListener {
|
|
16
|
+
readonly paths: string[];
|
|
17
|
+
stop(): void;
|
|
18
|
+
dispose(): Promise<void>;
|
|
19
|
+
}
|
|
20
|
+
export declare class GargabeCollector {
|
|
21
|
+
protected listeners: Set<CleanupListener>;
|
|
22
|
+
get pending(): boolean;
|
|
23
|
+
cleanup(cb?: () => any): Promise<void>;
|
|
24
|
+
cleanupOnFinish(cb: () => any): Promise<void>;
|
|
25
|
+
cleanupIfFail(cb: () => any): Promise<CleanupListener>;
|
|
26
|
+
}
|
package/utils/temp.js
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.GargabeCollector = exports.CleanupListener = exports.useTempFile = exports.useTempDir = exports.mkTmpDir = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.ensureFreeDiskTempSpace = exports.sessionTmpDir = exports.parentTmpDir = void 0;
|
|
7
|
+
const globalData_1 = __importDefault(require("../globalData"));
|
|
8
|
+
const fs_1 = require("./fs");
|
|
9
|
+
const crypto_1 = require("crypto");
|
|
10
|
+
const promises_1 = require("fs/promises");
|
|
11
|
+
const path_1 = require("path");
|
|
12
|
+
Symbol.dispose ??= Symbol("Symbol.dispose");
|
|
13
|
+
Symbol.asyncDispose ??= Symbol("Symbol.asyncDispose");
|
|
14
|
+
function parentTmpDir() {
|
|
15
|
+
return (0, path_1.join)(globalData_1.default.tempDir, "datatruck-temp");
|
|
16
|
+
}
|
|
17
|
+
exports.parentTmpDir = parentTmpDir;
|
|
18
|
+
function sessionTmpDir() {
|
|
19
|
+
return (0, path_1.join)(parentTmpDir(), process.pid.toString());
|
|
20
|
+
}
|
|
21
|
+
exports.sessionTmpDir = sessionTmpDir;
|
|
22
|
+
async function ensureFreeDiskTempSpace(size) {
|
|
23
|
+
const path = sessionTmpDir();
|
|
24
|
+
await (0, fs_1.mkdirIfNotExists)(sessionTmpDir());
|
|
25
|
+
await (0, fs_1.ensureFreeDiskSpace)([path], size);
|
|
26
|
+
}
|
|
27
|
+
exports.ensureFreeDiskTempSpace = ensureFreeDiskTempSpace;
|
|
28
|
+
function isTmpDir(path) {
|
|
29
|
+
return path.startsWith(sessionTmpDir()) && path.includes("datatruck-temp");
|
|
30
|
+
}
|
|
31
|
+
exports.isTmpDir = isTmpDir;
|
|
32
|
+
async function rmTmpDir(input) {
|
|
33
|
+
if (typeof input === "string") {
|
|
34
|
+
if (!isTmpDir(input))
|
|
35
|
+
throw new Error(`Path is not a temp dir: ${input}`);
|
|
36
|
+
await (0, promises_1.rm)(input, { recursive: true });
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
for (const path of input)
|
|
40
|
+
await rmTmpDir(path);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.rmTmpDir = rmTmpDir;
|
|
44
|
+
function tmpDir(...keys) {
|
|
45
|
+
const id = (0, crypto_1.randomUUID)().slice(0, 8);
|
|
46
|
+
const path = (0, path_1.join)(sessionTmpDir(), [...keys, id].map(encodeURIComponent).join("-"));
|
|
47
|
+
for (const listener of listeners)
|
|
48
|
+
listener.paths.push(path);
|
|
49
|
+
return path;
|
|
50
|
+
}
|
|
51
|
+
exports.tmpDir = tmpDir;
|
|
52
|
+
const listeners = new Set();
|
|
53
|
+
async function mkTmpDir(...keys) {
|
|
54
|
+
const path = tmpDir(...keys);
|
|
55
|
+
await (0, promises_1.mkdir)(path, { recursive: true });
|
|
56
|
+
return path;
|
|
57
|
+
}
|
|
58
|
+
exports.mkTmpDir = mkTmpDir;
|
|
59
|
+
async function useTempDir(...keys) {
|
|
60
|
+
const path = await mkTmpDir(...keys);
|
|
61
|
+
return {
|
|
62
|
+
path,
|
|
63
|
+
async [Symbol.asyncDispose]() {
|
|
64
|
+
try {
|
|
65
|
+
await rmTmpDir(path);
|
|
66
|
+
}
|
|
67
|
+
catch (_) { }
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
exports.useTempDir = useTempDir;
|
|
72
|
+
function useTempFile(path) {
|
|
73
|
+
return {
|
|
74
|
+
path,
|
|
75
|
+
async [Symbol.asyncDispose]() {
|
|
76
|
+
try {
|
|
77
|
+
await (0, promises_1.rm)(path, { recursive: true });
|
|
78
|
+
}
|
|
79
|
+
catch (_) { }
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
exports.useTempFile = useTempFile;
|
|
84
|
+
class CleanupListener {
|
|
85
|
+
paths = [];
|
|
86
|
+
stop() {
|
|
87
|
+
listeners.delete(this);
|
|
88
|
+
}
|
|
89
|
+
async dispose() {
|
|
90
|
+
this.stop();
|
|
91
|
+
await rmTmpDir(this.paths);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
exports.CleanupListener = CleanupListener;
|
|
95
|
+
class GargabeCollector {
|
|
96
|
+
listeners = new Set();
|
|
97
|
+
get pending() {
|
|
98
|
+
return this.listeners.size > 0;
|
|
99
|
+
}
|
|
100
|
+
async cleanup(cb) {
|
|
101
|
+
try {
|
|
102
|
+
await cb?.();
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
for (const listener of this.listeners) {
|
|
106
|
+
this.listeners.delete(listener);
|
|
107
|
+
await listener.dispose();
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
async cleanupOnFinish(cb) {
|
|
112
|
+
const cleanup = new CleanupListener();
|
|
113
|
+
try {
|
|
114
|
+
await cb();
|
|
115
|
+
}
|
|
116
|
+
finally {
|
|
117
|
+
cleanup.dispose();
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
async cleanupIfFail(cb) {
|
|
121
|
+
const cleanup = new CleanupListener();
|
|
122
|
+
try {
|
|
123
|
+
await cb();
|
|
124
|
+
}
|
|
125
|
+
catch (error) {
|
|
126
|
+
await cleanup.dispose();
|
|
127
|
+
throw error;
|
|
128
|
+
}
|
|
129
|
+
this.listeners.add(cleanup);
|
|
130
|
+
return cleanup;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
exports.GargabeCollector = GargabeCollector;
|
package/utils/virtual-fs.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { DiskStats } from "./fs";
|
|
1
2
|
export declare function resolvePath(path: string): string;
|
|
2
3
|
export type FsOptions = {
|
|
3
4
|
backend: string;
|
|
@@ -7,6 +8,7 @@ export declare abstract class AbstractFs {
|
|
|
7
8
|
constructor(options: FsOptions);
|
|
8
9
|
resolvePath(path: string): string;
|
|
9
10
|
abstract isLocal(): boolean;
|
|
11
|
+
isRemote(): boolean;
|
|
10
12
|
abstract existsDir(path: string): Promise<boolean>;
|
|
11
13
|
abstract mkdir(path: string): Promise<void>;
|
|
12
14
|
abstract readFile(path: string): Promise<string>;
|
|
@@ -15,8 +17,9 @@ export declare abstract class AbstractFs {
|
|
|
15
17
|
abstract readdir(path: string): Promise<string[]>;
|
|
16
18
|
abstract ensureEmptyDir(path: string): Promise<void>;
|
|
17
19
|
abstract writeFile(path: string, contents: string): Promise<void>;
|
|
18
|
-
abstract upload(
|
|
19
|
-
abstract download(
|
|
20
|
+
abstract upload(source: string, target: string): Promise<void>;
|
|
21
|
+
abstract download(source: string, target: string): Promise<void>;
|
|
22
|
+
abstract fetchDiskStats(source: string): Promise<DiskStats>;
|
|
20
23
|
}
|
|
21
24
|
export declare class LocalFs extends AbstractFs {
|
|
22
25
|
isLocal(): boolean;
|
|
@@ -28,6 +31,7 @@ export declare class LocalFs extends AbstractFs {
|
|
|
28
31
|
readdir(path: string): Promise<string[]>;
|
|
29
32
|
writeFile(path: string, contents: string): Promise<void>;
|
|
30
33
|
rmAll(path: string): Promise<void>;
|
|
34
|
+
fetchDiskStats(source: string): Promise<DiskStats>;
|
|
31
35
|
upload(source: string, target: string): Promise<void>;
|
|
32
36
|
download(source: string, target: string): Promise<void>;
|
|
33
37
|
}
|