@datatruck/cli 0.26.2 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Action/BackupAction.d.ts +69 -34
- package/Action/BackupAction.js +284 -244
- package/Action/CleanCacheAction.d.ts +8 -4
- package/Action/CleanCacheAction.js +8 -5
- package/Action/ConfigAction.d.ts +12 -5
- package/Action/ConfigAction.js +14 -18
- package/Action/CopyAction.d.ts +49 -0
- package/Action/CopyAction.js +144 -0
- package/Action/InitAction.d.ts +3 -3
- package/Action/InitAction.js +9 -9
- package/Action/PruneAction.d.ts +9 -9
- package/Action/PruneAction.js +39 -23
- package/Action/RestoreAction.d.ts +48 -23
- package/Action/RestoreAction.js +158 -195
- package/Action/SnapshotsAction.d.ts +8 -8
- package/Action/SnapshotsAction.js +8 -8
- package/CHANGELOG.md +495 -0
- package/Command/BackupCommand.d.ts +6 -4
- package/Command/BackupCommand.js +9 -26
- package/Command/CleanCacheCommand.d.ts +4 -4
- package/Command/CleanCacheCommand.js +26 -5
- package/Command/CommandAbstract.d.ts +10 -7
- package/Command/CommandAbstract.js +4 -1
- package/Command/ConfigCommand.d.ts +6 -9
- package/Command/ConfigCommand.js +13 -8
- package/Command/CopyCommand.d.ts +15 -0
- package/Command/CopyCommand.js +61 -0
- package/Command/InitCommand.d.ts +4 -4
- package/Command/InitCommand.js +11 -15
- package/Command/PruneCommand.d.ts +3 -3
- package/Command/PruneCommand.js +13 -12
- package/Command/RestoreCommand.js +9 -17
- package/Command/SnapshotsCommand.d.ts +4 -4
- package/Command/SnapshotsCommand.js +16 -15
- package/Command/StartServerCommand.d.ts +6 -0
- package/Command/StartServerCommand.js +24 -0
- package/Config/Config.d.ts +11 -0
- package/Config/Config.js +43 -0
- package/Config/PrunePolicyConfig.d.ts +2 -2
- package/Factory/CommandFactory.d.ts +29 -33
- package/Factory/CommandFactory.js +32 -53
- package/Factory/RepositoryFactory.d.ts +1 -1
- package/Factory/RepositoryFactory.js +3 -3
- package/Factory/TaskFactory.d.ts +1 -1
- package/Factory/TaskFactory.js +3 -3
- package/Repository/DatatruckRepository.d.ts +17 -16
- package/Repository/DatatruckRepository.js +131 -149
- package/Repository/GitRepository.d.ts +9 -8
- package/Repository/GitRepository.js +22 -25
- package/Repository/RepositoryAbstract.d.ts +39 -37
- package/Repository/RepositoryAbstract.js +4 -5
- package/Repository/ResticRepository.d.ts +9 -8
- package/Repository/ResticRepository.js +30 -28
- package/Task/GitTask.d.ts +6 -7
- package/Task/GitTask.js +24 -30
- package/Task/MariadbTask.d.ts +4 -5
- package/Task/MariadbTask.js +26 -32
- package/Task/MssqlTask.d.ts +5 -3
- package/Task/MssqlTask.js +11 -12
- package/Task/MysqlDumpTask.d.ts +10 -3
- package/Task/MysqlDumpTask.js +107 -31
- package/Task/ScriptTask.d.ts +23 -18
- package/Task/ScriptTask.js +34 -24
- package/Task/SqlDumpTaskAbstract.d.ts +8 -3
- package/Task/SqlDumpTaskAbstract.js +31 -19
- package/Task/TaskAbstract.d.ts +24 -25
- package/Task/TaskAbstract.js +6 -10
- package/cli.js +14 -5
- package/config.schema.json +124 -3
- package/package.json +4 -5
- package/utils/DataFormat.d.ts +23 -12
- package/utils/DataFormat.js +36 -14
- package/utils/cli.d.ts +2 -9
- package/utils/cli.js +9 -52
- package/utils/datatruck/client.d.ts +24 -0
- package/utils/datatruck/client.js +99 -0
- package/utils/datatruck/config.d.ts +8 -6
- package/utils/datatruck/config.js +18 -3
- package/utils/datatruck/paths.d.ts +5 -9
- package/utils/datatruck/paths.js +2 -2
- package/utils/datatruck/server.d.ts +21 -0
- package/utils/datatruck/server.js +96 -0
- package/utils/datatruck/snapshot.d.ts +2 -2
- package/utils/date.d.ts +7 -3
- package/utils/date.js +22 -14
- package/utils/fs.d.ts +27 -15
- package/utils/fs.js +110 -62
- package/utils/http.d.ts +21 -0
- package/utils/http.js +154 -0
- package/utils/list.d.ts +64 -0
- package/utils/list.js +145 -0
- package/utils/mysql.d.ts +2 -0
- package/utils/mysql.js +21 -2
- package/utils/process.d.ts +1 -0
- package/utils/process.js +24 -31
- package/utils/progress.d.ts +33 -0
- package/utils/progress.js +113 -0
- package/utils/steps.d.ts +11 -0
- package/utils/steps.js +22 -10
- package/utils/stream.d.ts +7 -0
- package/utils/stream.js +10 -0
- package/utils/string.d.ts +0 -1
- package/utils/string.js +1 -13
- package/utils/tar.d.ts +10 -3
- package/utils/tar.js +70 -44
- package/utils/temp.d.ts +26 -0
- package/utils/temp.js +133 -0
- package/utils/virtual-fs.d.ts +37 -0
- package/utils/virtual-fs.js +65 -0
- package/Action/BackupSessionsAction.d.ts +0 -13
- package/Action/BackupSessionsAction.js +0 -18
- package/Action/RestoreSessionsAction.d.ts +0 -13
- package/Action/RestoreSessionsAction.js +0 -18
- package/Command/BackupSessionsCommand.d.ts +0 -12
- package/Command/BackupSessionsCommand.js +0 -92
- package/Command/RestoreSessionsCommand.d.ts +0 -12
- package/Command/RestoreSessionsCommand.js +0 -91
- package/Decorator/EntityDecorator.d.ts +0 -11
- package/Decorator/EntityDecorator.js +0 -17
- package/Entity/BackupSessionEntity.d.ts +0 -6
- package/Entity/BackupSessionEntity.js +0 -25
- package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
- package/Entity/BackupSessionRepositoryEntity.js +0 -25
- package/Entity/BackupSessionTaskEntity.d.ts +0 -5
- package/Entity/BackupSessionTaskEntity.js +0 -24
- package/Entity/CrudEntityAbstract.d.ts +0 -5
- package/Entity/CrudEntityAbstract.js +0 -9
- package/Entity/RestoreSessionEntity.d.ts +0 -5
- package/Entity/RestoreSessionEntity.js +0 -24
- package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
- package/Entity/RestoreSessionRepositoryEntity.js +0 -25
- package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
- package/Entity/RestoreSessionTaskEntity.js +0 -24
- package/Entity/StateEntityAbstract.d.ts +0 -9
- package/Entity/StateEntityAbstract.js +0 -12
- package/Factory/EntityFactory.d.ts +0 -6
- package/Factory/EntityFactory.js +0 -40
- package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
- package/SessionDriver/ConsoleSessionDriver.js +0 -208
- package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
- package/SessionDriver/SessionDriverAbstract.js +0 -28
- package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
- package/SessionDriver/SqliteSessionDriver.js +0 -173
- package/SessionManager/BackupSessionManager.d.ts +0 -45
- package/SessionManager/BackupSessionManager.js +0 -218
- package/SessionManager/RestoreSessionManager.d.ts +0 -47
- package/SessionManager/RestoreSessionManager.js +0 -218
- package/SessionManager/SessionManagerAbstract.d.ts +0 -18
- package/SessionManager/SessionManagerAbstract.js +0 -36
- package/migrations/001-initial.sql +0 -98
- package/utils/entity.d.ts +0 -4
- package/utils/entity.js +0 -10
package/utils/date.js
CHANGED
|
@@ -3,13 +3,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.
|
|
7
|
-
const string_1 = require("./string");
|
|
6
|
+
exports.duration = exports.createTimer = exports.filterByLast = void 0;
|
|
8
7
|
const dayjs_1 = __importDefault(require("dayjs"));
|
|
9
8
|
const advancedFormat_1 = __importDefault(require("dayjs/plugin/advancedFormat"));
|
|
9
|
+
const duration_1 = __importDefault(require("dayjs/plugin/duration"));
|
|
10
10
|
const isoWeek_1 = __importDefault(require("dayjs/plugin/isoWeek"));
|
|
11
11
|
dayjs_1.default.extend(isoWeek_1.default);
|
|
12
12
|
dayjs_1.default.extend(advancedFormat_1.default);
|
|
13
|
+
dayjs_1.default.extend(duration_1.default);
|
|
13
14
|
function filterByLast(items, options, reasons) {
|
|
14
15
|
const filters = {
|
|
15
16
|
last: { handler: (_, i) => i.toString(), value: options.last },
|
|
@@ -73,18 +74,25 @@ function filterByLast(items, options, reasons) {
|
|
|
73
74
|
return items.filter((item) => validItems.includes(item));
|
|
74
75
|
}
|
|
75
76
|
exports.filterByLast = filterByLast;
|
|
76
|
-
function
|
|
77
|
-
let startTime;
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
elapsed: (
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
if (
|
|
85
|
-
return
|
|
86
|
-
|
|
77
|
+
function createTimer() {
|
|
78
|
+
let startTime = Date.now();
|
|
79
|
+
let endTime;
|
|
80
|
+
const timer = {
|
|
81
|
+
elapsed: () => (endTime || Date.now()) - startTime,
|
|
82
|
+
check: (ms) => timer.elapsed() > ms,
|
|
83
|
+
stop: () => (endTime = Date.now()),
|
|
84
|
+
reset: (min) => {
|
|
85
|
+
if (typeof min === "number" && !timer.check(min))
|
|
86
|
+
return false;
|
|
87
|
+
startTime = Date.now();
|
|
88
|
+
endTime = undefined;
|
|
89
|
+
return true;
|
|
87
90
|
},
|
|
88
91
|
};
|
|
92
|
+
return timer;
|
|
89
93
|
}
|
|
90
|
-
exports.
|
|
94
|
+
exports.createTimer = createTimer;
|
|
95
|
+
function duration(ms) {
|
|
96
|
+
return dayjs_1.default.duration(ms, "ms").format("HH:mm:ss");
|
|
97
|
+
}
|
|
98
|
+
exports.duration = duration;
|
package/utils/fs.d.ts
CHANGED
|
@@ -11,6 +11,8 @@ export declare function isEmptyDir(path: string): Promise<boolean>;
|
|
|
11
11
|
export declare function isLocalDir(path: string): boolean;
|
|
12
12
|
export declare function mkdirIfNotExists(path: string): Promise<string>;
|
|
13
13
|
export declare function ensureEmptyDir(path: string): Promise<void>;
|
|
14
|
+
export declare function ensureSingleFile(path: string): Promise<string>;
|
|
15
|
+
export declare function ensureExistsDir(path: string): Promise<void>;
|
|
14
16
|
export declare function safeStat(path: string): Promise<Stats | undefined>;
|
|
15
17
|
export declare function existsDir(path: string): Promise<boolean>;
|
|
16
18
|
export declare function existsFile(path: string): Promise<boolean>;
|
|
@@ -23,13 +25,7 @@ export declare function parsePackageFile(): {
|
|
|
23
25
|
description: string;
|
|
24
26
|
};
|
|
25
27
|
export declare function findFile(sourcePath: string, baseName: string, extensions: string[], errorMessage?: string): Promise<string>;
|
|
26
|
-
export declare function parentTmpDir(): string;
|
|
27
|
-
export declare function sessionTmpDir(): string;
|
|
28
|
-
export declare function isTmpDir(path: string): boolean;
|
|
29
|
-
export declare function rmTmpDir(input: string | string[]): Promise<void>;
|
|
30
|
-
export declare function tmpDir(prefix: string, id?: string): string;
|
|
31
28
|
export declare function fastFolderSizeAsync(path: string): Promise<number>;
|
|
32
|
-
export declare function mkTmpDir(prefix: string, id?: string): Promise<string>;
|
|
33
29
|
export declare function readPartialFile(path: string, positions: [number, number?]): Promise<string>;
|
|
34
30
|
export declare function readDir(path: string, optional?: boolean): Promise<string[]>;
|
|
35
31
|
export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
|
|
@@ -60,7 +56,7 @@ export declare function cpy(options: {
|
|
|
60
56
|
path: string;
|
|
61
57
|
basePath: string;
|
|
62
58
|
};
|
|
63
|
-
|
|
59
|
+
outPath: string;
|
|
64
60
|
/**
|
|
65
61
|
* @default 1
|
|
66
62
|
*/
|
|
@@ -87,18 +83,24 @@ export declare function cpy(options: {
|
|
|
87
83
|
files: number;
|
|
88
84
|
dirs: number;
|
|
89
85
|
}>;
|
|
86
|
+
type ProgressObject = {
|
|
87
|
+
disposed: boolean;
|
|
88
|
+
total: number;
|
|
89
|
+
current: number;
|
|
90
|
+
update: (description: string, path?: string, increment?: boolean) => void;
|
|
91
|
+
};
|
|
92
|
+
export declare function createProgress(options: {
|
|
93
|
+
onProgress: (data: Progress) => void;
|
|
94
|
+
}): ProgressObject;
|
|
90
95
|
export declare function createFileScanner(options: {
|
|
91
96
|
glob: Options & {
|
|
92
97
|
include: string[];
|
|
93
98
|
};
|
|
94
|
-
onProgress: (data: Progress) =>
|
|
95
|
-
}): Promise<{
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
progress: (description: string, path?: string, increment?: boolean) => Promise<void>;
|
|
100
|
-
end: () => Promise<void>;
|
|
101
|
-
start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
|
|
99
|
+
onProgress: (data: Progress) => void;
|
|
100
|
+
}): Promise<ProgressObject & {
|
|
101
|
+
progress: ProgressObject["update"];
|
|
102
|
+
start: (cb?: (entry: Required<Entry>) => any) => Promise<void>;
|
|
103
|
+
end: () => void;
|
|
102
104
|
}>;
|
|
103
105
|
type StreamItem = {
|
|
104
106
|
key: string;
|
|
@@ -121,4 +123,14 @@ export declare function createWriteStreamPool(options: {
|
|
|
121
123
|
export declare function countFileLines(path: string): Promise<number>;
|
|
122
124
|
export declare function fetchData<T>(input: T, onPath?: (input: Exclude<T, string>) => string | undefined): Promise<string | null>;
|
|
123
125
|
export declare function safeRename(oldPath: string, newPath: string): Promise<void>;
|
|
126
|
+
export declare function tryRm(path: string): Promise<void>;
|
|
127
|
+
export declare function initEmptyDir(path: string | undefined): Promise<string>;
|
|
128
|
+
export type DiskStats = {
|
|
129
|
+
total: number;
|
|
130
|
+
free: number;
|
|
131
|
+
};
|
|
132
|
+
export declare function fetchDiskStats(path: string): Promise<DiskStats>;
|
|
133
|
+
export declare function checkFreeDiskSpace(stat: DiskStats, inSize: string | number): Promise<void>;
|
|
134
|
+
export declare function ensureFreeDiskSpace(input: string[] | DiskStats, inSize: number | string): Promise<void>;
|
|
135
|
+
export declare function groupFiles(inFiles: string[], suffixes?: string[], gzSuffix?: string): [string[], Record<string, string>];
|
|
124
136
|
export {};
|
package/utils/fs.js
CHANGED
|
@@ -3,12 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.
|
|
7
|
-
const globalData_1 = __importDefault(require("../globalData"));
|
|
6
|
+
exports.groupFiles = exports.ensureFreeDiskSpace = exports.checkFreeDiskSpace = exports.fetchDiskStats = exports.initEmptyDir = exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.fastFolderSizeAsync = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureExistsDir = exports.ensureSingleFile = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
|
|
8
7
|
const math_1 = require("./math");
|
|
9
8
|
const path_1 = require("./path");
|
|
9
|
+
const string_1 = require("./string");
|
|
10
10
|
const async_1 = require("async");
|
|
11
|
-
const
|
|
11
|
+
const bytes_1 = __importDefault(require("bytes"));
|
|
12
12
|
const fast_folder_size_1 = __importDefault(require("fast-folder-size"));
|
|
13
13
|
const fast_glob_1 = __importDefault(require("fast-glob"));
|
|
14
14
|
const fs_1 = require("fs");
|
|
@@ -53,6 +53,19 @@ async function ensureEmptyDir(path) {
|
|
|
53
53
|
throw new Error(`Dir is not empty: ${path}`);
|
|
54
54
|
}
|
|
55
55
|
exports.ensureEmptyDir = ensureEmptyDir;
|
|
56
|
+
async function ensureSingleFile(path) {
|
|
57
|
+
const files = await readDir(path);
|
|
58
|
+
if (files.length !== 1)
|
|
59
|
+
throw new Error(`Dir has not one file: ${files.length}`);
|
|
60
|
+
const [file] = files;
|
|
61
|
+
return (0, path_2.join)(path, file);
|
|
62
|
+
}
|
|
63
|
+
exports.ensureSingleFile = ensureSingleFile;
|
|
64
|
+
async function ensureExistsDir(path) {
|
|
65
|
+
if (!(await existsDir(path)))
|
|
66
|
+
throw new Error(`Dir is not crated: ${path}`);
|
|
67
|
+
}
|
|
68
|
+
exports.ensureExistsDir = ensureExistsDir;
|
|
56
69
|
async function safeStat(path) {
|
|
57
70
|
try {
|
|
58
71
|
return await (0, promises_1.stat)(path);
|
|
@@ -116,48 +129,10 @@ async function findFile(sourcePath, baseName, extensions, errorMessage = "Path n
|
|
|
116
129
|
return path;
|
|
117
130
|
}
|
|
118
131
|
exports.findFile = findFile;
|
|
119
|
-
function parentTmpDir() {
|
|
120
|
-
return (0, path_2.join)(globalData_1.default.tempDir, "datatruck-temp");
|
|
121
|
-
}
|
|
122
|
-
exports.parentTmpDir = parentTmpDir;
|
|
123
|
-
function sessionTmpDir() {
|
|
124
|
-
return (0, path_2.join)(parentTmpDir(), process.pid.toString());
|
|
125
|
-
}
|
|
126
|
-
exports.sessionTmpDir = sessionTmpDir;
|
|
127
|
-
function isTmpDir(path) {
|
|
128
|
-
return path.startsWith(sessionTmpDir()) && path.includes("datatruck-temp");
|
|
129
|
-
}
|
|
130
|
-
exports.isTmpDir = isTmpDir;
|
|
131
|
-
async function rmTmpDir(input) {
|
|
132
|
-
if (typeof input === "string") {
|
|
133
|
-
if (!isTmpDir(input))
|
|
134
|
-
throw new Error(`Path is not a temp dir: ${input}`);
|
|
135
|
-
await (0, promises_1.rm)(input, {
|
|
136
|
-
recursive: true,
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
|
-
else {
|
|
140
|
-
for (const path of input)
|
|
141
|
-
await rmTmpDir(path);
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
exports.rmTmpDir = rmTmpDir;
|
|
145
|
-
function tmpDir(prefix, id) {
|
|
146
|
-
if (!id)
|
|
147
|
-
id = (0, crypto_1.randomUUID)().slice(0, 8);
|
|
148
|
-
return (0, path_2.join)(sessionTmpDir(), `${prefix}-${id}`);
|
|
149
|
-
}
|
|
150
|
-
exports.tmpDir = tmpDir;
|
|
151
132
|
async function fastFolderSizeAsync(path) {
|
|
152
133
|
return (await (0, util_1.promisify)(fast_folder_size_1.default)(path)) || 0;
|
|
153
134
|
}
|
|
154
135
|
exports.fastFolderSizeAsync = fastFolderSizeAsync;
|
|
155
|
-
async function mkTmpDir(prefix, id) {
|
|
156
|
-
const path = tmpDir(prefix, id);
|
|
157
|
-
await (0, promises_1.mkdir)(path, { recursive: true });
|
|
158
|
-
return path;
|
|
159
|
-
}
|
|
160
|
-
exports.mkTmpDir = mkTmpDir;
|
|
161
136
|
async function readPartialFile(path, positions) {
|
|
162
137
|
let result = "";
|
|
163
138
|
const statResult = await (0, promises_1.stat)(path);
|
|
@@ -316,7 +291,7 @@ async function cpy(options) {
|
|
|
316
291
|
const isDir = rawEntryPath.endsWith("/");
|
|
317
292
|
const entryPath = (0, path_2.normalize)(rawEntryPath);
|
|
318
293
|
const entrySourcePath = (0, path_2.resolve)((0, path_2.join)(basePath, rawEntryPath));
|
|
319
|
-
const entryTargetPath = (0, path_2.resolve)((0, path_2.join)(options.
|
|
294
|
+
const entryTargetPath = (0, path_2.resolve)((0, path_2.join)(options.outPath, rawEntryPath));
|
|
320
295
|
const onPathResult = await options?.onPath?.({
|
|
321
296
|
isDir,
|
|
322
297
|
entryPath,
|
|
@@ -404,57 +379,65 @@ async function cpy(options) {
|
|
|
404
379
|
return stats;
|
|
405
380
|
}
|
|
406
381
|
exports.cpy = cpy;
|
|
407
|
-
|
|
408
|
-
const
|
|
382
|
+
function createProgress(options) {
|
|
383
|
+
const progress = {
|
|
409
384
|
disposed: false,
|
|
410
385
|
total: 0,
|
|
411
386
|
current: 0,
|
|
412
|
-
|
|
413
|
-
if (
|
|
387
|
+
update: async (description, path, increment = true) => {
|
|
388
|
+
if (progress.disposed)
|
|
414
389
|
return;
|
|
415
390
|
if (path && increment)
|
|
416
|
-
|
|
417
|
-
|
|
391
|
+
progress.current++;
|
|
392
|
+
options.onProgress({
|
|
418
393
|
relative: {
|
|
419
394
|
description,
|
|
420
395
|
payload: path,
|
|
421
396
|
},
|
|
422
397
|
absolute: {
|
|
423
|
-
total:
|
|
424
|
-
current:
|
|
425
|
-
percent: (0, math_1.progressPercent)(
|
|
398
|
+
total: progress.total,
|
|
399
|
+
current: progress.current,
|
|
400
|
+
percent: (0, math_1.progressPercent)(progress.total, progress.current),
|
|
426
401
|
},
|
|
427
402
|
});
|
|
428
403
|
},
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
404
|
+
};
|
|
405
|
+
return progress;
|
|
406
|
+
}
|
|
407
|
+
exports.createProgress = createProgress;
|
|
408
|
+
async function createFileScanner(options) {
|
|
409
|
+
const progress = createProgress(options);
|
|
410
|
+
Object.assign(progress, {
|
|
411
|
+
progress: progress.update,
|
|
412
|
+
end: () => {
|
|
413
|
+
if (!progress.disposed)
|
|
414
|
+
progress.update("Finished");
|
|
415
|
+
progress.disposed = true;
|
|
433
416
|
},
|
|
434
417
|
start: async (cb) => {
|
|
435
418
|
let lastTime = performance.now();
|
|
436
|
-
|
|
419
|
+
progress.update("Scanning files");
|
|
437
420
|
for await (const entry of pathIterator(stream)) {
|
|
438
421
|
if (cb) {
|
|
439
422
|
if (await cb(entry))
|
|
440
|
-
|
|
423
|
+
progress.total++;
|
|
441
424
|
}
|
|
442
425
|
else {
|
|
443
|
-
|
|
426
|
+
progress.total++;
|
|
444
427
|
}
|
|
445
428
|
if (lastTime - performance.now() > 500)
|
|
446
|
-
|
|
429
|
+
progress.update("Scanning files");
|
|
447
430
|
}
|
|
448
|
-
|
|
431
|
+
progress.update("Scanned files");
|
|
449
432
|
},
|
|
450
|
-
};
|
|
433
|
+
});
|
|
451
434
|
const stream = fast_glob_1.default.stream(options.glob.include, {
|
|
452
435
|
dot: true,
|
|
453
436
|
markDirectories: true,
|
|
454
437
|
stats: true,
|
|
455
438
|
...options.glob,
|
|
456
439
|
});
|
|
457
|
-
return
|
|
440
|
+
return progress;
|
|
458
441
|
}
|
|
459
442
|
exports.createFileScanner = createFileScanner;
|
|
460
443
|
function createWriteStreamPool(options) {
|
|
@@ -560,3 +543,68 @@ async function safeRename(oldPath, newPath) {
|
|
|
560
543
|
}
|
|
561
544
|
}
|
|
562
545
|
exports.safeRename = safeRename;
|
|
546
|
+
async function tryRm(path) {
|
|
547
|
+
try {
|
|
548
|
+
await (0, promises_1.rm)(path);
|
|
549
|
+
}
|
|
550
|
+
catch (_) { }
|
|
551
|
+
}
|
|
552
|
+
exports.tryRm = tryRm;
|
|
553
|
+
async function initEmptyDir(path) {
|
|
554
|
+
if (!path)
|
|
555
|
+
throw new Error(`Path is not defined`);
|
|
556
|
+
await mkdirIfNotExists(path);
|
|
557
|
+
await ensureEmptyDir(path);
|
|
558
|
+
return path;
|
|
559
|
+
}
|
|
560
|
+
exports.initEmptyDir = initEmptyDir;
|
|
561
|
+
async function fetchDiskStats(path) {
|
|
562
|
+
const fs = await (0, promises_1.statfs)(path);
|
|
563
|
+
const total = fs.bsize * fs.blocks;
|
|
564
|
+
const free = fs.bsize * fs.bavail;
|
|
565
|
+
return { total, free };
|
|
566
|
+
}
|
|
567
|
+
exports.fetchDiskStats = fetchDiskStats;
|
|
568
|
+
async function checkFreeDiskSpace(stat, inSize) {
|
|
569
|
+
const humanSize = typeof inSize === "number" ? (0, bytes_1.default)(inSize) : inSize;
|
|
570
|
+
const size = bytes_1.default.parse(inSize);
|
|
571
|
+
if (stat.free < size)
|
|
572
|
+
throw new Error(`Free disk space is less than ${humanSize}: ${(0, bytes_1.default)(stat.free)}/${(0, bytes_1.default)(stat.total)}`);
|
|
573
|
+
}
|
|
574
|
+
exports.checkFreeDiskSpace = checkFreeDiskSpace;
|
|
575
|
+
async function ensureFreeDiskSpace(input, inSize) {
|
|
576
|
+
if (Array.isArray(input)) {
|
|
577
|
+
for (const path of input) {
|
|
578
|
+
const stat = await fetchDiskStats(path);
|
|
579
|
+
await checkFreeDiskSpace(stat, inSize);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
else {
|
|
583
|
+
await checkFreeDiskSpace(input, inSize);
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
exports.ensureFreeDiskSpace = ensureFreeDiskSpace;
|
|
587
|
+
function groupFiles(inFiles, suffixes, gzSuffix = ".tar.gz") {
|
|
588
|
+
const compressed = {};
|
|
589
|
+
if (suffixes) {
|
|
590
|
+
const validGzSuffixes = suffixes.map((f) => `${f}${gzSuffix}`);
|
|
591
|
+
inFiles = inFiles.filter((f) => (0, string_1.endsWith)(f, suffixes) || (0, string_1.endsWith)(f, validGzSuffixes));
|
|
592
|
+
}
|
|
593
|
+
const grouped = inFiles.reduce((items, name) => {
|
|
594
|
+
const key = name.endsWith(gzSuffix)
|
|
595
|
+
? name.slice(0, -gzSuffix.length)
|
|
596
|
+
: name;
|
|
597
|
+
if (!items[key])
|
|
598
|
+
items[key] = [];
|
|
599
|
+
items[key].push(name);
|
|
600
|
+
return items;
|
|
601
|
+
}, {});
|
|
602
|
+
for (const key in grouped) {
|
|
603
|
+
const suffixFile = grouped[key].find((v) => v.endsWith(gzSuffix));
|
|
604
|
+
if (suffixFile) {
|
|
605
|
+
compressed[key] = suffixFile;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
return [Object.keys(grouped), compressed];
|
|
609
|
+
}
|
|
610
|
+
exports.groupFiles = groupFiles;
|
package/utils/http.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { IncomingMessage, Server } from "http";
|
|
3
|
+
export declare function closeServer(server: Server): Promise<void>;
|
|
4
|
+
export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
|
|
5
|
+
export declare function fetchJson<T = any>(url: string, options?: {
|
|
6
|
+
headers?: Record<string, string>;
|
|
7
|
+
query?: Record<string, string>;
|
|
8
|
+
}): Promise<T>;
|
|
9
|
+
export declare function post(url: string, data: string, options?: {
|
|
10
|
+
headers?: Record<string, string>;
|
|
11
|
+
query?: Record<string, string>;
|
|
12
|
+
}): Promise<void>;
|
|
13
|
+
export declare function downloadFile(url: string, output: string, options?: {
|
|
14
|
+
headers?: Record<string, string>;
|
|
15
|
+
query?: Record<string, string>;
|
|
16
|
+
timeout?: number;
|
|
17
|
+
}): Promise<void>;
|
|
18
|
+
export declare function uploadFile(url: string, path: string, options?: {
|
|
19
|
+
headers?: Record<string, string>;
|
|
20
|
+
query?: Record<string, string>;
|
|
21
|
+
}): Promise<void>;
|
package/utils/http.js
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.readRequestData = exports.closeServer = void 0;
|
|
4
|
+
const fs_1 = require("fs");
|
|
5
|
+
const promises_1 = require("fs/promises");
|
|
6
|
+
const http_1 = require("http");
|
|
7
|
+
const https_1 = require("https");
|
|
8
|
+
const request = (url, options, callback) => {
|
|
9
|
+
return url.startsWith("https://")
|
|
10
|
+
? (0, https_1.request)(url, options, callback)
|
|
11
|
+
: (0, http_1.request)(url, options, callback);
|
|
12
|
+
};
|
|
13
|
+
function href(inUrl, query) {
|
|
14
|
+
const url = new URL(inUrl);
|
|
15
|
+
for (const key in query || {})
|
|
16
|
+
url.searchParams.set(key, query[key]);
|
|
17
|
+
return url.href;
|
|
18
|
+
}
|
|
19
|
+
async function closeServer(server) {
|
|
20
|
+
await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
|
|
21
|
+
}
|
|
22
|
+
exports.closeServer = closeServer;
|
|
23
|
+
function readRequestData(req) {
|
|
24
|
+
let data;
|
|
25
|
+
return new Promise((resolve, reject) => {
|
|
26
|
+
req
|
|
27
|
+
.on("error", reject)
|
|
28
|
+
.on("data", (chunk) => {
|
|
29
|
+
if (data === undefined)
|
|
30
|
+
data = "";
|
|
31
|
+
data += chunk;
|
|
32
|
+
})
|
|
33
|
+
.on("close", async () => {
|
|
34
|
+
resolve(data);
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
exports.readRequestData = readRequestData;
|
|
39
|
+
async function fetchJson(url, options = {}) {
|
|
40
|
+
return new Promise((resolve, reject) => {
|
|
41
|
+
let data;
|
|
42
|
+
request(href(url, options.query), {
|
|
43
|
+
method: "GET",
|
|
44
|
+
headers: options.headers,
|
|
45
|
+
}, (res) => {
|
|
46
|
+
if (res.statusCode !== 200)
|
|
47
|
+
return reject(new Error(`GET failed: ${res.statusCode} ${res.statusMessage}`));
|
|
48
|
+
res
|
|
49
|
+
.on("data", (chunk) => {
|
|
50
|
+
if (data === undefined)
|
|
51
|
+
data = "";
|
|
52
|
+
data += chunk;
|
|
53
|
+
})
|
|
54
|
+
.on("error", reject)
|
|
55
|
+
.on("close", () => {
|
|
56
|
+
if (data === undefined) {
|
|
57
|
+
resolve(undefined);
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
try {
|
|
61
|
+
resolve(JSON.parse(data));
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
reject(error);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
})
|
|
69
|
+
.on("error", reject)
|
|
70
|
+
.end();
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
exports.fetchJson = fetchJson;
|
|
74
|
+
async function post(url, data, options = {}) {
|
|
75
|
+
await new Promise((resolve, reject) => {
|
|
76
|
+
const req = request(href(url, options.query), { method: "POST", headers: options.headers }, (res) => {
|
|
77
|
+
res.on("error", reject);
|
|
78
|
+
if (res.statusCode !== 200) {
|
|
79
|
+
reject(new Error(`Post failed: ${res.statusCode} ${res.statusMessage}`));
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
resolve();
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
req.on("error", reject);
|
|
86
|
+
req.write(data);
|
|
87
|
+
req.end();
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
exports.post = post;
|
|
91
|
+
async function downloadFile(url, output, options = {}) {
|
|
92
|
+
const timeout = options.timeout ?? 3600 * 100;
|
|
93
|
+
const file = (0, fs_1.createWriteStream)(output);
|
|
94
|
+
await new Promise((resolve, reject) => {
|
|
95
|
+
const req = request(href(url, options.query), {
|
|
96
|
+
headers: options.headers,
|
|
97
|
+
}, (res) => {
|
|
98
|
+
if (res.statusCode === 200) {
|
|
99
|
+
res
|
|
100
|
+
.on("error", async (error) => {
|
|
101
|
+
file.destroy();
|
|
102
|
+
try {
|
|
103
|
+
await (0, promises_1.unlink)(output);
|
|
104
|
+
}
|
|
105
|
+
catch (_) { }
|
|
106
|
+
reject(error);
|
|
107
|
+
})
|
|
108
|
+
.pipe(file);
|
|
109
|
+
file.on("finish", () => {
|
|
110
|
+
file.close((error) => {
|
|
111
|
+
error ? reject(error) : resolve();
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
|
|
117
|
+
}
|
|
118
|
+
}).on("error", async (error) => {
|
|
119
|
+
try {
|
|
120
|
+
await (0, promises_1.unlink)(output);
|
|
121
|
+
}
|
|
122
|
+
catch (_) { }
|
|
123
|
+
reject(error);
|
|
124
|
+
});
|
|
125
|
+
req.setTimeout(timeout, () => {
|
|
126
|
+
req.destroy();
|
|
127
|
+
reject(new Error(`Request timeout after ${timeout / 1000}s`));
|
|
128
|
+
});
|
|
129
|
+
req.end();
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
exports.downloadFile = downloadFile;
|
|
133
|
+
async function uploadFile(url, path, options = {}) {
|
|
134
|
+
const { size } = await (0, promises_1.stat)(path);
|
|
135
|
+
const readStream = (0, fs_1.createReadStream)(path);
|
|
136
|
+
await new Promise((resolve, reject) => {
|
|
137
|
+
const req = request(href(url, options.query), {
|
|
138
|
+
method: "POST",
|
|
139
|
+
headers: {
|
|
140
|
+
...options.headers,
|
|
141
|
+
"Content-length": size,
|
|
142
|
+
},
|
|
143
|
+
}, (res) => {
|
|
144
|
+
if (res.statusCode !== 200) {
|
|
145
|
+
reject(new Error(`Upload failed: ${res.statusCode} ${res.statusMessage}`));
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
resolve();
|
|
149
|
+
}
|
|
150
|
+
}).on("error", reject);
|
|
151
|
+
readStream.on("error", reject).pipe(req);
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
exports.uploadFile = uploadFile;
|
package/utils/list.d.ts
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { Timer } from "./date";
|
|
2
|
+
import { ProgressManager } from "./progress";
|
|
3
|
+
import { Streams } from "./stream";
|
|
4
|
+
import { Listr, ListrGetRendererClassFromValue, ListrLogger, ListrTask, ListrTaskWrapper } from "listr2";
|
|
5
|
+
export declare class List3Logger<Levels extends string = string> extends ListrLogger<Levels> {
|
|
6
|
+
constructor(options?: {
|
|
7
|
+
streams?: Partial<Streams>;
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
export type Listr3Context = Record<string, Record<string, any>>;
|
|
11
|
+
type KeyIndex = number | string | (string | number)[];
|
|
12
|
+
type Listr3Task<T extends Listr3Context, K extends keyof T> = {
|
|
13
|
+
key: K;
|
|
14
|
+
keyIndex?: KeyIndex;
|
|
15
|
+
data: T[K];
|
|
16
|
+
title: string | {
|
|
17
|
+
initial: string;
|
|
18
|
+
started?: string;
|
|
19
|
+
failed?: string;
|
|
20
|
+
completed?: string;
|
|
21
|
+
};
|
|
22
|
+
runWrapper?: (cb: () => any) => any;
|
|
23
|
+
run: (task: ListrTaskWrapper<any, any, any>, data: T[K]) => Promise<void | ListrTask[] | Listr | undefined> | void | undefined | ListrTask[] | Listr;
|
|
24
|
+
exitOnError?: boolean;
|
|
25
|
+
enabled?: boolean;
|
|
26
|
+
skip?: boolean;
|
|
27
|
+
};
|
|
28
|
+
type List3TaskResultObject<K, D extends Record<string, any>> = {
|
|
29
|
+
key: K;
|
|
30
|
+
keyIndex?: string;
|
|
31
|
+
data: D;
|
|
32
|
+
elapsed: number;
|
|
33
|
+
error?: Error;
|
|
34
|
+
};
|
|
35
|
+
type SummaryResult = List3TaskResultObject<"summary", {
|
|
36
|
+
errors: number;
|
|
37
|
+
}>;
|
|
38
|
+
export type Listr3TaskResult<T extends Listr3Context> = {
|
|
39
|
+
[K in keyof T]: List3TaskResultObject<K, T[K]>;
|
|
40
|
+
}[keyof T];
|
|
41
|
+
export type Listr3TaskResultEnd<T extends Listr3Context> = Listr3TaskResult<T> | SummaryResult;
|
|
42
|
+
export declare class Listr3<T extends Listr3Context> extends Listr<void, "default", "simple"> {
|
|
43
|
+
readonly $options: {
|
|
44
|
+
streams?: Streams;
|
|
45
|
+
progressManager?: ProgressManager;
|
|
46
|
+
};
|
|
47
|
+
readonly resultMap: Record<string, Listr3TaskResult<T>>;
|
|
48
|
+
readonly resultList: Listr3TaskResult<T>[];
|
|
49
|
+
protected execTimer: Timer;
|
|
50
|
+
constructor($options: {
|
|
51
|
+
streams?: Streams;
|
|
52
|
+
progressManager?: ProgressManager;
|
|
53
|
+
});
|
|
54
|
+
private serializeKeyIndex;
|
|
55
|
+
private createResultIndex;
|
|
56
|
+
result(key: keyof T, keyIndex?: KeyIndex): Listr3TaskResult<T>;
|
|
57
|
+
$task<K extends keyof T>(item: Listr3Task<T, K>): ListrTask;
|
|
58
|
+
$tasks<K extends keyof T>(...items: (Listr3Task<T, K> | ListrTask | false)[]): ListrTask[];
|
|
59
|
+
add(tasks: ListrTask<void, ListrGetRendererClassFromValue<"default">> | ListrTask<void, ListrGetRendererClassFromValue<"default">>[]): this;
|
|
60
|
+
getSummaryResult(): SummaryResult;
|
|
61
|
+
getResult(): (SummaryResult | Listr3TaskResult<T>)[];
|
|
62
|
+
exec(): Promise<(Listr3TaskResult<T> | SummaryResult)[]>;
|
|
63
|
+
}
|
|
64
|
+
export {};
|