@datatruck/cli 0.26.2 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Command/StartServerCommand.d.ts +6 -0
- package/Command/StartServerCommand.js +24 -0
- package/Config/Config.d.ts +2 -0
- package/Config/Config.js +26 -0
- package/Factory/CommandFactory.d.ts +4 -1
- package/Factory/CommandFactory.js +6 -0
- package/Repository/DatatruckRepository.d.ts +8 -8
- package/Repository/DatatruckRepository.js +97 -132
- package/cli.js +1 -0
- package/config.schema.json +38 -2
- package/package.json +1 -1
- package/utils/datatruck/client.d.ts +22 -0
- package/utils/datatruck/client.js +96 -0
- package/utils/datatruck/config.d.ts +6 -6
- package/utils/datatruck/server.d.ts +21 -0
- package/utils/datatruck/server.js +96 -0
- package/utils/fs.d.ts +13 -6
- package/utils/fs.js +34 -19
- package/utils/http.d.ts +21 -0
- package/utils/http.js +154 -0
- package/utils/virtual-fs.d.ts +33 -0
- package/utils/virtual-fs.js +59 -0
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { CommandAbstract } from "./CommandAbstract";
|
|
2
|
+
export type StartServerCommandOptionsType<TResolved = false> = {};
|
|
3
|
+
export declare class StartServerCommand extends CommandAbstract<StartServerCommandOptionsType<false>, StartServerCommandOptionsType<true>> {
|
|
4
|
+
onOptions(): import("../utils/cli").OptionsType<StartServerCommandOptionsType<false>, StartServerCommandOptionsType<true>>;
|
|
5
|
+
onExec(): Promise<number>;
|
|
6
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.StartServerCommand = void 0;
|
|
4
|
+
const ConfigAction_1 = require("../Action/ConfigAction");
|
|
5
|
+
const server_1 = require("../utils/datatruck/server");
|
|
6
|
+
const CommandAbstract_1 = require("./CommandAbstract");
|
|
7
|
+
class StartServerCommand extends CommandAbstract_1.CommandAbstract {
|
|
8
|
+
onOptions() {
|
|
9
|
+
return this.returnsOptions({});
|
|
10
|
+
}
|
|
11
|
+
async onExec() {
|
|
12
|
+
const config = await ConfigAction_1.ConfigAction.fromGlobalOptions(this.globalOptions);
|
|
13
|
+
const server = (0, server_1.createDatatruckServer)(config.server || {});
|
|
14
|
+
const port = config.server?.listen?.port ?? 8888;
|
|
15
|
+
const address = config.server?.listen?.address ?? "127.0.0.1";
|
|
16
|
+
console.info(`Listening on http://${address}:${port}`);
|
|
17
|
+
await new Promise((resolve, reject) => {
|
|
18
|
+
server.listen(port, address);
|
|
19
|
+
server.on("error", reject);
|
|
20
|
+
});
|
|
21
|
+
return 0;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
exports.StartServerCommand = StartServerCommand;
|
package/Config/Config.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { DatatruckServerOptions } from "../utils/datatruck/server";
|
|
1
2
|
import { PackageConfigType } from "./PackageConfig";
|
|
2
3
|
import { RepositoryConfigType } from "./RepositoryConfig";
|
|
3
4
|
import type { JSONSchema7 } from "json-schema";
|
|
@@ -5,5 +6,6 @@ export type ConfigType = {
|
|
|
5
6
|
tempDir?: string;
|
|
6
7
|
repositories: RepositoryConfigType[];
|
|
7
8
|
packages: PackageConfigType[];
|
|
9
|
+
server?: DatatruckServerOptions;
|
|
8
10
|
};
|
|
9
11
|
export declare const configDefinition: JSONSchema7;
|
package/Config/Config.js
CHANGED
|
@@ -17,5 +17,31 @@ exports.configDefinition = {
|
|
|
17
17
|
type: "array",
|
|
18
18
|
items: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.package),
|
|
19
19
|
},
|
|
20
|
+
server: {
|
|
21
|
+
type: "object",
|
|
22
|
+
additionalProperties: false,
|
|
23
|
+
properties: {
|
|
24
|
+
path: { type: "string" },
|
|
25
|
+
users: {
|
|
26
|
+
type: "array",
|
|
27
|
+
items: {
|
|
28
|
+
type: "object",
|
|
29
|
+
additionalProperties: false,
|
|
30
|
+
properties: {
|
|
31
|
+
name: { type: "string" },
|
|
32
|
+
password: { type: "string" },
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
listen: {
|
|
37
|
+
type: "object",
|
|
38
|
+
additionalProperties: false,
|
|
39
|
+
properties: {
|
|
40
|
+
port: { type: "integer" },
|
|
41
|
+
address: { type: "string" },
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
},
|
|
20
46
|
},
|
|
21
47
|
};
|
|
@@ -9,6 +9,7 @@ import { PruneCommandOptionsType } from "../Command/PruneCommand";
|
|
|
9
9
|
import { RestoreCommandOptionsType } from "../Command/RestoreCommand";
|
|
10
10
|
import { RestoreSessionsCommandOptionsType } from "../Command/RestoreSessionsCommand";
|
|
11
11
|
import { SnapshotsCommandLogType, SnapshotsCommandOptionsType } from "../Command/SnapshotsCommand";
|
|
12
|
+
import { StartServerCommandOptionsType } from "../Command/StartServerCommand";
|
|
12
13
|
export declare enum CommandEnum {
|
|
13
14
|
config = "config",
|
|
14
15
|
init = "init",
|
|
@@ -18,7 +19,8 @@ export declare enum CommandEnum {
|
|
|
18
19
|
backupSessions = "backup-sessions",
|
|
19
20
|
restore = "restore",
|
|
20
21
|
restoreSessions = "restore-sessions",
|
|
21
|
-
cleanCache = "clean-cache"
|
|
22
|
+
cleanCache = "clean-cache",
|
|
23
|
+
startServer = "start-server"
|
|
22
24
|
}
|
|
23
25
|
export type OptionsMapType = {
|
|
24
26
|
[CommandEnum.config]: ConfigCommandOptionsType;
|
|
@@ -30,6 +32,7 @@ export type OptionsMapType = {
|
|
|
30
32
|
[CommandEnum.restore]: RestoreCommandOptionsType;
|
|
31
33
|
[CommandEnum.restoreSessions]: RestoreSessionsCommandOptionsType;
|
|
32
34
|
[CommandEnum.cleanCache]: CleanCacheActionOptionsType;
|
|
35
|
+
[CommandEnum.startServer]: StartServerCommandOptionsType;
|
|
33
36
|
};
|
|
34
37
|
export type LogMapType = {
|
|
35
38
|
[CommandEnum.config]: ConfigCommandLogType;
|
|
@@ -10,6 +10,7 @@ const PruneCommand_1 = require("../Command/PruneCommand");
|
|
|
10
10
|
const RestoreCommand_1 = require("../Command/RestoreCommand");
|
|
11
11
|
const RestoreSessionsCommand_1 = require("../Command/RestoreSessionsCommand");
|
|
12
12
|
const SnapshotsCommand_1 = require("../Command/SnapshotsCommand");
|
|
13
|
+
const StartServerCommand_1 = require("../Command/StartServerCommand");
|
|
13
14
|
const AppError_1 = require("../Error/AppError");
|
|
14
15
|
var CommandEnum;
|
|
15
16
|
(function (CommandEnum) {
|
|
@@ -22,6 +23,7 @@ var CommandEnum;
|
|
|
22
23
|
CommandEnum["restore"] = "restore";
|
|
23
24
|
CommandEnum["restoreSessions"] = "restore-sessions";
|
|
24
25
|
CommandEnum["cleanCache"] = "clean-cache";
|
|
26
|
+
CommandEnum["startServer"] = "start-server";
|
|
25
27
|
})(CommandEnum || (exports.CommandEnum = CommandEnum = {}));
|
|
26
28
|
function CommandFactory(type, globalOptions, options) {
|
|
27
29
|
const constructor = CommandConstructorFactory(type);
|
|
@@ -74,6 +76,7 @@ function makeParseLog(type) {
|
|
|
74
76
|
const data = [];
|
|
75
77
|
const consoleLog = console.log;
|
|
76
78
|
console.log = console.info = (...items) => {
|
|
79
|
+
consoleLog.bind(console)(...items);
|
|
77
80
|
data.push(...items);
|
|
78
81
|
};
|
|
79
82
|
return function parseLog() {
|
|
@@ -110,6 +113,9 @@ function CommandConstructorFactory(type) {
|
|
|
110
113
|
else if (type === CommandEnum.cleanCache) {
|
|
111
114
|
return CleanCacheCommand_1.CleanCacheCommand;
|
|
112
115
|
}
|
|
116
|
+
else if (type === CommandEnum.startServer) {
|
|
117
|
+
return StartServerCommand_1.StartServerCommand;
|
|
118
|
+
}
|
|
113
119
|
else {
|
|
114
120
|
throw new AppError_1.AppError(`Invalid command type: ${type}`);
|
|
115
121
|
}
|
|
@@ -11,10 +11,11 @@ export type MetaDataType = {
|
|
|
11
11
|
size: number;
|
|
12
12
|
tarStats?: Record<string, {
|
|
13
13
|
files: number;
|
|
14
|
+
size: number;
|
|
14
15
|
}>;
|
|
15
16
|
};
|
|
16
17
|
export type DatatruckRepositoryConfigType = {
|
|
17
|
-
|
|
18
|
+
backend: string;
|
|
18
19
|
compress?: boolean | CompressOptions;
|
|
19
20
|
};
|
|
20
21
|
type PackObject = {
|
|
@@ -33,10 +34,11 @@ export declare const datatruckRepositoryDefinition: JSONSchema7;
|
|
|
33
34
|
export declare const datatruckPackageRepositoryDefinition: JSONSchema7;
|
|
34
35
|
export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRepositoryConfigType> {
|
|
35
36
|
static zipBasenameTpl: string;
|
|
36
|
-
static buildSnapshotName(
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
static buildSnapshotName(snapshot: {
|
|
38
|
+
id: string;
|
|
39
|
+
date: string;
|
|
40
|
+
}, pkg: {
|
|
41
|
+
name: string;
|
|
40
42
|
}): string;
|
|
41
43
|
static parseSnapshotName(name: string): {
|
|
42
44
|
snapshotDate: string;
|
|
@@ -44,9 +46,7 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
|
|
|
44
46
|
snapshotShortId: string;
|
|
45
47
|
sourcePath: string;
|
|
46
48
|
} | null;
|
|
47
|
-
|
|
48
|
-
static parseMetaData(path: string): Promise<MetaDataType | undefined>;
|
|
49
|
-
static stringifyMetaData(data: MetaDataType): string;
|
|
49
|
+
static parseMetaData(data: string): Promise<MetaDataType>;
|
|
50
50
|
onGetSource(): string;
|
|
51
51
|
onInit(data: InitDataType): Promise<void>;
|
|
52
52
|
onPrune(data: PruneDataType): Promise<void>;
|
|
@@ -4,6 +4,7 @@ exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exp
|
|
|
4
4
|
const AppError_1 = require("../Error/AppError");
|
|
5
5
|
const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
|
|
6
6
|
const cli_1 = require("../utils/cli");
|
|
7
|
+
const client_1 = require("../utils/datatruck/client");
|
|
7
8
|
const paths_1 = require("../utils/datatruck/paths");
|
|
8
9
|
const fs_1 = require("../utils/fs");
|
|
9
10
|
const string_1 = require("../utils/string");
|
|
@@ -16,10 +17,10 @@ const path_1 = require("path");
|
|
|
16
17
|
exports.datatruckRepositoryName = "datatruck";
|
|
17
18
|
exports.datatruckRepositoryDefinition = {
|
|
18
19
|
type: "object",
|
|
19
|
-
required: ["
|
|
20
|
+
required: ["backend"],
|
|
20
21
|
additionalProperties: false,
|
|
21
22
|
properties: {
|
|
22
|
-
|
|
23
|
+
backend: { type: "string" },
|
|
23
24
|
compress: {
|
|
24
25
|
anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
|
|
25
26
|
},
|
|
@@ -52,10 +53,10 @@ exports.datatruckPackageRepositoryDefinition = {
|
|
|
52
53
|
};
|
|
53
54
|
class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
54
55
|
static zipBasenameTpl = `.*.dd.tar.gz`;
|
|
55
|
-
static buildSnapshotName(
|
|
56
|
-
const date =
|
|
57
|
-
const pkgName = encodeURIComponent(
|
|
58
|
-
const snapshotShortId =
|
|
56
|
+
static buildSnapshotName(snapshot, pkg) {
|
|
57
|
+
const date = snapshot.date.replace(/:/g, "-");
|
|
58
|
+
const pkgName = encodeURIComponent(pkg.name).replace(/%40/g, "@");
|
|
59
|
+
const snapshotShortId = snapshot.id.slice(0, 8);
|
|
59
60
|
return `${date}_${pkgName}_${snapshotShortId}`;
|
|
60
61
|
}
|
|
61
62
|
static parseSnapshotName(name) {
|
|
@@ -69,49 +70,32 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
69
70
|
packageName = decodeURIComponent(packageName);
|
|
70
71
|
return { snapshotDate, packageName, snapshotShortId, sourcePath: name };
|
|
71
72
|
}
|
|
72
|
-
|
|
73
|
-
return (
|
|
74
|
-
}
|
|
75
|
-
static async parseMetaData(path) {
|
|
76
|
-
let contents;
|
|
77
|
-
try {
|
|
78
|
-
contents = await (0, promises_1.readFile)(path);
|
|
79
|
-
}
|
|
80
|
-
catch (error) {
|
|
81
|
-
if ((0, fs_1.isNotFoundError)(error))
|
|
82
|
-
return;
|
|
83
|
-
throw error;
|
|
84
|
-
}
|
|
85
|
-
return JSON.parse(contents.toString());
|
|
86
|
-
}
|
|
87
|
-
static stringifyMetaData(data) {
|
|
88
|
-
return JSON.stringify(data);
|
|
73
|
+
static async parseMetaData(data) {
|
|
74
|
+
return JSON.parse(data.toString());
|
|
89
75
|
}
|
|
90
76
|
onGetSource() {
|
|
91
|
-
return this.config.
|
|
77
|
+
return this.config.backend;
|
|
92
78
|
}
|
|
93
79
|
async onInit(data) {
|
|
94
|
-
|
|
80
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
81
|
+
await fs.mkdir(".");
|
|
95
82
|
}
|
|
96
83
|
async onPrune(data) {
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
packageName: data.snapshot.packageName,
|
|
84
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
85
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, {
|
|
86
|
+
name: data.snapshot.packageName,
|
|
101
87
|
});
|
|
102
|
-
const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
103
88
|
if (data.options.verbose)
|
|
104
|
-
(0, cli_1.logExec)(`Deleting ${
|
|
105
|
-
if (await
|
|
106
|
-
await (
|
|
107
|
-
recursive: true,
|
|
108
|
-
});
|
|
89
|
+
(0, cli_1.logExec)(`Deleting ${fs.resolvePath(snapshotName)}`);
|
|
90
|
+
if (await fs.existsDir(snapshotName))
|
|
91
|
+
await fs.rmAll(snapshotName);
|
|
109
92
|
}
|
|
110
93
|
async onSnapshots(data) {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
94
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
95
|
+
if (!(await fs.existsDir(".")))
|
|
96
|
+
throw new Error(`Repository (${this.repository.name}) out path does not exist: ${fs.resolvePath(".")}`);
|
|
114
97
|
const snapshots = [];
|
|
98
|
+
const snapshotNames = await fs.readdir(".");
|
|
115
99
|
const packagePatterns = (0, string_1.makePathPatterns)(data.options.packageNames);
|
|
116
100
|
const taskPatterns = (0, string_1.makePathPatterns)(data.options.packageTaskNames);
|
|
117
101
|
for (const snapshotName of snapshotNames) {
|
|
@@ -124,8 +108,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
124
108
|
if (data.options.ids &&
|
|
125
109
|
!data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
|
|
126
110
|
continue;
|
|
127
|
-
const
|
|
128
|
-
const meta = await DatatruckRepository.parseMetaData(
|
|
111
|
+
const metaData = await fs.readFileIfExists(`${snapshotName}/meta.json`);
|
|
112
|
+
const meta = !!metaData && (await DatatruckRepository.parseMetaData(metaData));
|
|
129
113
|
if (!meta)
|
|
130
114
|
continue;
|
|
131
115
|
if (taskPatterns && !(0, string_1.checkMatch)(meta.task, taskPatterns))
|
|
@@ -149,16 +133,15 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
149
133
|
return snapshots;
|
|
150
134
|
}
|
|
151
135
|
async onBackup(data) {
|
|
152
|
-
const
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
const outPath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
|
|
136
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
137
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
138
|
+
const outPath = fs.isLocal()
|
|
139
|
+
? fs.resolvePath(snapshotName)
|
|
140
|
+
: await this.mkTmpDir("datatruck-backup");
|
|
158
141
|
const pkg = data.package;
|
|
159
142
|
const sourcePath = data.targetPath ?? pkg.path;
|
|
160
143
|
(0, assert_1.ok)(sourcePath);
|
|
161
|
-
await
|
|
144
|
+
await fs.mkdir(snapshotName);
|
|
162
145
|
const backupPathsOptions = {
|
|
163
146
|
package: data.package,
|
|
164
147
|
snapshot: data.snapshot,
|
|
@@ -224,21 +207,28 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
224
207
|
if (includeList) {
|
|
225
208
|
tarStats[packBasename] = {
|
|
226
209
|
files: stream.lines(packIndex),
|
|
210
|
+
size: 0,
|
|
227
211
|
};
|
|
212
|
+
const tarPath = (0, path_1.join)(outPath, packBasename);
|
|
228
213
|
await (0, tar_1.createTar)({
|
|
229
214
|
compress: pack.compress,
|
|
230
215
|
verbose: data.options.verbose,
|
|
231
216
|
includeList,
|
|
232
217
|
path: sourcePath,
|
|
233
|
-
output:
|
|
218
|
+
output: tarPath,
|
|
234
219
|
onEntry: async (data) => await scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
|
|
235
220
|
});
|
|
221
|
+
tarStats[packBasename].size = (await (0, promises_1.stat)(tarPath)).size;
|
|
222
|
+
if (!fs.isLocal()) {
|
|
223
|
+
await fs.upload(tarPath, `${snapshotName}/${packBasename}`);
|
|
224
|
+
await (0, promises_1.rm)(tarPath);
|
|
225
|
+
}
|
|
236
226
|
}
|
|
237
227
|
packIndex++;
|
|
238
228
|
}
|
|
239
229
|
await scanner.end();
|
|
240
230
|
// Meta
|
|
241
|
-
const metaPath = `${
|
|
231
|
+
const metaPath = `${snapshotName}/meta.json`;
|
|
242
232
|
const nodePkg = (0, fs_1.parsePackageFile)();
|
|
243
233
|
const meta = {
|
|
244
234
|
id: data.snapshot.id,
|
|
@@ -247,47 +237,42 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
247
237
|
package: data.package.name,
|
|
248
238
|
task: data.package.task?.name,
|
|
249
239
|
version: nodePkg.version,
|
|
250
|
-
size:
|
|
240
|
+
size: Object.values(tarStats).reduce((total, { size }) => total + size, 0),
|
|
251
241
|
tarStats,
|
|
252
242
|
};
|
|
253
243
|
if (data.options.verbose)
|
|
254
|
-
(0, cli_1.logExec)(`Writing metadata into ${metaPath}`);
|
|
255
|
-
await
|
|
244
|
+
(0, cli_1.logExec)(`Writing metadata into ${fs.resolvePath(metaPath)}`);
|
|
245
|
+
await fs.writeFile(`${snapshotName}/meta.json`, JSON.stringify(meta));
|
|
256
246
|
}
|
|
257
247
|
async onCopyBackup(data) {
|
|
258
|
-
const
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
packageName: data.package.name,
|
|
262
|
-
});
|
|
263
|
-
const sourcePath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
|
|
264
|
-
const targetPath = (0, path_1.resolve)((0, path_1.join)(data.mirrorRepositoryConfig.outPath, snapshotName));
|
|
248
|
+
const sourceFs = (0, client_1.createFs)(this.config.backend);
|
|
249
|
+
const targetFs = (0, client_1.createFs)(data.mirrorRepositoryConfig.backend);
|
|
250
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
|
|
265
251
|
if (data.options.verbose)
|
|
266
|
-
(0, cli_1.logExec)(`Copying backup files to ${
|
|
267
|
-
await
|
|
268
|
-
await
|
|
269
|
-
const
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
await (0, promises_1.cp)(sourceFile, targetFile);
|
|
252
|
+
(0, cli_1.logExec)(`Copying backup files to ${data.mirrorRepositoryConfig.backend}`);
|
|
253
|
+
await targetFs.mkdir(snapshotName);
|
|
254
|
+
await targetFs.ensureEmptyDir(snapshotName);
|
|
255
|
+
const entries = await sourceFs.readdir(snapshotName);
|
|
256
|
+
for (const entry of entries) {
|
|
257
|
+
const sourceEntry = `${snapshotName}/${entry}`;
|
|
258
|
+
if (targetFs.isLocal()) {
|
|
259
|
+
await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
|
|
260
|
+
}
|
|
261
|
+
else {
|
|
262
|
+
const tempDir = await this.mkTmpDir("remote-copy", entry);
|
|
263
|
+
const tempFile = (0, path_1.join)(tempDir, entry);
|
|
264
|
+
try {
|
|
265
|
+
await sourceFs.download(sourceEntry, tempFile);
|
|
266
|
+
await targetFs.upload(tempFile, sourceEntry);
|
|
267
|
+
}
|
|
268
|
+
finally {
|
|
269
|
+
await (0, fs_1.tryRm)(tempFile);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
287
272
|
}
|
|
288
|
-
await scanner.end();
|
|
289
273
|
}
|
|
290
274
|
async onRestore(data) {
|
|
275
|
+
const fs = (0, client_1.createFs)(this.config.backend);
|
|
291
276
|
const relRestorePath = data.targetPath ?? data.package.restorePath;
|
|
292
277
|
(0, assert_1.ok)(relRestorePath);
|
|
293
278
|
const restorePath = (0, path_1.resolve)(relRestorePath);
|
|
@@ -298,61 +283,41 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
|
|
|
298
283
|
});
|
|
299
284
|
if (!snapshot)
|
|
300
285
|
throw new AppError_1.AppError("Snapshot not found");
|
|
301
|
-
const snapshotName = DatatruckRepository.buildSnapshotName(
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
|
|
307
|
-
const metaPath = (0, path_1.join)(sourcePath, "meta.json");
|
|
308
|
-
const meta = await DatatruckRepository.parseMetaData(metaPath);
|
|
309
|
-
const scanner = await (0, fs_1.createFileScanner)({
|
|
310
|
-
onProgress: data.onProgress,
|
|
311
|
-
glob: {
|
|
312
|
-
cwd: sourcePath,
|
|
313
|
-
include: ["**/*"],
|
|
314
|
-
},
|
|
315
|
-
});
|
|
316
|
-
const tarFiles = [];
|
|
286
|
+
const snapshotName = DatatruckRepository.buildSnapshotName(snapshot, data.package);
|
|
287
|
+
const meta = await DatatruckRepository.parseMetaData(await fs.readFile(`${snapshotName}/meta.json`));
|
|
288
|
+
const progress = (0, fs_1.createProgress)({ onProgress: data.onProgress });
|
|
289
|
+
await progress.update("Scanning files");
|
|
290
|
+
const entries = (await fs.readdir(snapshotName)).filter((v) => v.endsWith(".tar") || v.endsWith(".tar.gz"));
|
|
317
291
|
const tarStats = meta?.tarStats || {};
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
const isTarGz = entry.name.endsWith(".tar.gz");
|
|
322
|
-
if (isTar || isTarGz) {
|
|
323
|
-
tarFiles.push(path);
|
|
324
|
-
if (typeof tarStats[entry.name]?.files === "number") {
|
|
325
|
-
scanner.total += tarStats[entry.name].files;
|
|
326
|
-
}
|
|
327
|
-
else {
|
|
328
|
-
scanner.progress("Scanning", entry.name, false);
|
|
329
|
-
const selfTarStats = (tarStats[entry.name] = { files: 0 });
|
|
330
|
-
await (0, tar_1.listTar)({
|
|
331
|
-
input: path,
|
|
332
|
-
verbose: data.options.verbose,
|
|
333
|
-
onEntry: () => {
|
|
334
|
-
scanner.total++;
|
|
335
|
-
selfTarStats.files++;
|
|
336
|
-
},
|
|
337
|
-
});
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
return false;
|
|
341
|
-
});
|
|
292
|
+
for (const file in tarStats)
|
|
293
|
+
progress.total += tarStats[file].files;
|
|
294
|
+
await progress.update(`Scanned files: ${progress.total}`);
|
|
342
295
|
if (data.options.verbose)
|
|
343
296
|
(0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
|
|
344
|
-
for (const
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
297
|
+
for (const entry of entries) {
|
|
298
|
+
let tempEntry;
|
|
299
|
+
try {
|
|
300
|
+
const sourceEntry = `${snapshotName}/${entry}`;
|
|
301
|
+
if (!fs.isLocal()) {
|
|
302
|
+
const tempDir = await this.mkTmpDir("remote-restore", entry);
|
|
303
|
+
tempEntry = `${tempDir}/${entry}`;
|
|
304
|
+
await fs.download(sourceEntry, tempEntry);
|
|
305
|
+
}
|
|
306
|
+
await (0, tar_1.extractTar)({
|
|
307
|
+
total: tarStats[entry].files,
|
|
308
|
+
input: tempEntry ?? fs.resolvePath(sourceEntry),
|
|
309
|
+
output: restorePath,
|
|
310
|
+
decompress: entry.endsWith(".tar.gz"),
|
|
311
|
+
verbose: data.options.verbose,
|
|
312
|
+
onEntry: async (data) => await progress.update(entry.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
finally {
|
|
316
|
+
if (tempEntry)
|
|
317
|
+
await (0, fs_1.tryRm)(tempEntry);
|
|
318
|
+
}
|
|
354
319
|
}
|
|
355
|
-
await
|
|
320
|
+
await progress.update("Finished");
|
|
356
321
|
}
|
|
357
322
|
}
|
|
358
323
|
exports.DatatruckRepository = DatatruckRepository;
|
package/cli.js
CHANGED
|
@@ -83,6 +83,7 @@ program.option("-c,--config <path>", "Config path", process.env["DATATRUCK_CONFI
|
|
|
83
83
|
program.option("--progress <value>", "Progress type (auto, plain, tty)", "auto");
|
|
84
84
|
program.option("--progress-interval <ms>", "Progress interval", Number, 1000);
|
|
85
85
|
program.option("-o,--output-format <format>", "Output format (json, pjson, yaml, table, custom=$, tpl=name)", "table");
|
|
86
|
+
makeCommand(CommandFactory_1.CommandEnum.startServer).alias("start");
|
|
86
87
|
makeCommand(CommandFactory_1.CommandEnum.config).alias("c");
|
|
87
88
|
makeCommand(CommandFactory_1.CommandEnum.init).alias("i");
|
|
88
89
|
makeCommand(CommandFactory_1.CommandEnum.snapshots).alias("s");
|
package/config.schema.json
CHANGED
|
@@ -444,11 +444,11 @@
|
|
|
444
444
|
"datatruck-repository": {
|
|
445
445
|
"type": "object",
|
|
446
446
|
"required": [
|
|
447
|
-
"
|
|
447
|
+
"backend"
|
|
448
448
|
],
|
|
449
449
|
"additionalProperties": false,
|
|
450
450
|
"properties": {
|
|
451
|
-
"
|
|
451
|
+
"backend": {
|
|
452
452
|
"type": "string"
|
|
453
453
|
},
|
|
454
454
|
"compress": {
|
|
@@ -996,6 +996,42 @@
|
|
|
996
996
|
"items": {
|
|
997
997
|
"$ref": "#/definitions/package"
|
|
998
998
|
}
|
|
999
|
+
},
|
|
1000
|
+
"server": {
|
|
1001
|
+
"type": "object",
|
|
1002
|
+
"additionalProperties": false,
|
|
1003
|
+
"properties": {
|
|
1004
|
+
"path": {
|
|
1005
|
+
"type": "string"
|
|
1006
|
+
},
|
|
1007
|
+
"users": {
|
|
1008
|
+
"type": "array",
|
|
1009
|
+
"items": {
|
|
1010
|
+
"type": "object",
|
|
1011
|
+
"additionalProperties": false,
|
|
1012
|
+
"properties": {
|
|
1013
|
+
"name": {
|
|
1014
|
+
"type": "string"
|
|
1015
|
+
},
|
|
1016
|
+
"password": {
|
|
1017
|
+
"type": "string"
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
},
|
|
1022
|
+
"listen": {
|
|
1023
|
+
"type": "object",
|
|
1024
|
+
"additionalProperties": false,
|
|
1025
|
+
"properties": {
|
|
1026
|
+
"port": {
|
|
1027
|
+
"type": "integer"
|
|
1028
|
+
},
|
|
1029
|
+
"address": {
|
|
1030
|
+
"type": "string"
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
999
1035
|
}
|
|
1000
1036
|
}
|
|
1001
1037
|
},
|
package/package.json
CHANGED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { AbstractFs, FsOptions } from "../virtual-fs";
|
|
2
|
+
export declare class RemoteFs extends AbstractFs {
|
|
3
|
+
readonly options: FsOptions;
|
|
4
|
+
protected url: string;
|
|
5
|
+
protected headers: Record<string, string>;
|
|
6
|
+
constructor(options: FsOptions);
|
|
7
|
+
isLocal(): boolean;
|
|
8
|
+
protected fetchJson(name: string, params: any[]): Promise<any>;
|
|
9
|
+
protected post(name: string, params: any[], data: string): Promise<void>;
|
|
10
|
+
existsDir(path: string): Promise<any>;
|
|
11
|
+
mkdir(path: string): Promise<any>;
|
|
12
|
+
readFile(path: string): Promise<any>;
|
|
13
|
+
readdir(path: string): Promise<any>;
|
|
14
|
+
readFileIfExists(path: string): Promise<string | undefined>;
|
|
15
|
+
ensureEmptyDir(path: string): Promise<void>;
|
|
16
|
+
writeFile(path: string, contents: string): Promise<void>;
|
|
17
|
+
rmAll(path: string): Promise<void>;
|
|
18
|
+
upload(source: string, target: string): Promise<void>;
|
|
19
|
+
download(source: string, target: string, timeout?: number): Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export declare function isRemoteBackend(backend: string): boolean;
|
|
22
|
+
export declare function createFs(backend: string): AbstractFs;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createFs = exports.isRemoteBackend = exports.RemoteFs = void 0;
|
|
4
|
+
const http_1 = require("../http");
|
|
5
|
+
const virtual_fs_1 = require("../virtual-fs");
|
|
6
|
+
const server_1 = require("./server");
|
|
7
|
+
class RemoteFs extends virtual_fs_1.AbstractFs {
|
|
8
|
+
options;
|
|
9
|
+
url;
|
|
10
|
+
headers;
|
|
11
|
+
constructor(options) {
|
|
12
|
+
super(options);
|
|
13
|
+
this.options = options;
|
|
14
|
+
const url = new URL(options.backend);
|
|
15
|
+
this.headers = {
|
|
16
|
+
[server_1.headerKey.user]: url.username,
|
|
17
|
+
[server_1.headerKey.password]: url.password,
|
|
18
|
+
};
|
|
19
|
+
url.username = "";
|
|
20
|
+
url.password = "";
|
|
21
|
+
this.url = url.href;
|
|
22
|
+
if (this.url.endsWith("/"))
|
|
23
|
+
this.url = this.url.slice(0, -1);
|
|
24
|
+
}
|
|
25
|
+
isLocal() {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
async fetchJson(name, params) {
|
|
29
|
+
return await (0, http_1.fetchJson)(`${this.url}/${name}`, {
|
|
30
|
+
headers: this.headers,
|
|
31
|
+
query: {
|
|
32
|
+
params: JSON.stringify(params),
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
async post(name, params, data) {
|
|
37
|
+
return await (0, http_1.post)(`${this.url}/${name}`, data, {
|
|
38
|
+
headers: this.headers,
|
|
39
|
+
query: {
|
|
40
|
+
params: JSON.stringify(params),
|
|
41
|
+
},
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
async existsDir(path) {
|
|
45
|
+
return await this.fetchJson("existsDir", [path]);
|
|
46
|
+
}
|
|
47
|
+
async mkdir(path) {
|
|
48
|
+
return await this.fetchJson("mkdir", [path]);
|
|
49
|
+
}
|
|
50
|
+
async readFile(path) {
|
|
51
|
+
return await this.fetchJson("readFile", [path]);
|
|
52
|
+
}
|
|
53
|
+
async readdir(path) {
|
|
54
|
+
return await this.fetchJson("readdir", [path]);
|
|
55
|
+
}
|
|
56
|
+
async readFileIfExists(path) {
|
|
57
|
+
return await this.fetchJson("readFileIfExists", [path]);
|
|
58
|
+
}
|
|
59
|
+
async ensureEmptyDir(path) {
|
|
60
|
+
return await this.fetchJson("readdir", [path]);
|
|
61
|
+
}
|
|
62
|
+
async writeFile(path, contents) {
|
|
63
|
+
await this.post("writeFile", [path], contents);
|
|
64
|
+
}
|
|
65
|
+
async rmAll(path) {
|
|
66
|
+
await this.fetchJson("rmAll", [path]);
|
|
67
|
+
}
|
|
68
|
+
async upload(source, target) {
|
|
69
|
+
await (0, http_1.uploadFile)(`${this.url}/upload`, source, {
|
|
70
|
+
headers: this.headers,
|
|
71
|
+
query: {
|
|
72
|
+
params: JSON.stringify([target]),
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
async download(source, target, timeout = 100000) {
|
|
77
|
+
await (0, http_1.downloadFile)(`${this.url}/download`, target, {
|
|
78
|
+
timeout,
|
|
79
|
+
headers: this.headers,
|
|
80
|
+
query: {
|
|
81
|
+
params: JSON.stringify([source]),
|
|
82
|
+
},
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
exports.RemoteFs = RemoteFs;
|
|
87
|
+
function isRemoteBackend(backend) {
|
|
88
|
+
return backend.startsWith("http:") || backend.startsWith("https:");
|
|
89
|
+
}
|
|
90
|
+
exports.isRemoteBackend = isRemoteBackend;
|
|
91
|
+
function createFs(backend) {
|
|
92
|
+
return isRemoteBackend(backend)
|
|
93
|
+
? new RemoteFs({ backend })
|
|
94
|
+
: new virtual_fs_1.LocalFs({ backend });
|
|
95
|
+
}
|
|
96
|
+
exports.createFs = createFs;
|
|
@@ -31,18 +31,18 @@ export declare const pkgPathParams: {
|
|
|
31
31
|
[name in "temp" | Exclude<keyof ResolvePackagePathParamsType, "path">]: string;
|
|
32
32
|
};
|
|
33
33
|
export declare const pkgIncludeParams: {
|
|
34
|
+
action: string;
|
|
34
35
|
packageName: string;
|
|
35
36
|
temp: string;
|
|
36
37
|
snapshotId: string;
|
|
37
38
|
snapshotDate: string;
|
|
38
|
-
action: string;
|
|
39
39
|
};
|
|
40
40
|
export declare const pkgExcludeParams: {
|
|
41
|
+
action: string;
|
|
41
42
|
packageName: string;
|
|
42
43
|
temp: string;
|
|
43
44
|
snapshotId: string;
|
|
44
45
|
snapshotDate: string;
|
|
45
|
-
action: string;
|
|
46
46
|
};
|
|
47
47
|
export declare const pkgRestorePathParams: {
|
|
48
48
|
[name in "temp" | keyof ResolvePackagePathParamsType]: string;
|
|
@@ -52,33 +52,33 @@ export declare const dbNameParams: {
|
|
|
52
52
|
};
|
|
53
53
|
export declare const params: {
|
|
54
54
|
pkgPath: {
|
|
55
|
+
action: string;
|
|
55
56
|
packageName: string;
|
|
56
57
|
temp: string;
|
|
57
58
|
snapshotId: string;
|
|
58
59
|
snapshotDate: string;
|
|
59
|
-
action: string;
|
|
60
60
|
};
|
|
61
61
|
pkgRestorePath: {
|
|
62
62
|
path: string;
|
|
63
|
+
action: string;
|
|
63
64
|
packageName: string;
|
|
64
65
|
temp: string;
|
|
65
66
|
snapshotId: string;
|
|
66
67
|
snapshotDate: string;
|
|
67
|
-
action: string;
|
|
68
68
|
};
|
|
69
69
|
pkgInclude: {
|
|
70
|
+
action: string;
|
|
70
71
|
packageName: string;
|
|
71
72
|
temp: string;
|
|
72
73
|
snapshotId: string;
|
|
73
74
|
snapshotDate: string;
|
|
74
|
-
action: string;
|
|
75
75
|
};
|
|
76
76
|
pkgExclude: {
|
|
77
|
+
action: string;
|
|
77
78
|
packageName: string;
|
|
78
79
|
temp: string;
|
|
79
80
|
snapshotId: string;
|
|
80
81
|
snapshotDate: string;
|
|
81
|
-
action: string;
|
|
82
82
|
};
|
|
83
83
|
dbName: {
|
|
84
84
|
snapshotId: string;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { IncomingMessage } from "http";
|
|
3
|
+
type User = {
|
|
4
|
+
name: string;
|
|
5
|
+
password: string;
|
|
6
|
+
};
|
|
7
|
+
export type DatatruckServerOptions = {
|
|
8
|
+
path?: string;
|
|
9
|
+
log?: boolean;
|
|
10
|
+
listen?: {
|
|
11
|
+
port?: number;
|
|
12
|
+
address?: string;
|
|
13
|
+
};
|
|
14
|
+
users?: User[];
|
|
15
|
+
};
|
|
16
|
+
export declare const headerKey: {
|
|
17
|
+
user: string;
|
|
18
|
+
password: string;
|
|
19
|
+
};
|
|
20
|
+
export declare function createDatatruckServer(options: DatatruckServerOptions): import("node:http").Server<typeof IncomingMessage, typeof import("node:http").ServerResponse>;
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createDatatruckServer = exports.headerKey = void 0;
|
|
4
|
+
const http_1 = require("../http");
|
|
5
|
+
const virtual_fs_1 = require("../virtual-fs");
|
|
6
|
+
const fs_1 = require("fs");
|
|
7
|
+
const http_2 = require("http");
|
|
8
|
+
function parseUrl(inUrl) {
|
|
9
|
+
const url = new URL(`http://127.0.0.1${inUrl}`);
|
|
10
|
+
const inParams = url.searchParams.get("params");
|
|
11
|
+
const action = url.pathname.slice(1);
|
|
12
|
+
if (typeof inParams === "string") {
|
|
13
|
+
const params = JSON.parse(inParams);
|
|
14
|
+
if (!Array.isArray(params))
|
|
15
|
+
throw new Error(`Invalid params`);
|
|
16
|
+
return { action, params };
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
return { action, params: [] };
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
exports.headerKey = {
|
|
23
|
+
user: "x-dtt-user",
|
|
24
|
+
password: "x-dtt-password",
|
|
25
|
+
};
|
|
26
|
+
function validateRequest(req, users) {
|
|
27
|
+
const name = req.headers[exports.headerKey.user];
|
|
28
|
+
const password = req.headers[exports.headerKey.password];
|
|
29
|
+
return users.some((user) => user.name.length &&
|
|
30
|
+
user.password.length &&
|
|
31
|
+
user.name === name &&
|
|
32
|
+
user.password === password);
|
|
33
|
+
}
|
|
34
|
+
function createDatatruckServer(options) {
|
|
35
|
+
const log = options.log ?? true;
|
|
36
|
+
return (0, http_2.createServer)(async (req, res) => {
|
|
37
|
+
try {
|
|
38
|
+
if (req.url === "/" || req.url === "/favicon.ico") {
|
|
39
|
+
return res.end();
|
|
40
|
+
}
|
|
41
|
+
else if (!validateRequest(req, options.users || [])) {
|
|
42
|
+
res.statusCode = 401;
|
|
43
|
+
return res.end();
|
|
44
|
+
}
|
|
45
|
+
if (log)
|
|
46
|
+
console.info(`> ${req.url}`);
|
|
47
|
+
const fs = new virtual_fs_1.LocalFs({
|
|
48
|
+
backend: options.path ?? ".",
|
|
49
|
+
});
|
|
50
|
+
const { action, params } = parseUrl(req.url);
|
|
51
|
+
if (action === "upload") {
|
|
52
|
+
const [target] = params;
|
|
53
|
+
const file = (0, fs_1.createWriteStream)(fs.resolvePath(target));
|
|
54
|
+
req.pipe(file);
|
|
55
|
+
await new Promise((resolve, reject) => {
|
|
56
|
+
req.on("error", reject);
|
|
57
|
+
file.on("error", reject);
|
|
58
|
+
file.on("close", resolve);
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
else if (action === "download") {
|
|
62
|
+
const [target] = params;
|
|
63
|
+
const file = (0, fs_1.createReadStream)(fs.resolvePath(target));
|
|
64
|
+
file.pipe(res);
|
|
65
|
+
await new Promise((resolve, reject) => {
|
|
66
|
+
req.on("error", reject);
|
|
67
|
+
file.on("error", reject);
|
|
68
|
+
res.on("error", reject);
|
|
69
|
+
res.on("close", resolve);
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
else if (action === "writeFile") {
|
|
73
|
+
const data = await (0, http_1.readRequestData)(req);
|
|
74
|
+
const [target] = params;
|
|
75
|
+
await fs.writeFile(target, data);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
const object = fs[action]?.bind(fs);
|
|
79
|
+
if (!object)
|
|
80
|
+
throw new Error(`Invalid action: ${action}`);
|
|
81
|
+
const json = await object(...params);
|
|
82
|
+
if (json !== undefined)
|
|
83
|
+
res.write(JSON.stringify(json));
|
|
84
|
+
}
|
|
85
|
+
res.end();
|
|
86
|
+
if (log)
|
|
87
|
+
console.info(`<${action}`);
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
if (log)
|
|
91
|
+
console.error(`<${req.url}`, error);
|
|
92
|
+
res.statusCode = 500;
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
exports.createDatatruckServer = createDatatruckServer;
|
package/utils/fs.d.ts
CHANGED
|
@@ -87,18 +87,24 @@ export declare function cpy(options: {
|
|
|
87
87
|
files: number;
|
|
88
88
|
dirs: number;
|
|
89
89
|
}>;
|
|
90
|
+
type ProgressObject = {
|
|
91
|
+
disposed: boolean;
|
|
92
|
+
total: number;
|
|
93
|
+
current: number;
|
|
94
|
+
update: (description: string, path?: string, increment?: boolean) => Promise<void>;
|
|
95
|
+
};
|
|
96
|
+
export declare function createProgress(options: {
|
|
97
|
+
onProgress: (data: Progress) => Promise<void>;
|
|
98
|
+
}): ProgressObject;
|
|
90
99
|
export declare function createFileScanner(options: {
|
|
91
100
|
glob: Options & {
|
|
92
101
|
include: string[];
|
|
93
102
|
};
|
|
94
103
|
onProgress: (data: Progress) => Promise<void>;
|
|
95
|
-
}): Promise<{
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
current: number;
|
|
99
|
-
progress: (description: string, path?: string, increment?: boolean) => Promise<void>;
|
|
104
|
+
}): Promise<ProgressObject & {
|
|
105
|
+
progress: ProgressObject["update"];
|
|
106
|
+
start: (cb?: (entry: Required<Entry>) => any) => Promise<void>;
|
|
100
107
|
end: () => Promise<void>;
|
|
101
|
-
start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
|
|
102
108
|
}>;
|
|
103
109
|
type StreamItem = {
|
|
104
110
|
key: string;
|
|
@@ -121,4 +127,5 @@ export declare function createWriteStreamPool(options: {
|
|
|
121
127
|
export declare function countFileLines(path: string): Promise<number>;
|
|
122
128
|
export declare function fetchData<T>(input: T, onPath?: (input: Exclude<T, string>) => string | undefined): Promise<string | null>;
|
|
123
129
|
export declare function safeRename(oldPath: string, newPath: string): Promise<void>;
|
|
130
|
+
export declare function tryRm(path: string): Promise<void>;
|
|
124
131
|
export {};
|
package/utils/fs.js
CHANGED
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
|
|
6
|
+
exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
|
|
7
7
|
const globalData_1 = __importDefault(require("../globalData"));
|
|
8
8
|
const math_1 = require("./math");
|
|
9
9
|
const path_1 = require("./path");
|
|
@@ -404,57 +404,65 @@ async function cpy(options) {
|
|
|
404
404
|
return stats;
|
|
405
405
|
}
|
|
406
406
|
exports.cpy = cpy;
|
|
407
|
-
|
|
408
|
-
const
|
|
407
|
+
function createProgress(options) {
|
|
408
|
+
const progress = {
|
|
409
409
|
disposed: false,
|
|
410
410
|
total: 0,
|
|
411
411
|
current: 0,
|
|
412
|
-
|
|
413
|
-
if (
|
|
412
|
+
update: async (description, path, increment = true) => {
|
|
413
|
+
if (progress.disposed)
|
|
414
414
|
return;
|
|
415
415
|
if (path && increment)
|
|
416
|
-
|
|
416
|
+
progress.current++;
|
|
417
417
|
await options.onProgress({
|
|
418
418
|
relative: {
|
|
419
419
|
description,
|
|
420
420
|
payload: path,
|
|
421
421
|
},
|
|
422
422
|
absolute: {
|
|
423
|
-
total:
|
|
424
|
-
current:
|
|
425
|
-
percent: (0, math_1.progressPercent)(
|
|
423
|
+
total: progress.total,
|
|
424
|
+
current: progress.current,
|
|
425
|
+
percent: (0, math_1.progressPercent)(progress.total, progress.current),
|
|
426
426
|
},
|
|
427
427
|
});
|
|
428
428
|
},
|
|
429
|
+
};
|
|
430
|
+
return progress;
|
|
431
|
+
}
|
|
432
|
+
exports.createProgress = createProgress;
|
|
433
|
+
async function createFileScanner(options) {
|
|
434
|
+
const progress = createProgress(options);
|
|
435
|
+
Object.assign(progress, {
|
|
436
|
+
progress: progress.update,
|
|
429
437
|
end: async () => {
|
|
430
|
-
if (!
|
|
431
|
-
await
|
|
432
|
-
|
|
438
|
+
if (!progress.disposed)
|
|
439
|
+
await progress.update("Finished");
|
|
440
|
+
progress.disposed = true;
|
|
433
441
|
},
|
|
434
442
|
start: async (cb) => {
|
|
435
443
|
let lastTime = performance.now();
|
|
436
|
-
await
|
|
444
|
+
await progress.update("Scanning files");
|
|
437
445
|
for await (const entry of pathIterator(stream)) {
|
|
438
446
|
if (cb) {
|
|
439
447
|
if (await cb(entry))
|
|
440
|
-
|
|
448
|
+
progress.total++;
|
|
441
449
|
}
|
|
442
450
|
else {
|
|
443
|
-
|
|
451
|
+
progress.total++;
|
|
444
452
|
}
|
|
445
453
|
if (lastTime - performance.now() > 500)
|
|
446
|
-
await
|
|
454
|
+
await progress.update("Scanning files");
|
|
447
455
|
}
|
|
448
|
-
await
|
|
456
|
+
await progress.update("Scanned files");
|
|
449
457
|
},
|
|
450
|
-
};
|
|
458
|
+
});
|
|
451
459
|
const stream = fast_glob_1.default.stream(options.glob.include, {
|
|
452
460
|
dot: true,
|
|
453
461
|
markDirectories: true,
|
|
454
462
|
stats: true,
|
|
455
463
|
...options.glob,
|
|
456
464
|
});
|
|
457
|
-
return
|
|
465
|
+
return progress;
|
|
458
466
|
}
|
|
459
467
|
exports.createFileScanner = createFileScanner;
|
|
460
468
|
function createWriteStreamPool(options) {
|
|
@@ -560,3 +568,10 @@ async function safeRename(oldPath, newPath) {
|
|
|
560
568
|
}
|
|
561
569
|
}
|
|
562
570
|
exports.safeRename = safeRename;
|
|
571
|
+
async function tryRm(path) {
|
|
572
|
+
try {
|
|
573
|
+
await (0, promises_1.rm)(path);
|
|
574
|
+
}
|
|
575
|
+
catch (_) { }
|
|
576
|
+
}
|
|
577
|
+
exports.tryRm = tryRm;
|
package/utils/http.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { IncomingMessage, Server } from "http";
|
|
3
|
+
export declare function closeServer(server: Server): Promise<void>;
|
|
4
|
+
export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
|
|
5
|
+
export declare function fetchJson<T = any>(url: string, options?: {
|
|
6
|
+
headers?: Record<string, string>;
|
|
7
|
+
query?: Record<string, string>;
|
|
8
|
+
}): Promise<T>;
|
|
9
|
+
export declare function post(url: string, data: string, options?: {
|
|
10
|
+
headers?: Record<string, string>;
|
|
11
|
+
query?: Record<string, string>;
|
|
12
|
+
}): Promise<void>;
|
|
13
|
+
export declare function downloadFile(url: string, output: string, options?: {
|
|
14
|
+
headers?: Record<string, string>;
|
|
15
|
+
query?: Record<string, string>;
|
|
16
|
+
timeout?: number;
|
|
17
|
+
}): Promise<void>;
|
|
18
|
+
export declare function uploadFile(url: string, path: string, options?: {
|
|
19
|
+
headers?: Record<string, string>;
|
|
20
|
+
query?: Record<string, string>;
|
|
21
|
+
}): Promise<void>;
|
package/utils/http.js
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.readRequestData = exports.closeServer = void 0;
|
|
4
|
+
const fs_1 = require("fs");
|
|
5
|
+
const promises_1 = require("fs/promises");
|
|
6
|
+
const http_1 = require("http");
|
|
7
|
+
const https_1 = require("https");
|
|
8
|
+
const request = (url, options, callback) => {
|
|
9
|
+
return url.startsWith("https://")
|
|
10
|
+
? (0, https_1.request)(url, options, callback)
|
|
11
|
+
: (0, http_1.request)(url, options, callback);
|
|
12
|
+
};
|
|
13
|
+
function href(inUrl, query) {
|
|
14
|
+
const url = new URL(inUrl);
|
|
15
|
+
for (const key in query || {})
|
|
16
|
+
url.searchParams.set(key, query[key]);
|
|
17
|
+
return url.href;
|
|
18
|
+
}
|
|
19
|
+
async function closeServer(server) {
|
|
20
|
+
await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
|
|
21
|
+
}
|
|
22
|
+
exports.closeServer = closeServer;
|
|
23
|
+
function readRequestData(req) {
|
|
24
|
+
let data;
|
|
25
|
+
return new Promise((resolve, reject) => {
|
|
26
|
+
req
|
|
27
|
+
.on("error", reject)
|
|
28
|
+
.on("data", (chunk) => {
|
|
29
|
+
if (data === undefined)
|
|
30
|
+
data = "";
|
|
31
|
+
data += chunk;
|
|
32
|
+
})
|
|
33
|
+
.on("close", async () => {
|
|
34
|
+
resolve(data);
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
exports.readRequestData = readRequestData;
|
|
39
|
+
async function fetchJson(url, options = {}) {
|
|
40
|
+
return new Promise((resolve, reject) => {
|
|
41
|
+
let data;
|
|
42
|
+
request(href(url, options.query), {
|
|
43
|
+
method: "GET",
|
|
44
|
+
headers: options.headers,
|
|
45
|
+
}, (res) => {
|
|
46
|
+
if (res.statusCode !== 200)
|
|
47
|
+
return reject(new Error(`GET failed: ${res.statusCode} ${res.statusMessage}`));
|
|
48
|
+
res
|
|
49
|
+
.on("data", (chunk) => {
|
|
50
|
+
if (data === undefined)
|
|
51
|
+
data = "";
|
|
52
|
+
data += chunk;
|
|
53
|
+
})
|
|
54
|
+
.on("error", reject)
|
|
55
|
+
.on("close", () => {
|
|
56
|
+
if (data === undefined) {
|
|
57
|
+
resolve(undefined);
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
try {
|
|
61
|
+
resolve(JSON.parse(data));
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
reject(error);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
})
|
|
69
|
+
.on("error", reject)
|
|
70
|
+
.end();
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
exports.fetchJson = fetchJson;
|
|
74
|
+
async function post(url, data, options = {}) {
|
|
75
|
+
await new Promise((resolve, reject) => {
|
|
76
|
+
const req = request(href(url, options.query), { method: "POST", headers: options.headers }, (res) => {
|
|
77
|
+
res.on("error", reject);
|
|
78
|
+
if (res.statusCode !== 200) {
|
|
79
|
+
reject(new Error(`Post failed: ${res.statusCode} ${res.statusMessage}`));
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
resolve();
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
req.on("error", reject);
|
|
86
|
+
req.write(data);
|
|
87
|
+
req.end();
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
exports.post = post;
|
|
91
|
+
async function downloadFile(url, output, options = {}) {
|
|
92
|
+
const timeout = options.timeout ?? 3600 * 100;
|
|
93
|
+
const file = (0, fs_1.createWriteStream)(output);
|
|
94
|
+
await new Promise((resolve, reject) => {
|
|
95
|
+
const req = request(href(url, options.query), {
|
|
96
|
+
headers: options.headers,
|
|
97
|
+
}, (res) => {
|
|
98
|
+
if (res.statusCode === 200) {
|
|
99
|
+
res
|
|
100
|
+
.on("error", async (error) => {
|
|
101
|
+
file.destroy();
|
|
102
|
+
try {
|
|
103
|
+
await (0, promises_1.unlink)(output);
|
|
104
|
+
}
|
|
105
|
+
catch (_) { }
|
|
106
|
+
reject(error);
|
|
107
|
+
})
|
|
108
|
+
.pipe(file);
|
|
109
|
+
file.on("finish", () => {
|
|
110
|
+
file.close((error) => {
|
|
111
|
+
error ? reject(error) : resolve();
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
|
|
117
|
+
}
|
|
118
|
+
}).on("error", async (error) => {
|
|
119
|
+
try {
|
|
120
|
+
await (0, promises_1.unlink)(output);
|
|
121
|
+
}
|
|
122
|
+
catch (_) { }
|
|
123
|
+
reject(error);
|
|
124
|
+
});
|
|
125
|
+
req.setTimeout(timeout, () => {
|
|
126
|
+
req.destroy();
|
|
127
|
+
reject(new Error(`Request timeout after ${timeout / 1000}s`));
|
|
128
|
+
});
|
|
129
|
+
req.end();
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
exports.downloadFile = downloadFile;
|
|
133
|
+
async function uploadFile(url, path, options = {}) {
|
|
134
|
+
const { size } = await (0, promises_1.stat)(path);
|
|
135
|
+
const readStream = (0, fs_1.createReadStream)(path);
|
|
136
|
+
await new Promise((resolve, reject) => {
|
|
137
|
+
const req = request(href(url, options.query), {
|
|
138
|
+
method: "POST",
|
|
139
|
+
headers: {
|
|
140
|
+
...options.headers,
|
|
141
|
+
"Content-length": size,
|
|
142
|
+
},
|
|
143
|
+
}, (res) => {
|
|
144
|
+
if (res.statusCode !== 200) {
|
|
145
|
+
reject(new Error(`Upload failed: ${res.statusCode} ${res.statusMessage}`));
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
resolve();
|
|
149
|
+
}
|
|
150
|
+
}).on("error", reject);
|
|
151
|
+
readStream.on("error", reject).pipe(req);
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
exports.uploadFile = uploadFile;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export declare function resolvePath(path: string): string;
|
|
2
|
+
export type FsOptions = {
|
|
3
|
+
backend: string;
|
|
4
|
+
};
|
|
5
|
+
export declare abstract class AbstractFs {
|
|
6
|
+
readonly options: FsOptions;
|
|
7
|
+
constructor(options: FsOptions);
|
|
8
|
+
resolvePath(path: string): string;
|
|
9
|
+
abstract isLocal(): boolean;
|
|
10
|
+
abstract existsDir(path: string): Promise<boolean>;
|
|
11
|
+
abstract mkdir(path: string): Promise<void>;
|
|
12
|
+
abstract readFile(path: string): Promise<string>;
|
|
13
|
+
abstract rmAll(path: string): Promise<void>;
|
|
14
|
+
abstract readFileIfExists(path: string): Promise<string | undefined>;
|
|
15
|
+
abstract readdir(path: string): Promise<string[]>;
|
|
16
|
+
abstract ensureEmptyDir(path: string): Promise<void>;
|
|
17
|
+
abstract writeFile(path: string, contents: string): Promise<void>;
|
|
18
|
+
abstract upload(url: string, path: string): Promise<void>;
|
|
19
|
+
abstract download(url: string, path: string): Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export declare class LocalFs extends AbstractFs {
|
|
22
|
+
isLocal(): boolean;
|
|
23
|
+
existsDir(path: string): Promise<boolean>;
|
|
24
|
+
mkdir(path: string): Promise<void>;
|
|
25
|
+
ensureEmptyDir(path: string): Promise<void>;
|
|
26
|
+
readFile(path: string): Promise<string>;
|
|
27
|
+
readFileIfExists(inPath: string): Promise<string | undefined>;
|
|
28
|
+
readdir(path: string): Promise<string[]>;
|
|
29
|
+
writeFile(path: string, contents: string): Promise<void>;
|
|
30
|
+
rmAll(path: string): Promise<void>;
|
|
31
|
+
upload(source: string, target: string): Promise<void>;
|
|
32
|
+
download(source: string, target: string): Promise<void>;
|
|
33
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LocalFs = exports.AbstractFs = exports.resolvePath = void 0;
|
|
4
|
+
const fs_1 = require("./fs");
|
|
5
|
+
const promises_1 = require("fs/promises");
|
|
6
|
+
const path_1 = require("path");
|
|
7
|
+
function resolvePath(path) {
|
|
8
|
+
const { pathname } = new URL(`file:///${path}`);
|
|
9
|
+
return pathname;
|
|
10
|
+
}
|
|
11
|
+
exports.resolvePath = resolvePath;
|
|
12
|
+
class AbstractFs {
|
|
13
|
+
options;
|
|
14
|
+
constructor(options) {
|
|
15
|
+
this.options = options;
|
|
16
|
+
}
|
|
17
|
+
resolvePath(path) {
|
|
18
|
+
return (0, path_1.resolve)((0, path_1.join)(this.options.backend ?? ".", resolvePath(path)));
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
exports.AbstractFs = AbstractFs;
|
|
22
|
+
class LocalFs extends AbstractFs {
|
|
23
|
+
isLocal() {
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
async existsDir(path) {
|
|
27
|
+
return (0, fs_1.existsDir)(this.resolvePath(path));
|
|
28
|
+
}
|
|
29
|
+
async mkdir(path) {
|
|
30
|
+
await (0, fs_1.mkdirIfNotExists)(this.resolvePath(path));
|
|
31
|
+
}
|
|
32
|
+
async ensureEmptyDir(path) {
|
|
33
|
+
await (0, fs_1.ensureEmptyDir)(this.resolvePath(path));
|
|
34
|
+
}
|
|
35
|
+
async readFile(path) {
|
|
36
|
+
return (await (0, promises_1.readFile)(this.resolvePath(path))).toString();
|
|
37
|
+
}
|
|
38
|
+
async readFileIfExists(inPath) {
|
|
39
|
+
return (await (0, fs_1.existsFile)(this.resolvePath(inPath)))
|
|
40
|
+
? await this.readFile(inPath)
|
|
41
|
+
: undefined;
|
|
42
|
+
}
|
|
43
|
+
async readdir(path) {
|
|
44
|
+
return await (0, promises_1.readdir)(this.resolvePath(path));
|
|
45
|
+
}
|
|
46
|
+
async writeFile(path, contents) {
|
|
47
|
+
await (0, promises_1.writeFile)(this.resolvePath(path), contents);
|
|
48
|
+
}
|
|
49
|
+
async rmAll(path) {
|
|
50
|
+
await (0, promises_1.rm)(this.resolvePath(path), { recursive: true });
|
|
51
|
+
}
|
|
52
|
+
async upload(source, target) {
|
|
53
|
+
await (0, promises_1.cp)(source, this.resolvePath(target));
|
|
54
|
+
}
|
|
55
|
+
async download(source, target) {
|
|
56
|
+
await (0, promises_1.cp)(this.resolvePath(source), target);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
exports.LocalFs = LocalFs;
|