@datatruck/cli 0.26.2 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,6 @@
1
+ import { CommandAbstract } from "./CommandAbstract";
2
+ export type StartServerCommandOptionsType<TResolved = false> = {};
3
+ export declare class StartServerCommand extends CommandAbstract<StartServerCommandOptionsType<false>, StartServerCommandOptionsType<true>> {
4
+ onOptions(): import("../utils/cli").OptionsType<StartServerCommandOptionsType<false>, StartServerCommandOptionsType<true>>;
5
+ onExec(): Promise<number>;
6
+ }
@@ -0,0 +1,24 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.StartServerCommand = void 0;
4
+ const ConfigAction_1 = require("../Action/ConfigAction");
5
+ const server_1 = require("../utils/datatruck/server");
6
+ const CommandAbstract_1 = require("./CommandAbstract");
7
+ class StartServerCommand extends CommandAbstract_1.CommandAbstract {
8
+ onOptions() {
9
+ return this.returnsOptions({});
10
+ }
11
+ async onExec() {
12
+ const config = await ConfigAction_1.ConfigAction.fromGlobalOptions(this.globalOptions);
13
+ const server = (0, server_1.createDatatruckServer)(config.server || {});
14
+ const port = config.server?.listen?.port ?? 8888;
15
+ const address = config.server?.listen?.address ?? "127.0.0.1";
16
+ console.info(`Listening on http://${address}:${port}`);
17
+ await new Promise((resolve, reject) => {
18
+ server.listen(port, address);
19
+ server.on("error", reject);
20
+ });
21
+ return 0;
22
+ }
23
+ }
24
+ exports.StartServerCommand = StartServerCommand;
@@ -1,3 +1,4 @@
1
+ import { DatatruckServerOptions } from "../utils/datatruck/server";
1
2
  import { PackageConfigType } from "./PackageConfig";
2
3
  import { RepositoryConfigType } from "./RepositoryConfig";
3
4
  import type { JSONSchema7 } from "json-schema";
@@ -5,5 +6,6 @@ export type ConfigType = {
5
6
  tempDir?: string;
6
7
  repositories: RepositoryConfigType[];
7
8
  packages: PackageConfigType[];
9
+ server?: DatatruckServerOptions;
8
10
  };
9
11
  export declare const configDefinition: JSONSchema7;
package/Config/Config.js CHANGED
@@ -17,5 +17,31 @@ exports.configDefinition = {
17
17
  type: "array",
18
18
  items: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.package),
19
19
  },
20
+ server: {
21
+ type: "object",
22
+ additionalProperties: false,
23
+ properties: {
24
+ path: { type: "string" },
25
+ users: {
26
+ type: "array",
27
+ items: {
28
+ type: "object",
29
+ additionalProperties: false,
30
+ properties: {
31
+ name: { type: "string" },
32
+ password: { type: "string" },
33
+ },
34
+ },
35
+ },
36
+ listen: {
37
+ type: "object",
38
+ additionalProperties: false,
39
+ properties: {
40
+ port: { type: "integer" },
41
+ address: { type: "string" },
42
+ },
43
+ },
44
+ },
45
+ },
20
46
  },
21
47
  };
@@ -9,6 +9,7 @@ import { PruneCommandOptionsType } from "../Command/PruneCommand";
9
9
  import { RestoreCommandOptionsType } from "../Command/RestoreCommand";
10
10
  import { RestoreSessionsCommandOptionsType } from "../Command/RestoreSessionsCommand";
11
11
  import { SnapshotsCommandLogType, SnapshotsCommandOptionsType } from "../Command/SnapshotsCommand";
12
+ import { StartServerCommandOptionsType } from "../Command/StartServerCommand";
12
13
  export declare enum CommandEnum {
13
14
  config = "config",
14
15
  init = "init",
@@ -18,7 +19,8 @@ export declare enum CommandEnum {
18
19
  backupSessions = "backup-sessions",
19
20
  restore = "restore",
20
21
  restoreSessions = "restore-sessions",
21
- cleanCache = "clean-cache"
22
+ cleanCache = "clean-cache",
23
+ startServer = "start-server"
22
24
  }
23
25
  export type OptionsMapType = {
24
26
  [CommandEnum.config]: ConfigCommandOptionsType;
@@ -30,6 +32,7 @@ export type OptionsMapType = {
30
32
  [CommandEnum.restore]: RestoreCommandOptionsType;
31
33
  [CommandEnum.restoreSessions]: RestoreSessionsCommandOptionsType;
32
34
  [CommandEnum.cleanCache]: CleanCacheActionOptionsType;
35
+ [CommandEnum.startServer]: StartServerCommandOptionsType;
33
36
  };
34
37
  export type LogMapType = {
35
38
  [CommandEnum.config]: ConfigCommandLogType;
@@ -10,6 +10,7 @@ const PruneCommand_1 = require("../Command/PruneCommand");
10
10
  const RestoreCommand_1 = require("../Command/RestoreCommand");
11
11
  const RestoreSessionsCommand_1 = require("../Command/RestoreSessionsCommand");
12
12
  const SnapshotsCommand_1 = require("../Command/SnapshotsCommand");
13
+ const StartServerCommand_1 = require("../Command/StartServerCommand");
13
14
  const AppError_1 = require("../Error/AppError");
14
15
  var CommandEnum;
15
16
  (function (CommandEnum) {
@@ -22,6 +23,7 @@ var CommandEnum;
22
23
  CommandEnum["restore"] = "restore";
23
24
  CommandEnum["restoreSessions"] = "restore-sessions";
24
25
  CommandEnum["cleanCache"] = "clean-cache";
26
+ CommandEnum["startServer"] = "start-server";
25
27
  })(CommandEnum || (exports.CommandEnum = CommandEnum = {}));
26
28
  function CommandFactory(type, globalOptions, options) {
27
29
  const constructor = CommandConstructorFactory(type);
@@ -74,6 +76,7 @@ function makeParseLog(type) {
74
76
  const data = [];
75
77
  const consoleLog = console.log;
76
78
  console.log = console.info = (...items) => {
79
+ consoleLog.bind(console)(...items);
77
80
  data.push(...items);
78
81
  };
79
82
  return function parseLog() {
@@ -110,6 +113,9 @@ function CommandConstructorFactory(type) {
110
113
  else if (type === CommandEnum.cleanCache) {
111
114
  return CleanCacheCommand_1.CleanCacheCommand;
112
115
  }
116
+ else if (type === CommandEnum.startServer) {
117
+ return StartServerCommand_1.StartServerCommand;
118
+ }
113
119
  else {
114
120
  throw new AppError_1.AppError(`Invalid command type: ${type}`);
115
121
  }
@@ -11,10 +11,11 @@ export type MetaDataType = {
11
11
  size: number;
12
12
  tarStats?: Record<string, {
13
13
  files: number;
14
+ size: number;
14
15
  }>;
15
16
  };
16
17
  export type DatatruckRepositoryConfigType = {
17
- outPath: string;
18
+ backend: string;
18
19
  compress?: boolean | CompressOptions;
19
20
  };
20
21
  type PackObject = {
@@ -33,10 +34,11 @@ export declare const datatruckRepositoryDefinition: JSONSchema7;
33
34
  export declare const datatruckPackageRepositoryDefinition: JSONSchema7;
34
35
  export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRepositoryConfigType> {
35
36
  static zipBasenameTpl: string;
36
- static buildSnapshotName(data: {
37
- snapshotId: string;
38
- snapshotDate: string;
39
- packageName: string;
37
+ static buildSnapshotName(snapshot: {
38
+ id: string;
39
+ date: string;
40
+ }, pkg: {
41
+ name: string;
40
42
  }): string;
41
43
  static parseSnapshotName(name: string): {
42
44
  snapshotDate: string;
@@ -44,9 +46,7 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
44
46
  snapshotShortId: string;
45
47
  sourcePath: string;
46
48
  } | null;
47
- protected buildMetaPath(snapshotName: string, packageName: string): string;
48
- static parseMetaData(path: string): Promise<MetaDataType | undefined>;
49
- static stringifyMetaData(data: MetaDataType): string;
49
+ static parseMetaData(data: string): Promise<MetaDataType>;
50
50
  onGetSource(): string;
51
51
  onInit(data: InitDataType): Promise<void>;
52
52
  onPrune(data: PruneDataType): Promise<void>;
@@ -4,6 +4,7 @@ exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exp
4
4
  const AppError_1 = require("../Error/AppError");
5
5
  const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
6
6
  const cli_1 = require("../utils/cli");
7
+ const client_1 = require("../utils/datatruck/client");
7
8
  const paths_1 = require("../utils/datatruck/paths");
8
9
  const fs_1 = require("../utils/fs");
9
10
  const string_1 = require("../utils/string");
@@ -16,10 +17,10 @@ const path_1 = require("path");
16
17
  exports.datatruckRepositoryName = "datatruck";
17
18
  exports.datatruckRepositoryDefinition = {
18
19
  type: "object",
19
- required: ["outPath"],
20
+ required: ["backend"],
20
21
  additionalProperties: false,
21
22
  properties: {
22
- outPath: { type: "string" },
23
+ backend: { type: "string" },
23
24
  compress: {
24
25
  anyOf: [{ type: "boolean" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.compressUtil)],
25
26
  },
@@ -52,10 +53,10 @@ exports.datatruckPackageRepositoryDefinition = {
52
53
  };
53
54
  class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
54
55
  static zipBasenameTpl = `.*.dd.tar.gz`;
55
- static buildSnapshotName(data) {
56
- const date = data.snapshotDate.replace(/:/g, "-");
57
- const pkgName = encodeURIComponent(data.packageName).replace(/%40/g, "@");
58
- const snapshotShortId = data.snapshotId.slice(0, 8);
56
+ static buildSnapshotName(snapshot, pkg) {
57
+ const date = snapshot.date.replace(/:/g, "-");
58
+ const pkgName = encodeURIComponent(pkg.name).replace(/%40/g, "@");
59
+ const snapshotShortId = snapshot.id.slice(0, 8);
59
60
  return `${date}_${pkgName}_${snapshotShortId}`;
60
61
  }
61
62
  static parseSnapshotName(name) {
@@ -69,49 +70,32 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
69
70
  packageName = decodeURIComponent(packageName);
70
71
  return { snapshotDate, packageName, snapshotShortId, sourcePath: name };
71
72
  }
72
- buildMetaPath(snapshotName, packageName) {
73
- return (0, path_1.join)(this.config.outPath, snapshotName, packageName) + ".meta.json";
74
- }
75
- static async parseMetaData(path) {
76
- let contents;
77
- try {
78
- contents = await (0, promises_1.readFile)(path);
79
- }
80
- catch (error) {
81
- if ((0, fs_1.isNotFoundError)(error))
82
- return;
83
- throw error;
84
- }
85
- return JSON.parse(contents.toString());
86
- }
87
- static stringifyMetaData(data) {
88
- return JSON.stringify(data);
73
+ static async parseMetaData(data) {
74
+ return JSON.parse(data.toString());
89
75
  }
90
76
  onGetSource() {
91
- return this.config.outPath;
77
+ return this.config.backend;
92
78
  }
93
79
  async onInit(data) {
94
- await (0, fs_1.mkdirIfNotExists)(this.config.outPath);
80
+ const fs = (0, client_1.createFs)(this.config.backend);
81
+ await fs.mkdir(".");
95
82
  }
96
83
  async onPrune(data) {
97
- const snapshotName = DatatruckRepository.buildSnapshotName({
98
- snapshotId: data.snapshot.id,
99
- snapshotDate: data.snapshot.date,
100
- packageName: data.snapshot.packageName,
84
+ const fs = (0, client_1.createFs)(this.config.backend);
85
+ const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, {
86
+ name: data.snapshot.packageName,
101
87
  });
102
- const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
103
88
  if (data.options.verbose)
104
- (0, cli_1.logExec)(`Deleting ${snapshotPath}`);
105
- if (await (0, fs_1.existsDir)(snapshotPath))
106
- await (0, promises_1.rm)(snapshotPath, {
107
- recursive: true,
108
- });
89
+ (0, cli_1.logExec)(`Deleting ${fs.resolvePath(snapshotName)}`);
90
+ if (await fs.existsDir(snapshotName))
91
+ await fs.rmAll(snapshotName);
109
92
  }
110
93
  async onSnapshots(data) {
111
- if (!(await (0, fs_1.existsDir)(this.config.outPath)))
112
- throw new Error(`Repository (${this.repository.name}) out path does not exist: ${this.config.outPath}`);
113
- const snapshotNames = await (0, fs_1.readDir)(this.config.outPath);
94
+ const fs = (0, client_1.createFs)(this.config.backend);
95
+ if (!(await fs.existsDir(".")))
96
+ throw new Error(`Repository (${this.repository.name}) out path does not exist: ${fs.resolvePath(".")}`);
114
97
  const snapshots = [];
98
+ const snapshotNames = await fs.readdir(".");
115
99
  const packagePatterns = (0, string_1.makePathPatterns)(data.options.packageNames);
116
100
  const taskPatterns = (0, string_1.makePathPatterns)(data.options.packageTaskNames);
117
101
  for (const snapshotName of snapshotNames) {
@@ -124,8 +108,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
124
108
  if (data.options.ids &&
125
109
  !data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
126
110
  continue;
127
- const metaPath = (0, path_1.join)(this.config.outPath, snapshotName, "meta.json");
128
- const meta = await DatatruckRepository.parseMetaData(metaPath);
111
+ const metaData = await fs.readFileIfExists(`${snapshotName}/meta.json`);
112
+ const meta = !!metaData && (await DatatruckRepository.parseMetaData(metaData));
129
113
  if (!meta)
130
114
  continue;
131
115
  if (taskPatterns && !(0, string_1.checkMatch)(meta.task, taskPatterns))
@@ -149,16 +133,15 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
149
133
  return snapshots;
150
134
  }
151
135
  async onBackup(data) {
152
- const snapshotName = DatatruckRepository.buildSnapshotName({
153
- snapshotId: data.snapshot.id,
154
- snapshotDate: data.snapshot.date,
155
- packageName: data.package.name,
156
- });
157
- const outPath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
136
+ const fs = (0, client_1.createFs)(this.config.backend);
137
+ const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
138
+ const outPath = fs.isLocal()
139
+ ? fs.resolvePath(snapshotName)
140
+ : await this.mkTmpDir("datatruck-backup");
158
141
  const pkg = data.package;
159
142
  const sourcePath = data.targetPath ?? pkg.path;
160
143
  (0, assert_1.ok)(sourcePath);
161
- await (0, promises_1.mkdir)(outPath, { recursive: true });
144
+ await fs.mkdir(snapshotName);
162
145
  const backupPathsOptions = {
163
146
  package: data.package,
164
147
  snapshot: data.snapshot,
@@ -224,21 +207,28 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
224
207
  if (includeList) {
225
208
  tarStats[packBasename] = {
226
209
  files: stream.lines(packIndex),
210
+ size: 0,
227
211
  };
212
+ const tarPath = (0, path_1.join)(outPath, packBasename);
228
213
  await (0, tar_1.createTar)({
229
214
  compress: pack.compress,
230
215
  verbose: data.options.verbose,
231
216
  includeList,
232
217
  path: sourcePath,
233
- output: (0, path_1.join)(outPath, packBasename),
218
+ output: tarPath,
234
219
  onEntry: async (data) => await scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
235
220
  });
221
+ tarStats[packBasename].size = (await (0, promises_1.stat)(tarPath)).size;
222
+ if (!fs.isLocal()) {
223
+ await fs.upload(tarPath, `${snapshotName}/${packBasename}`);
224
+ await (0, promises_1.rm)(tarPath);
225
+ }
236
226
  }
237
227
  packIndex++;
238
228
  }
239
229
  await scanner.end();
240
230
  // Meta
241
- const metaPath = `${outPath}/meta.json`;
231
+ const metaPath = `${snapshotName}/meta.json`;
242
232
  const nodePkg = (0, fs_1.parsePackageFile)();
243
233
  const meta = {
244
234
  id: data.snapshot.id,
@@ -247,47 +237,42 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
247
237
  package: data.package.name,
248
238
  task: data.package.task?.name,
249
239
  version: nodePkg.version,
250
- size: await (0, fs_1.fastFolderSizeAsync)(outPath),
240
+ size: Object.values(tarStats).reduce((total, { size }) => total + size, 0),
251
241
  tarStats,
252
242
  };
253
243
  if (data.options.verbose)
254
- (0, cli_1.logExec)(`Writing metadata into ${metaPath}`);
255
- await (0, promises_1.writeFile)(metaPath, DatatruckRepository.stringifyMetaData(meta));
244
+ (0, cli_1.logExec)(`Writing metadata into ${fs.resolvePath(metaPath)}`);
245
+ await fs.writeFile(`${snapshotName}/meta.json`, JSON.stringify(meta));
256
246
  }
257
247
  async onCopyBackup(data) {
258
- const snapshotName = DatatruckRepository.buildSnapshotName({
259
- snapshotId: data.snapshot.id,
260
- snapshotDate: data.snapshot.date,
261
- packageName: data.package.name,
262
- });
263
- const sourcePath = (0, path_1.resolve)((0, path_1.join)(this.config.outPath, snapshotName));
264
- const targetPath = (0, path_1.resolve)((0, path_1.join)(data.mirrorRepositoryConfig.outPath, snapshotName));
248
+ const sourceFs = (0, client_1.createFs)(this.config.backend);
249
+ const targetFs = (0, client_1.createFs)(data.mirrorRepositoryConfig.backend);
250
+ const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
265
251
  if (data.options.verbose)
266
- (0, cli_1.logExec)(`Copying backup files to ${targetPath}`);
267
- await (0, promises_1.mkdir)(targetPath, { recursive: true });
268
- await (0, fs_1.ensureEmptyDir)(targetPath);
269
- const scanner = await (0, fs_1.createFileScanner)({
270
- onProgress: data.onProgress,
271
- glob: {
272
- include: ["**/*"],
273
- cwd: sourcePath,
274
- },
275
- });
276
- const entryPaths = [];
277
- await scanner.start(async (entry) => {
278
- entryPaths.push(entry.path);
279
- return true;
280
- });
281
- for (const entryPath of entryPaths) {
282
- const sourceFile = (0, path_1.join)(sourcePath, entryPath);
283
- const targetFile = (0, path_1.join)(targetPath, entryPath);
284
- await scanner.progress("Copying", entryPath);
285
- await (0, promises_1.mkdir)((0, path_1.dirname)(targetFile), { recursive: true });
286
- await (0, promises_1.cp)(sourceFile, targetFile);
252
+ (0, cli_1.logExec)(`Copying backup files to ${data.mirrorRepositoryConfig.backend}`);
253
+ await targetFs.mkdir(snapshotName);
254
+ await targetFs.ensureEmptyDir(snapshotName);
255
+ const entries = await sourceFs.readdir(snapshotName);
256
+ for (const entry of entries) {
257
+ const sourceEntry = `${snapshotName}/${entry}`;
258
+ if (targetFs.isLocal()) {
259
+ await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
260
+ }
261
+ else {
262
+ const tempDir = await this.mkTmpDir("remote-copy", entry);
263
+ const tempFile = (0, path_1.join)(tempDir, entry);
264
+ try {
265
+ await sourceFs.download(sourceEntry, tempFile);
266
+ await targetFs.upload(tempFile, sourceEntry);
267
+ }
268
+ finally {
269
+ await (0, fs_1.tryRm)(tempFile);
270
+ }
271
+ }
287
272
  }
288
- await scanner.end();
289
273
  }
290
274
  async onRestore(data) {
275
+ const fs = (0, client_1.createFs)(this.config.backend);
291
276
  const relRestorePath = data.targetPath ?? data.package.restorePath;
292
277
  (0, assert_1.ok)(relRestorePath);
293
278
  const restorePath = (0, path_1.resolve)(relRestorePath);
@@ -298,61 +283,41 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
298
283
  });
299
284
  if (!snapshot)
300
285
  throw new AppError_1.AppError("Snapshot not found");
301
- const snapshotName = DatatruckRepository.buildSnapshotName({
302
- snapshotId: data.snapshot.id,
303
- snapshotDate: data.snapshot.date,
304
- packageName: data.package.name,
305
- });
306
- const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
307
- const metaPath = (0, path_1.join)(sourcePath, "meta.json");
308
- const meta = await DatatruckRepository.parseMetaData(metaPath);
309
- const scanner = await (0, fs_1.createFileScanner)({
310
- onProgress: data.onProgress,
311
- glob: {
312
- cwd: sourcePath,
313
- include: ["**/*"],
314
- },
315
- });
316
- const tarFiles = [];
286
+ const snapshotName = DatatruckRepository.buildSnapshotName(snapshot, data.package);
287
+ const meta = await DatatruckRepository.parseMetaData(await fs.readFile(`${snapshotName}/meta.json`));
288
+ const progress = (0, fs_1.createProgress)({ onProgress: data.onProgress });
289
+ await progress.update("Scanning files");
290
+ const entries = (await fs.readdir(snapshotName)).filter((v) => v.endsWith(".tar") || v.endsWith(".tar.gz"));
317
291
  const tarStats = meta?.tarStats || {};
318
- await scanner.start(async (entry) => {
319
- const path = (0, path_1.join)(sourcePath, entry.name);
320
- const isTar = entry.name.endsWith(".tar");
321
- const isTarGz = entry.name.endsWith(".tar.gz");
322
- if (isTar || isTarGz) {
323
- tarFiles.push(path);
324
- if (typeof tarStats[entry.name]?.files === "number") {
325
- scanner.total += tarStats[entry.name].files;
326
- }
327
- else {
328
- scanner.progress("Scanning", entry.name, false);
329
- const selfTarStats = (tarStats[entry.name] = { files: 0 });
330
- await (0, tar_1.listTar)({
331
- input: path,
332
- verbose: data.options.verbose,
333
- onEntry: () => {
334
- scanner.total++;
335
- selfTarStats.files++;
336
- },
337
- });
338
- }
339
- }
340
- return false;
341
- });
292
+ for (const file in tarStats)
293
+ progress.total += tarStats[file].files;
294
+ await progress.update(`Scanned files: ${progress.total}`);
342
295
  if (data.options.verbose)
343
296
  (0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
344
- for (const tarFile of tarFiles) {
345
- const entryName = (0, path_1.basename)(tarFile);
346
- await (0, tar_1.extractTar)({
347
- total: tarStats[entryName].files,
348
- input: tarFile,
349
- output: restorePath,
350
- decompress: tarFile.endsWith(".tar.gz"),
351
- verbose: data.options.verbose,
352
- onEntry: async (data) => await scanner.progress(tarFile.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
353
- });
297
+ for (const entry of entries) {
298
+ let tempEntry;
299
+ try {
300
+ const sourceEntry = `${snapshotName}/${entry}`;
301
+ if (!fs.isLocal()) {
302
+ const tempDir = await this.mkTmpDir("remote-restore", entry);
303
+ tempEntry = `${tempDir}/${entry}`;
304
+ await fs.download(sourceEntry, tempEntry);
305
+ }
306
+ await (0, tar_1.extractTar)({
307
+ total: tarStats[entry].files,
308
+ input: tempEntry ?? fs.resolvePath(sourceEntry),
309
+ output: restorePath,
310
+ decompress: entry.endsWith(".tar.gz"),
311
+ verbose: data.options.verbose,
312
+ onEntry: async (data) => await progress.update(entry.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
313
+ });
314
+ }
315
+ finally {
316
+ if (tempEntry)
317
+ await (0, fs_1.tryRm)(tempEntry);
318
+ }
354
319
  }
355
- await scanner.end();
320
+ await progress.update("Finished");
356
321
  }
357
322
  }
358
323
  exports.DatatruckRepository = DatatruckRepository;
package/cli.js CHANGED
@@ -83,6 +83,7 @@ program.option("-c,--config <path>", "Config path", process.env["DATATRUCK_CONFI
83
83
  program.option("--progress <value>", "Progress type (auto, plain, tty)", "auto");
84
84
  program.option("--progress-interval <ms>", "Progress interval", Number, 1000);
85
85
  program.option("-o,--output-format <format>", "Output format (json, pjson, yaml, table, custom=$, tpl=name)", "table");
86
+ makeCommand(CommandFactory_1.CommandEnum.startServer).alias("start");
86
87
  makeCommand(CommandFactory_1.CommandEnum.config).alias("c");
87
88
  makeCommand(CommandFactory_1.CommandEnum.init).alias("i");
88
89
  makeCommand(CommandFactory_1.CommandEnum.snapshots).alias("s");
@@ -444,11 +444,11 @@
444
444
  "datatruck-repository": {
445
445
  "type": "object",
446
446
  "required": [
447
- "outPath"
447
+ "backend"
448
448
  ],
449
449
  "additionalProperties": false,
450
450
  "properties": {
451
- "outPath": {
451
+ "backend": {
452
452
  "type": "string"
453
453
  },
454
454
  "compress": {
@@ -996,6 +996,42 @@
996
996
  "items": {
997
997
  "$ref": "#/definitions/package"
998
998
  }
999
+ },
1000
+ "server": {
1001
+ "type": "object",
1002
+ "additionalProperties": false,
1003
+ "properties": {
1004
+ "path": {
1005
+ "type": "string"
1006
+ },
1007
+ "users": {
1008
+ "type": "array",
1009
+ "items": {
1010
+ "type": "object",
1011
+ "additionalProperties": false,
1012
+ "properties": {
1013
+ "name": {
1014
+ "type": "string"
1015
+ },
1016
+ "password": {
1017
+ "type": "string"
1018
+ }
1019
+ }
1020
+ }
1021
+ },
1022
+ "listen": {
1023
+ "type": "object",
1024
+ "additionalProperties": false,
1025
+ "properties": {
1026
+ "port": {
1027
+ "type": "integer"
1028
+ },
1029
+ "address": {
1030
+ "type": "string"
1031
+ }
1032
+ }
1033
+ }
1034
+ }
999
1035
  }
1000
1036
  }
1001
1037
  },
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@datatruck/cli",
3
- "version": "0.26.2",
3
+ "version": "0.27.0",
4
4
  "dependencies": {
5
5
  "@supercharge/promise-pool": "^3.1.0",
6
6
  "ajv": "^8.12.0",
@@ -0,0 +1,22 @@
1
+ import { AbstractFs, FsOptions } from "../virtual-fs";
2
+ export declare class RemoteFs extends AbstractFs {
3
+ readonly options: FsOptions;
4
+ protected url: string;
5
+ protected headers: Record<string, string>;
6
+ constructor(options: FsOptions);
7
+ isLocal(): boolean;
8
+ protected fetchJson(name: string, params: any[]): Promise<any>;
9
+ protected post(name: string, params: any[], data: string): Promise<void>;
10
+ existsDir(path: string): Promise<any>;
11
+ mkdir(path: string): Promise<any>;
12
+ readFile(path: string): Promise<any>;
13
+ readdir(path: string): Promise<any>;
14
+ readFileIfExists(path: string): Promise<string | undefined>;
15
+ ensureEmptyDir(path: string): Promise<void>;
16
+ writeFile(path: string, contents: string): Promise<void>;
17
+ rmAll(path: string): Promise<void>;
18
+ upload(source: string, target: string): Promise<void>;
19
+ download(source: string, target: string, timeout?: number): Promise<void>;
20
+ }
21
+ export declare function isRemoteBackend(backend: string): boolean;
22
+ export declare function createFs(backend: string): AbstractFs;
@@ -0,0 +1,96 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createFs = exports.isRemoteBackend = exports.RemoteFs = void 0;
4
+ const http_1 = require("../http");
5
+ const virtual_fs_1 = require("../virtual-fs");
6
+ const server_1 = require("./server");
7
+ class RemoteFs extends virtual_fs_1.AbstractFs {
8
+ options;
9
+ url;
10
+ headers;
11
+ constructor(options) {
12
+ super(options);
13
+ this.options = options;
14
+ const url = new URL(options.backend);
15
+ this.headers = {
16
+ [server_1.headerKey.user]: url.username,
17
+ [server_1.headerKey.password]: url.password,
18
+ };
19
+ url.username = "";
20
+ url.password = "";
21
+ this.url = url.href;
22
+ if (this.url.endsWith("/"))
23
+ this.url = this.url.slice(0, -1);
24
+ }
25
+ isLocal() {
26
+ return false;
27
+ }
28
+ async fetchJson(name, params) {
29
+ return await (0, http_1.fetchJson)(`${this.url}/${name}`, {
30
+ headers: this.headers,
31
+ query: {
32
+ params: JSON.stringify(params),
33
+ },
34
+ });
35
+ }
36
+ async post(name, params, data) {
37
+ return await (0, http_1.post)(`${this.url}/${name}`, data, {
38
+ headers: this.headers,
39
+ query: {
40
+ params: JSON.stringify(params),
41
+ },
42
+ });
43
+ }
44
+ async existsDir(path) {
45
+ return await this.fetchJson("existsDir", [path]);
46
+ }
47
+ async mkdir(path) {
48
+ return await this.fetchJson("mkdir", [path]);
49
+ }
50
+ async readFile(path) {
51
+ return await this.fetchJson("readFile", [path]);
52
+ }
53
+ async readdir(path) {
54
+ return await this.fetchJson("readdir", [path]);
55
+ }
56
+ async readFileIfExists(path) {
57
+ return await this.fetchJson("readFileIfExists", [path]);
58
+ }
59
+ async ensureEmptyDir(path) {
60
+ return await this.fetchJson("readdir", [path]);
61
+ }
62
+ async writeFile(path, contents) {
63
+ await this.post("writeFile", [path], contents);
64
+ }
65
+ async rmAll(path) {
66
+ await this.fetchJson("rmAll", [path]);
67
+ }
68
+ async upload(source, target) {
69
+ await (0, http_1.uploadFile)(`${this.url}/upload`, source, {
70
+ headers: this.headers,
71
+ query: {
72
+ params: JSON.stringify([target]),
73
+ },
74
+ });
75
+ }
76
+ async download(source, target, timeout = 100000) {
77
+ await (0, http_1.downloadFile)(`${this.url}/download`, target, {
78
+ timeout,
79
+ headers: this.headers,
80
+ query: {
81
+ params: JSON.stringify([source]),
82
+ },
83
+ });
84
+ }
85
+ }
86
+ exports.RemoteFs = RemoteFs;
87
+ function isRemoteBackend(backend) {
88
+ return backend.startsWith("http:") || backend.startsWith("https:");
89
+ }
90
+ exports.isRemoteBackend = isRemoteBackend;
91
+ function createFs(backend) {
92
+ return isRemoteBackend(backend)
93
+ ? new RemoteFs({ backend })
94
+ : new virtual_fs_1.LocalFs({ backend });
95
+ }
96
+ exports.createFs = createFs;
@@ -31,18 +31,18 @@ export declare const pkgPathParams: {
31
31
  [name in "temp" | Exclude<keyof ResolvePackagePathParamsType, "path">]: string;
32
32
  };
33
33
  export declare const pkgIncludeParams: {
34
+ action: string;
34
35
  packageName: string;
35
36
  temp: string;
36
37
  snapshotId: string;
37
38
  snapshotDate: string;
38
- action: string;
39
39
  };
40
40
  export declare const pkgExcludeParams: {
41
+ action: string;
41
42
  packageName: string;
42
43
  temp: string;
43
44
  snapshotId: string;
44
45
  snapshotDate: string;
45
- action: string;
46
46
  };
47
47
  export declare const pkgRestorePathParams: {
48
48
  [name in "temp" | keyof ResolvePackagePathParamsType]: string;
@@ -52,33 +52,33 @@ export declare const dbNameParams: {
52
52
  };
53
53
  export declare const params: {
54
54
  pkgPath: {
55
+ action: string;
55
56
  packageName: string;
56
57
  temp: string;
57
58
  snapshotId: string;
58
59
  snapshotDate: string;
59
- action: string;
60
60
  };
61
61
  pkgRestorePath: {
62
62
  path: string;
63
+ action: string;
63
64
  packageName: string;
64
65
  temp: string;
65
66
  snapshotId: string;
66
67
  snapshotDate: string;
67
- action: string;
68
68
  };
69
69
  pkgInclude: {
70
+ action: string;
70
71
  packageName: string;
71
72
  temp: string;
72
73
  snapshotId: string;
73
74
  snapshotDate: string;
74
- action: string;
75
75
  };
76
76
  pkgExclude: {
77
+ action: string;
77
78
  packageName: string;
78
79
  temp: string;
79
80
  snapshotId: string;
80
81
  snapshotDate: string;
81
- action: string;
82
82
  };
83
83
  dbName: {
84
84
  snapshotId: string;
@@ -0,0 +1,21 @@
1
+ /// <reference types="node" />
2
+ import { IncomingMessage } from "http";
3
+ type User = {
4
+ name: string;
5
+ password: string;
6
+ };
7
+ export type DatatruckServerOptions = {
8
+ path?: string;
9
+ log?: boolean;
10
+ listen?: {
11
+ port?: number;
12
+ address?: string;
13
+ };
14
+ users?: User[];
15
+ };
16
+ export declare const headerKey: {
17
+ user: string;
18
+ password: string;
19
+ };
20
+ export declare function createDatatruckServer(options: DatatruckServerOptions): import("node:http").Server<typeof IncomingMessage, typeof import("node:http").ServerResponse>;
21
+ export {};
@@ -0,0 +1,96 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createDatatruckServer = exports.headerKey = void 0;
4
+ const http_1 = require("../http");
5
+ const virtual_fs_1 = require("../virtual-fs");
6
+ const fs_1 = require("fs");
7
+ const http_2 = require("http");
8
+ function parseUrl(inUrl) {
9
+ const url = new URL(`http://127.0.0.1${inUrl}`);
10
+ const inParams = url.searchParams.get("params");
11
+ const action = url.pathname.slice(1);
12
+ if (typeof inParams === "string") {
13
+ const params = JSON.parse(inParams);
14
+ if (!Array.isArray(params))
15
+ throw new Error(`Invalid params`);
16
+ return { action, params };
17
+ }
18
+ else {
19
+ return { action, params: [] };
20
+ }
21
+ }
22
+ exports.headerKey = {
23
+ user: "x-dtt-user",
24
+ password: "x-dtt-password",
25
+ };
26
+ function validateRequest(req, users) {
27
+ const name = req.headers[exports.headerKey.user];
28
+ const password = req.headers[exports.headerKey.password];
29
+ return users.some((user) => user.name.length &&
30
+ user.password.length &&
31
+ user.name === name &&
32
+ user.password === password);
33
+ }
34
+ function createDatatruckServer(options) {
35
+ const log = options.log ?? true;
36
+ return (0, http_2.createServer)(async (req, res) => {
37
+ try {
38
+ if (req.url === "/" || req.url === "/favicon.ico") {
39
+ return res.end();
40
+ }
41
+ else if (!validateRequest(req, options.users || [])) {
42
+ res.statusCode = 401;
43
+ return res.end();
44
+ }
45
+ if (log)
46
+ console.info(`> ${req.url}`);
47
+ const fs = new virtual_fs_1.LocalFs({
48
+ backend: options.path ?? ".",
49
+ });
50
+ const { action, params } = parseUrl(req.url);
51
+ if (action === "upload") {
52
+ const [target] = params;
53
+ const file = (0, fs_1.createWriteStream)(fs.resolvePath(target));
54
+ req.pipe(file);
55
+ await new Promise((resolve, reject) => {
56
+ req.on("error", reject);
57
+ file.on("error", reject);
58
+ file.on("close", resolve);
59
+ });
60
+ }
61
+ else if (action === "download") {
62
+ const [target] = params;
63
+ const file = (0, fs_1.createReadStream)(fs.resolvePath(target));
64
+ file.pipe(res);
65
+ await new Promise((resolve, reject) => {
66
+ req.on("error", reject);
67
+ file.on("error", reject);
68
+ res.on("error", reject);
69
+ res.on("close", resolve);
70
+ });
71
+ }
72
+ else if (action === "writeFile") {
73
+ const data = await (0, http_1.readRequestData)(req);
74
+ const [target] = params;
75
+ await fs.writeFile(target, data);
76
+ }
77
+ else {
78
+ const object = fs[action]?.bind(fs);
79
+ if (!object)
80
+ throw new Error(`Invalid action: ${action}`);
81
+ const json = await object(...params);
82
+ if (json !== undefined)
83
+ res.write(JSON.stringify(json));
84
+ }
85
+ res.end();
86
+ if (log)
87
+ console.info(`<${action}`);
88
+ }
89
+ catch (error) {
90
+ if (log)
91
+ console.error(`<${req.url}`, error);
92
+ res.statusCode = 500;
93
+ }
94
+ });
95
+ }
96
+ exports.createDatatruckServer = createDatatruckServer;
package/utils/fs.d.ts CHANGED
@@ -87,18 +87,24 @@ export declare function cpy(options: {
87
87
  files: number;
88
88
  dirs: number;
89
89
  }>;
90
+ type ProgressObject = {
91
+ disposed: boolean;
92
+ total: number;
93
+ current: number;
94
+ update: (description: string, path?: string, increment?: boolean) => Promise<void>;
95
+ };
96
+ export declare function createProgress(options: {
97
+ onProgress: (data: Progress) => Promise<void>;
98
+ }): ProgressObject;
90
99
  export declare function createFileScanner(options: {
91
100
  glob: Options & {
92
101
  include: string[];
93
102
  };
94
103
  onProgress: (data: Progress) => Promise<void>;
95
- }): Promise<{
96
- disposed: boolean;
97
- total: number;
98
- current: number;
99
- progress: (description: string, path?: string, increment?: boolean) => Promise<void>;
104
+ }): Promise<ProgressObject & {
105
+ progress: ProgressObject["update"];
106
+ start: (cb?: (entry: Required<Entry>) => any) => Promise<void>;
100
107
  end: () => Promise<void>;
101
- start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
102
108
  }>;
103
109
  type StreamItem = {
104
110
  key: string;
@@ -121,4 +127,5 @@ export declare function createWriteStreamPool(options: {
121
127
  export declare function countFileLines(path: string): Promise<number>;
122
128
  export declare function fetchData<T>(input: T, onPath?: (input: Exclude<T, string>) => string | undefined): Promise<string | null>;
123
129
  export declare function safeRename(oldPath: string, newPath: string): Promise<void>;
130
+ export declare function tryRm(path: string): Promise<void>;
124
131
  export {};
package/utils/fs.js CHANGED
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
6
+ exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
7
7
  const globalData_1 = __importDefault(require("../globalData"));
8
8
  const math_1 = require("./math");
9
9
  const path_1 = require("./path");
@@ -404,57 +404,65 @@ async function cpy(options) {
404
404
  return stats;
405
405
  }
406
406
  exports.cpy = cpy;
407
- async function createFileScanner(options) {
408
- const object = {
407
+ function createProgress(options) {
408
+ const progress = {
409
409
  disposed: false,
410
410
  total: 0,
411
411
  current: 0,
412
- progress: async (description, path, increment = true) => {
413
- if (object.disposed)
412
+ update: async (description, path, increment = true) => {
413
+ if (progress.disposed)
414
414
  return;
415
415
  if (path && increment)
416
- object.current++;
416
+ progress.current++;
417
417
  await options.onProgress({
418
418
  relative: {
419
419
  description,
420
420
  payload: path,
421
421
  },
422
422
  absolute: {
423
- total: object.total,
424
- current: object.current,
425
- percent: (0, math_1.progressPercent)(object.total, object.current),
423
+ total: progress.total,
424
+ current: progress.current,
425
+ percent: (0, math_1.progressPercent)(progress.total, progress.current),
426
426
  },
427
427
  });
428
428
  },
429
+ };
430
+ return progress;
431
+ }
432
+ exports.createProgress = createProgress;
433
+ async function createFileScanner(options) {
434
+ const progress = createProgress(options);
435
+ Object.assign(progress, {
436
+ progress: progress.update,
429
437
  end: async () => {
430
- if (!object.disposed)
431
- await object.progress("Finished");
432
- object.disposed = true;
438
+ if (!progress.disposed)
439
+ await progress.update("Finished");
440
+ progress.disposed = true;
433
441
  },
434
442
  start: async (cb) => {
435
443
  let lastTime = performance.now();
436
- await object.progress("Scanning files");
444
+ await progress.update("Scanning files");
437
445
  for await (const entry of pathIterator(stream)) {
438
446
  if (cb) {
439
447
  if (await cb(entry))
440
- object.total++;
448
+ progress.total++;
441
449
  }
442
450
  else {
443
- object.total++;
451
+ progress.total++;
444
452
  }
445
453
  if (lastTime - performance.now() > 500)
446
- await object.progress("Scanning files");
454
+ await progress.update("Scanning files");
447
455
  }
448
- await object.progress("Scanned files");
456
+ await progress.update("Scanned files");
449
457
  },
450
- };
458
+ });
451
459
  const stream = fast_glob_1.default.stream(options.glob.include, {
452
460
  dot: true,
453
461
  markDirectories: true,
454
462
  stats: true,
455
463
  ...options.glob,
456
464
  });
457
- return object;
465
+ return progress;
458
466
  }
459
467
  exports.createFileScanner = createFileScanner;
460
468
  function createWriteStreamPool(options) {
@@ -560,3 +568,10 @@ async function safeRename(oldPath, newPath) {
560
568
  }
561
569
  }
562
570
  exports.safeRename = safeRename;
571
+ async function tryRm(path) {
572
+ try {
573
+ await (0, promises_1.rm)(path);
574
+ }
575
+ catch (_) { }
576
+ }
577
+ exports.tryRm = tryRm;
@@ -0,0 +1,21 @@
1
+ /// <reference types="node" />
2
+ import { IncomingMessage, Server } from "http";
3
+ export declare function closeServer(server: Server): Promise<void>;
4
+ export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
5
+ export declare function fetchJson<T = any>(url: string, options?: {
6
+ headers?: Record<string, string>;
7
+ query?: Record<string, string>;
8
+ }): Promise<T>;
9
+ export declare function post(url: string, data: string, options?: {
10
+ headers?: Record<string, string>;
11
+ query?: Record<string, string>;
12
+ }): Promise<void>;
13
+ export declare function downloadFile(url: string, output: string, options?: {
14
+ headers?: Record<string, string>;
15
+ query?: Record<string, string>;
16
+ timeout?: number;
17
+ }): Promise<void>;
18
+ export declare function uploadFile(url: string, path: string, options?: {
19
+ headers?: Record<string, string>;
20
+ query?: Record<string, string>;
21
+ }): Promise<void>;
package/utils/http.js ADDED
@@ -0,0 +1,154 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.readRequestData = exports.closeServer = void 0;
4
+ const fs_1 = require("fs");
5
+ const promises_1 = require("fs/promises");
6
+ const http_1 = require("http");
7
+ const https_1 = require("https");
8
+ const request = (url, options, callback) => {
9
+ return url.startsWith("https://")
10
+ ? (0, https_1.request)(url, options, callback)
11
+ : (0, http_1.request)(url, options, callback);
12
+ };
13
+ function href(inUrl, query) {
14
+ const url = new URL(inUrl);
15
+ for (const key in query || {})
16
+ url.searchParams.set(key, query[key]);
17
+ return url.href;
18
+ }
19
+ async function closeServer(server) {
20
+ await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
21
+ }
22
+ exports.closeServer = closeServer;
23
+ function readRequestData(req) {
24
+ let data;
25
+ return new Promise((resolve, reject) => {
26
+ req
27
+ .on("error", reject)
28
+ .on("data", (chunk) => {
29
+ if (data === undefined)
30
+ data = "";
31
+ data += chunk;
32
+ })
33
+ .on("close", async () => {
34
+ resolve(data);
35
+ });
36
+ });
37
+ }
38
+ exports.readRequestData = readRequestData;
39
+ async function fetchJson(url, options = {}) {
40
+ return new Promise((resolve, reject) => {
41
+ let data;
42
+ request(href(url, options.query), {
43
+ method: "GET",
44
+ headers: options.headers,
45
+ }, (res) => {
46
+ if (res.statusCode !== 200)
47
+ return reject(new Error(`GET failed: ${res.statusCode} ${res.statusMessage}`));
48
+ res
49
+ .on("data", (chunk) => {
50
+ if (data === undefined)
51
+ data = "";
52
+ data += chunk;
53
+ })
54
+ .on("error", reject)
55
+ .on("close", () => {
56
+ if (data === undefined) {
57
+ resolve(undefined);
58
+ }
59
+ else {
60
+ try {
61
+ resolve(JSON.parse(data));
62
+ }
63
+ catch (error) {
64
+ reject(error);
65
+ }
66
+ }
67
+ });
68
+ })
69
+ .on("error", reject)
70
+ .end();
71
+ });
72
+ }
73
+ exports.fetchJson = fetchJson;
74
+ async function post(url, data, options = {}) {
75
+ await new Promise((resolve, reject) => {
76
+ const req = request(href(url, options.query), { method: "POST", headers: options.headers }, (res) => {
77
+ res.on("error", reject);
78
+ if (res.statusCode !== 200) {
79
+ reject(new Error(`Post failed: ${res.statusCode} ${res.statusMessage}`));
80
+ }
81
+ else {
82
+ resolve();
83
+ }
84
+ });
85
+ req.on("error", reject);
86
+ req.write(data);
87
+ req.end();
88
+ });
89
+ }
90
+ exports.post = post;
91
+ async function downloadFile(url, output, options = {}) {
92
+ const timeout = options.timeout ?? 3600 * 100;
93
+ const file = (0, fs_1.createWriteStream)(output);
94
+ await new Promise((resolve, reject) => {
95
+ const req = request(href(url, options.query), {
96
+ headers: options.headers,
97
+ }, (res) => {
98
+ if (res.statusCode === 200) {
99
+ res
100
+ .on("error", async (error) => {
101
+ file.destroy();
102
+ try {
103
+ await (0, promises_1.unlink)(output);
104
+ }
105
+ catch (_) { }
106
+ reject(error);
107
+ })
108
+ .pipe(file);
109
+ file.on("finish", () => {
110
+ file.close((error) => {
111
+ error ? reject(error) : resolve();
112
+ });
113
+ });
114
+ }
115
+ else {
116
+ reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
117
+ }
118
+ }).on("error", async (error) => {
119
+ try {
120
+ await (0, promises_1.unlink)(output);
121
+ }
122
+ catch (_) { }
123
+ reject(error);
124
+ });
125
+ req.setTimeout(timeout, () => {
126
+ req.destroy();
127
+ reject(new Error(`Request timeout after ${timeout / 1000}s`));
128
+ });
129
+ req.end();
130
+ });
131
+ }
132
+ exports.downloadFile = downloadFile;
133
+ async function uploadFile(url, path, options = {}) {
134
+ const { size } = await (0, promises_1.stat)(path);
135
+ const readStream = (0, fs_1.createReadStream)(path);
136
+ await new Promise((resolve, reject) => {
137
+ const req = request(href(url, options.query), {
138
+ method: "POST",
139
+ headers: {
140
+ ...options.headers,
141
+ "Content-length": size,
142
+ },
143
+ }, (res) => {
144
+ if (res.statusCode !== 200) {
145
+ reject(new Error(`Upload failed: ${res.statusCode} ${res.statusMessage}`));
146
+ }
147
+ else {
148
+ resolve();
149
+ }
150
+ }).on("error", reject);
151
+ readStream.on("error", reject).pipe(req);
152
+ });
153
+ }
154
+ exports.uploadFile = uploadFile;
@@ -0,0 +1,33 @@
1
+ export declare function resolvePath(path: string): string;
2
+ export type FsOptions = {
3
+ backend: string;
4
+ };
5
+ export declare abstract class AbstractFs {
6
+ readonly options: FsOptions;
7
+ constructor(options: FsOptions);
8
+ resolvePath(path: string): string;
9
+ abstract isLocal(): boolean;
10
+ abstract existsDir(path: string): Promise<boolean>;
11
+ abstract mkdir(path: string): Promise<void>;
12
+ abstract readFile(path: string): Promise<string>;
13
+ abstract rmAll(path: string): Promise<void>;
14
+ abstract readFileIfExists(path: string): Promise<string | undefined>;
15
+ abstract readdir(path: string): Promise<string[]>;
16
+ abstract ensureEmptyDir(path: string): Promise<void>;
17
+ abstract writeFile(path: string, contents: string): Promise<void>;
18
+ abstract upload(url: string, path: string): Promise<void>;
19
+ abstract download(url: string, path: string): Promise<void>;
20
+ }
21
+ export declare class LocalFs extends AbstractFs {
22
+ isLocal(): boolean;
23
+ existsDir(path: string): Promise<boolean>;
24
+ mkdir(path: string): Promise<void>;
25
+ ensureEmptyDir(path: string): Promise<void>;
26
+ readFile(path: string): Promise<string>;
27
+ readFileIfExists(inPath: string): Promise<string | undefined>;
28
+ readdir(path: string): Promise<string[]>;
29
+ writeFile(path: string, contents: string): Promise<void>;
30
+ rmAll(path: string): Promise<void>;
31
+ upload(source: string, target: string): Promise<void>;
32
+ download(source: string, target: string): Promise<void>;
33
+ }
@@ -0,0 +1,59 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.LocalFs = exports.AbstractFs = exports.resolvePath = void 0;
4
+ const fs_1 = require("./fs");
5
+ const promises_1 = require("fs/promises");
6
+ const path_1 = require("path");
7
+ function resolvePath(path) {
8
+ const { pathname } = new URL(`file:///${path}`);
9
+ return pathname;
10
+ }
11
+ exports.resolvePath = resolvePath;
12
+ class AbstractFs {
13
+ options;
14
+ constructor(options) {
15
+ this.options = options;
16
+ }
17
+ resolvePath(path) {
18
+ return (0, path_1.resolve)((0, path_1.join)(this.options.backend ?? ".", resolvePath(path)));
19
+ }
20
+ }
21
+ exports.AbstractFs = AbstractFs;
22
+ class LocalFs extends AbstractFs {
23
+ isLocal() {
24
+ return true;
25
+ }
26
+ async existsDir(path) {
27
+ return (0, fs_1.existsDir)(this.resolvePath(path));
28
+ }
29
+ async mkdir(path) {
30
+ await (0, fs_1.mkdirIfNotExists)(this.resolvePath(path));
31
+ }
32
+ async ensureEmptyDir(path) {
33
+ await (0, fs_1.ensureEmptyDir)(this.resolvePath(path));
34
+ }
35
+ async readFile(path) {
36
+ return (await (0, promises_1.readFile)(this.resolvePath(path))).toString();
37
+ }
38
+ async readFileIfExists(inPath) {
39
+ return (await (0, fs_1.existsFile)(this.resolvePath(inPath)))
40
+ ? await this.readFile(inPath)
41
+ : undefined;
42
+ }
43
+ async readdir(path) {
44
+ return await (0, promises_1.readdir)(this.resolvePath(path));
45
+ }
46
+ async writeFile(path, contents) {
47
+ await (0, promises_1.writeFile)(this.resolvePath(path), contents);
48
+ }
49
+ async rmAll(path) {
50
+ await (0, promises_1.rm)(this.resolvePath(path), { recursive: true });
51
+ }
52
+ async upload(source, target) {
53
+ await (0, promises_1.cp)(source, this.resolvePath(target));
54
+ }
55
+ async download(source, target) {
56
+ await (0, promises_1.cp)(this.resolvePath(source), target);
57
+ }
58
+ }
59
+ exports.LocalFs = LocalFs;