@datatruck/cli 0.26.2 → 0.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/Action/BackupAction.d.ts +69 -34
  2. package/Action/BackupAction.js +284 -244
  3. package/Action/CleanCacheAction.d.ts +8 -4
  4. package/Action/CleanCacheAction.js +8 -5
  5. package/Action/ConfigAction.d.ts +12 -5
  6. package/Action/ConfigAction.js +14 -18
  7. package/Action/CopyAction.d.ts +49 -0
  8. package/Action/CopyAction.js +144 -0
  9. package/Action/InitAction.d.ts +3 -3
  10. package/Action/InitAction.js +9 -9
  11. package/Action/PruneAction.d.ts +9 -9
  12. package/Action/PruneAction.js +39 -23
  13. package/Action/RestoreAction.d.ts +48 -23
  14. package/Action/RestoreAction.js +158 -195
  15. package/Action/SnapshotsAction.d.ts +8 -8
  16. package/Action/SnapshotsAction.js +8 -8
  17. package/CHANGELOG.md +495 -0
  18. package/Command/BackupCommand.d.ts +6 -4
  19. package/Command/BackupCommand.js +9 -26
  20. package/Command/CleanCacheCommand.d.ts +4 -4
  21. package/Command/CleanCacheCommand.js +26 -5
  22. package/Command/CommandAbstract.d.ts +10 -7
  23. package/Command/CommandAbstract.js +4 -1
  24. package/Command/ConfigCommand.d.ts +6 -9
  25. package/Command/ConfigCommand.js +13 -8
  26. package/Command/CopyCommand.d.ts +15 -0
  27. package/Command/CopyCommand.js +61 -0
  28. package/Command/InitCommand.d.ts +4 -4
  29. package/Command/InitCommand.js +11 -15
  30. package/Command/PruneCommand.d.ts +3 -3
  31. package/Command/PruneCommand.js +13 -12
  32. package/Command/RestoreCommand.js +9 -17
  33. package/Command/SnapshotsCommand.d.ts +4 -4
  34. package/Command/SnapshotsCommand.js +16 -15
  35. package/Command/StartServerCommand.d.ts +6 -0
  36. package/Command/StartServerCommand.js +24 -0
  37. package/Config/Config.d.ts +11 -0
  38. package/Config/Config.js +43 -0
  39. package/Config/PrunePolicyConfig.d.ts +2 -2
  40. package/Factory/CommandFactory.d.ts +29 -33
  41. package/Factory/CommandFactory.js +32 -53
  42. package/Factory/RepositoryFactory.d.ts +1 -1
  43. package/Factory/RepositoryFactory.js +3 -3
  44. package/Factory/TaskFactory.d.ts +1 -1
  45. package/Factory/TaskFactory.js +3 -3
  46. package/Repository/DatatruckRepository.d.ts +17 -16
  47. package/Repository/DatatruckRepository.js +131 -149
  48. package/Repository/GitRepository.d.ts +9 -8
  49. package/Repository/GitRepository.js +22 -25
  50. package/Repository/RepositoryAbstract.d.ts +39 -37
  51. package/Repository/RepositoryAbstract.js +4 -5
  52. package/Repository/ResticRepository.d.ts +9 -8
  53. package/Repository/ResticRepository.js +30 -28
  54. package/Task/GitTask.d.ts +6 -7
  55. package/Task/GitTask.js +24 -30
  56. package/Task/MariadbTask.d.ts +4 -5
  57. package/Task/MariadbTask.js +26 -32
  58. package/Task/MssqlTask.d.ts +5 -3
  59. package/Task/MssqlTask.js +11 -12
  60. package/Task/MysqlDumpTask.d.ts +10 -3
  61. package/Task/MysqlDumpTask.js +107 -31
  62. package/Task/ScriptTask.d.ts +23 -18
  63. package/Task/ScriptTask.js +34 -24
  64. package/Task/SqlDumpTaskAbstract.d.ts +8 -3
  65. package/Task/SqlDumpTaskAbstract.js +31 -19
  66. package/Task/TaskAbstract.d.ts +24 -25
  67. package/Task/TaskAbstract.js +6 -10
  68. package/cli.js +14 -5
  69. package/config.schema.json +124 -3
  70. package/package.json +4 -5
  71. package/utils/DataFormat.d.ts +23 -12
  72. package/utils/DataFormat.js +36 -14
  73. package/utils/cli.d.ts +2 -9
  74. package/utils/cli.js +9 -52
  75. package/utils/datatruck/client.d.ts +24 -0
  76. package/utils/datatruck/client.js +99 -0
  77. package/utils/datatruck/config.d.ts +8 -6
  78. package/utils/datatruck/config.js +18 -3
  79. package/utils/datatruck/paths.d.ts +5 -9
  80. package/utils/datatruck/paths.js +2 -2
  81. package/utils/datatruck/server.d.ts +21 -0
  82. package/utils/datatruck/server.js +96 -0
  83. package/utils/datatruck/snapshot.d.ts +2 -2
  84. package/utils/date.d.ts +7 -3
  85. package/utils/date.js +22 -14
  86. package/utils/fs.d.ts +27 -15
  87. package/utils/fs.js +110 -62
  88. package/utils/http.d.ts +21 -0
  89. package/utils/http.js +154 -0
  90. package/utils/list.d.ts +64 -0
  91. package/utils/list.js +145 -0
  92. package/utils/mysql.d.ts +2 -0
  93. package/utils/mysql.js +21 -2
  94. package/utils/process.d.ts +1 -0
  95. package/utils/process.js +24 -31
  96. package/utils/progress.d.ts +33 -0
  97. package/utils/progress.js +113 -0
  98. package/utils/steps.d.ts +11 -0
  99. package/utils/steps.js +22 -10
  100. package/utils/stream.d.ts +7 -0
  101. package/utils/stream.js +10 -0
  102. package/utils/string.d.ts +0 -1
  103. package/utils/string.js +1 -13
  104. package/utils/tar.d.ts +10 -3
  105. package/utils/tar.js +70 -44
  106. package/utils/temp.d.ts +26 -0
  107. package/utils/temp.js +133 -0
  108. package/utils/virtual-fs.d.ts +37 -0
  109. package/utils/virtual-fs.js +65 -0
  110. package/Action/BackupSessionsAction.d.ts +0 -13
  111. package/Action/BackupSessionsAction.js +0 -18
  112. package/Action/RestoreSessionsAction.d.ts +0 -13
  113. package/Action/RestoreSessionsAction.js +0 -18
  114. package/Command/BackupSessionsCommand.d.ts +0 -12
  115. package/Command/BackupSessionsCommand.js +0 -92
  116. package/Command/RestoreSessionsCommand.d.ts +0 -12
  117. package/Command/RestoreSessionsCommand.js +0 -91
  118. package/Decorator/EntityDecorator.d.ts +0 -11
  119. package/Decorator/EntityDecorator.js +0 -17
  120. package/Entity/BackupSessionEntity.d.ts +0 -6
  121. package/Entity/BackupSessionEntity.js +0 -25
  122. package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
  123. package/Entity/BackupSessionRepositoryEntity.js +0 -25
  124. package/Entity/BackupSessionTaskEntity.d.ts +0 -5
  125. package/Entity/BackupSessionTaskEntity.js +0 -24
  126. package/Entity/CrudEntityAbstract.d.ts +0 -5
  127. package/Entity/CrudEntityAbstract.js +0 -9
  128. package/Entity/RestoreSessionEntity.d.ts +0 -5
  129. package/Entity/RestoreSessionEntity.js +0 -24
  130. package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
  131. package/Entity/RestoreSessionRepositoryEntity.js +0 -25
  132. package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
  133. package/Entity/RestoreSessionTaskEntity.js +0 -24
  134. package/Entity/StateEntityAbstract.d.ts +0 -9
  135. package/Entity/StateEntityAbstract.js +0 -12
  136. package/Factory/EntityFactory.d.ts +0 -6
  137. package/Factory/EntityFactory.js +0 -40
  138. package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
  139. package/SessionDriver/ConsoleSessionDriver.js +0 -208
  140. package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
  141. package/SessionDriver/SessionDriverAbstract.js +0 -28
  142. package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
  143. package/SessionDriver/SqliteSessionDriver.js +0 -173
  144. package/SessionManager/BackupSessionManager.d.ts +0 -45
  145. package/SessionManager/BackupSessionManager.js +0 -218
  146. package/SessionManager/RestoreSessionManager.d.ts +0 -47
  147. package/SessionManager/RestoreSessionManager.js +0 -218
  148. package/SessionManager/SessionManagerAbstract.d.ts +0 -18
  149. package/SessionManager/SessionManagerAbstract.js +0 -36
  150. package/migrations/001-initial.sql +0 -98
  151. package/utils/entity.d.ts +0 -4
  152. package/utils/entity.js +0 -10
package/utils/date.js CHANGED
@@ -3,13 +3,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.createChron = exports.filterByLast = void 0;
7
- const string_1 = require("./string");
6
+ exports.duration = exports.createTimer = exports.filterByLast = void 0;
8
7
  const dayjs_1 = __importDefault(require("dayjs"));
9
8
  const advancedFormat_1 = __importDefault(require("dayjs/plugin/advancedFormat"));
9
+ const duration_1 = __importDefault(require("dayjs/plugin/duration"));
10
10
  const isoWeek_1 = __importDefault(require("dayjs/plugin/isoWeek"));
11
11
  dayjs_1.default.extend(isoWeek_1.default);
12
12
  dayjs_1.default.extend(advancedFormat_1.default);
13
+ dayjs_1.default.extend(duration_1.default);
13
14
  function filterByLast(items, options, reasons) {
14
15
  const filters = {
15
16
  last: { handler: (_, i) => i.toString(), value: options.last },
@@ -73,18 +74,25 @@ function filterByLast(items, options, reasons) {
73
74
  return items.filter((item) => validItems.includes(item));
74
75
  }
75
76
  exports.filterByLast = filterByLast;
76
- function createChron() {
77
- let startTime;
78
- return {
79
- start: () => (startTime = Date.now()),
80
- elapsed: (formatted) => {
81
- if (!startTime)
82
- throw new Error(`Chron was not started`);
83
- const seconds = (Date.now() - startTime) / 1000;
84
- if (formatted)
85
- return (0, string_1.formatSeconds)(seconds);
86
- return seconds;
77
+ function createTimer() {
78
+ let startTime = Date.now();
79
+ let endTime;
80
+ const timer = {
81
+ elapsed: () => (endTime || Date.now()) - startTime,
82
+ check: (ms) => timer.elapsed() > ms,
83
+ stop: () => (endTime = Date.now()),
84
+ reset: (min) => {
85
+ if (typeof min === "number" && !timer.check(min))
86
+ return false;
87
+ startTime = Date.now();
88
+ endTime = undefined;
89
+ return true;
87
90
  },
88
91
  };
92
+ return timer;
89
93
  }
90
- exports.createChron = createChron;
94
+ exports.createTimer = createTimer;
95
+ function duration(ms) {
96
+ return dayjs_1.default.duration(ms, "ms").format("HH:mm:ss");
97
+ }
98
+ exports.duration = duration;
package/utils/fs.d.ts CHANGED
@@ -11,6 +11,8 @@ export declare function isEmptyDir(path: string): Promise<boolean>;
11
11
  export declare function isLocalDir(path: string): boolean;
12
12
  export declare function mkdirIfNotExists(path: string): Promise<string>;
13
13
  export declare function ensureEmptyDir(path: string): Promise<void>;
14
+ export declare function ensureSingleFile(path: string): Promise<string>;
15
+ export declare function ensureExistsDir(path: string): Promise<void>;
14
16
  export declare function safeStat(path: string): Promise<Stats | undefined>;
15
17
  export declare function existsDir(path: string): Promise<boolean>;
16
18
  export declare function existsFile(path: string): Promise<boolean>;
@@ -23,13 +25,7 @@ export declare function parsePackageFile(): {
23
25
  description: string;
24
26
  };
25
27
  export declare function findFile(sourcePath: string, baseName: string, extensions: string[], errorMessage?: string): Promise<string>;
26
- export declare function parentTmpDir(): string;
27
- export declare function sessionTmpDir(): string;
28
- export declare function isTmpDir(path: string): boolean;
29
- export declare function rmTmpDir(input: string | string[]): Promise<void>;
30
- export declare function tmpDir(prefix: string, id?: string): string;
31
28
  export declare function fastFolderSizeAsync(path: string): Promise<number>;
32
- export declare function mkTmpDir(prefix: string, id?: string): Promise<string>;
33
29
  export declare function readPartialFile(path: string, positions: [number, number?]): Promise<string>;
34
30
  export declare function readDir(path: string, optional?: boolean): Promise<string[]>;
35
31
  export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
@@ -60,7 +56,7 @@ export declare function cpy(options: {
60
56
  path: string;
61
57
  basePath: string;
62
58
  };
63
- targetPath: string;
59
+ outPath: string;
64
60
  /**
65
61
  * @default 1
66
62
  */
@@ -87,18 +83,24 @@ export declare function cpy(options: {
87
83
  files: number;
88
84
  dirs: number;
89
85
  }>;
86
+ type ProgressObject = {
87
+ disposed: boolean;
88
+ total: number;
89
+ current: number;
90
+ update: (description: string, path?: string, increment?: boolean) => void;
91
+ };
92
+ export declare function createProgress(options: {
93
+ onProgress: (data: Progress) => void;
94
+ }): ProgressObject;
90
95
  export declare function createFileScanner(options: {
91
96
  glob: Options & {
92
97
  include: string[];
93
98
  };
94
- onProgress: (data: Progress) => Promise<void>;
95
- }): Promise<{
96
- disposed: boolean;
97
- total: number;
98
- current: number;
99
- progress: (description: string, path?: string, increment?: boolean) => Promise<void>;
100
- end: () => Promise<void>;
101
- start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
99
+ onProgress: (data: Progress) => void;
100
+ }): Promise<ProgressObject & {
101
+ progress: ProgressObject["update"];
102
+ start: (cb?: (entry: Required<Entry>) => any) => Promise<void>;
103
+ end: () => void;
102
104
  }>;
103
105
  type StreamItem = {
104
106
  key: string;
@@ -121,4 +123,14 @@ export declare function createWriteStreamPool(options: {
121
123
  export declare function countFileLines(path: string): Promise<number>;
122
124
  export declare function fetchData<T>(input: T, onPath?: (input: Exclude<T, string>) => string | undefined): Promise<string | null>;
123
125
  export declare function safeRename(oldPath: string, newPath: string): Promise<void>;
126
+ export declare function tryRm(path: string): Promise<void>;
127
+ export declare function initEmptyDir(path: string | undefined): Promise<string>;
128
+ export type DiskStats = {
129
+ total: number;
130
+ free: number;
131
+ };
132
+ export declare function fetchDiskStats(path: string): Promise<DiskStats>;
133
+ export declare function checkFreeDiskSpace(stat: DiskStats, inSize: string | number): Promise<void>;
134
+ export declare function ensureFreeDiskSpace(input: string[] | DiskStats, inSize: number | string): Promise<void>;
135
+ export declare function groupFiles(inFiles: string[], suffixes?: string[], gzSuffix?: string): [string[], Record<string, string>];
124
136
  export {};
package/utils/fs.js CHANGED
@@ -3,12 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
3
3
  return (mod && mod.__esModule) ? mod : { "default": mod };
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
7
- const globalData_1 = __importDefault(require("../globalData"));
6
+ exports.groupFiles = exports.ensureFreeDiskSpace = exports.checkFreeDiskSpace = exports.fetchDiskStats = exports.initEmptyDir = exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.fastFolderSizeAsync = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureExistsDir = exports.ensureSingleFile = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
8
7
  const math_1 = require("./math");
9
8
  const path_1 = require("./path");
9
+ const string_1 = require("./string");
10
10
  const async_1 = require("async");
11
- const crypto_1 = require("crypto");
11
+ const bytes_1 = __importDefault(require("bytes"));
12
12
  const fast_folder_size_1 = __importDefault(require("fast-folder-size"));
13
13
  const fast_glob_1 = __importDefault(require("fast-glob"));
14
14
  const fs_1 = require("fs");
@@ -53,6 +53,19 @@ async function ensureEmptyDir(path) {
53
53
  throw new Error(`Dir is not empty: ${path}`);
54
54
  }
55
55
  exports.ensureEmptyDir = ensureEmptyDir;
56
+ async function ensureSingleFile(path) {
57
+ const files = await readDir(path);
58
+ if (files.length !== 1)
59
+ throw new Error(`Dir has not one file: ${files.length}`);
60
+ const [file] = files;
61
+ return (0, path_2.join)(path, file);
62
+ }
63
+ exports.ensureSingleFile = ensureSingleFile;
64
+ async function ensureExistsDir(path) {
65
+ if (!(await existsDir(path)))
66
+ throw new Error(`Dir is not crated: ${path}`);
67
+ }
68
+ exports.ensureExistsDir = ensureExistsDir;
56
69
  async function safeStat(path) {
57
70
  try {
58
71
  return await (0, promises_1.stat)(path);
@@ -116,48 +129,10 @@ async function findFile(sourcePath, baseName, extensions, errorMessage = "Path n
116
129
  return path;
117
130
  }
118
131
  exports.findFile = findFile;
119
- function parentTmpDir() {
120
- return (0, path_2.join)(globalData_1.default.tempDir, "datatruck-temp");
121
- }
122
- exports.parentTmpDir = parentTmpDir;
123
- function sessionTmpDir() {
124
- return (0, path_2.join)(parentTmpDir(), process.pid.toString());
125
- }
126
- exports.sessionTmpDir = sessionTmpDir;
127
- function isTmpDir(path) {
128
- return path.startsWith(sessionTmpDir()) && path.includes("datatruck-temp");
129
- }
130
- exports.isTmpDir = isTmpDir;
131
- async function rmTmpDir(input) {
132
- if (typeof input === "string") {
133
- if (!isTmpDir(input))
134
- throw new Error(`Path is not a temp dir: ${input}`);
135
- await (0, promises_1.rm)(input, {
136
- recursive: true,
137
- });
138
- }
139
- else {
140
- for (const path of input)
141
- await rmTmpDir(path);
142
- }
143
- }
144
- exports.rmTmpDir = rmTmpDir;
145
- function tmpDir(prefix, id) {
146
- if (!id)
147
- id = (0, crypto_1.randomUUID)().slice(0, 8);
148
- return (0, path_2.join)(sessionTmpDir(), `${prefix}-${id}`);
149
- }
150
- exports.tmpDir = tmpDir;
151
132
  async function fastFolderSizeAsync(path) {
152
133
  return (await (0, util_1.promisify)(fast_folder_size_1.default)(path)) || 0;
153
134
  }
154
135
  exports.fastFolderSizeAsync = fastFolderSizeAsync;
155
- async function mkTmpDir(prefix, id) {
156
- const path = tmpDir(prefix, id);
157
- await (0, promises_1.mkdir)(path, { recursive: true });
158
- return path;
159
- }
160
- exports.mkTmpDir = mkTmpDir;
161
136
  async function readPartialFile(path, positions) {
162
137
  let result = "";
163
138
  const statResult = await (0, promises_1.stat)(path);
@@ -316,7 +291,7 @@ async function cpy(options) {
316
291
  const isDir = rawEntryPath.endsWith("/");
317
292
  const entryPath = (0, path_2.normalize)(rawEntryPath);
318
293
  const entrySourcePath = (0, path_2.resolve)((0, path_2.join)(basePath, rawEntryPath));
319
- const entryTargetPath = (0, path_2.resolve)((0, path_2.join)(options.targetPath, rawEntryPath));
294
+ const entryTargetPath = (0, path_2.resolve)((0, path_2.join)(options.outPath, rawEntryPath));
320
295
  const onPathResult = await options?.onPath?.({
321
296
  isDir,
322
297
  entryPath,
@@ -404,57 +379,65 @@ async function cpy(options) {
404
379
  return stats;
405
380
  }
406
381
  exports.cpy = cpy;
407
- async function createFileScanner(options) {
408
- const object = {
382
+ function createProgress(options) {
383
+ const progress = {
409
384
  disposed: false,
410
385
  total: 0,
411
386
  current: 0,
412
- progress: async (description, path, increment = true) => {
413
- if (object.disposed)
387
+ update: async (description, path, increment = true) => {
388
+ if (progress.disposed)
414
389
  return;
415
390
  if (path && increment)
416
- object.current++;
417
- await options.onProgress({
391
+ progress.current++;
392
+ options.onProgress({
418
393
  relative: {
419
394
  description,
420
395
  payload: path,
421
396
  },
422
397
  absolute: {
423
- total: object.total,
424
- current: object.current,
425
- percent: (0, math_1.progressPercent)(object.total, object.current),
398
+ total: progress.total,
399
+ current: progress.current,
400
+ percent: (0, math_1.progressPercent)(progress.total, progress.current),
426
401
  },
427
402
  });
428
403
  },
429
- end: async () => {
430
- if (!object.disposed)
431
- await object.progress("Finished");
432
- object.disposed = true;
404
+ };
405
+ return progress;
406
+ }
407
+ exports.createProgress = createProgress;
408
+ async function createFileScanner(options) {
409
+ const progress = createProgress(options);
410
+ Object.assign(progress, {
411
+ progress: progress.update,
412
+ end: () => {
413
+ if (!progress.disposed)
414
+ progress.update("Finished");
415
+ progress.disposed = true;
433
416
  },
434
417
  start: async (cb) => {
435
418
  let lastTime = performance.now();
436
- await object.progress("Scanning files");
419
+ progress.update("Scanning files");
437
420
  for await (const entry of pathIterator(stream)) {
438
421
  if (cb) {
439
422
  if (await cb(entry))
440
- object.total++;
423
+ progress.total++;
441
424
  }
442
425
  else {
443
- object.total++;
426
+ progress.total++;
444
427
  }
445
428
  if (lastTime - performance.now() > 500)
446
- await object.progress("Scanning files");
429
+ progress.update("Scanning files");
447
430
  }
448
- await object.progress("Scanned files");
431
+ progress.update("Scanned files");
449
432
  },
450
- };
433
+ });
451
434
  const stream = fast_glob_1.default.stream(options.glob.include, {
452
435
  dot: true,
453
436
  markDirectories: true,
454
437
  stats: true,
455
438
  ...options.glob,
456
439
  });
457
- return object;
440
+ return progress;
458
441
  }
459
442
  exports.createFileScanner = createFileScanner;
460
443
  function createWriteStreamPool(options) {
@@ -560,3 +543,68 @@ async function safeRename(oldPath, newPath) {
560
543
  }
561
544
  }
562
545
  exports.safeRename = safeRename;
546
+ async function tryRm(path) {
547
+ try {
548
+ await (0, promises_1.rm)(path);
549
+ }
550
+ catch (_) { }
551
+ }
552
+ exports.tryRm = tryRm;
553
+ async function initEmptyDir(path) {
554
+ if (!path)
555
+ throw new Error(`Path is not defined`);
556
+ await mkdirIfNotExists(path);
557
+ await ensureEmptyDir(path);
558
+ return path;
559
+ }
560
+ exports.initEmptyDir = initEmptyDir;
561
+ async function fetchDiskStats(path) {
562
+ const fs = await (0, promises_1.statfs)(path);
563
+ const total = fs.bsize * fs.blocks;
564
+ const free = fs.bsize * fs.bavail;
565
+ return { total, free };
566
+ }
567
+ exports.fetchDiskStats = fetchDiskStats;
568
+ async function checkFreeDiskSpace(stat, inSize) {
569
+ const humanSize = typeof inSize === "number" ? (0, bytes_1.default)(inSize) : inSize;
570
+ const size = bytes_1.default.parse(inSize);
571
+ if (stat.free < size)
572
+ throw new Error(`Free disk space is less than ${humanSize}: ${(0, bytes_1.default)(stat.free)}/${(0, bytes_1.default)(stat.total)}`);
573
+ }
574
+ exports.checkFreeDiskSpace = checkFreeDiskSpace;
575
+ async function ensureFreeDiskSpace(input, inSize) {
576
+ if (Array.isArray(input)) {
577
+ for (const path of input) {
578
+ const stat = await fetchDiskStats(path);
579
+ await checkFreeDiskSpace(stat, inSize);
580
+ }
581
+ }
582
+ else {
583
+ await checkFreeDiskSpace(input, inSize);
584
+ }
585
+ }
586
+ exports.ensureFreeDiskSpace = ensureFreeDiskSpace;
587
+ function groupFiles(inFiles, suffixes, gzSuffix = ".tar.gz") {
588
+ const compressed = {};
589
+ if (suffixes) {
590
+ const validGzSuffixes = suffixes.map((f) => `${f}${gzSuffix}`);
591
+ inFiles = inFiles.filter((f) => (0, string_1.endsWith)(f, suffixes) || (0, string_1.endsWith)(f, validGzSuffixes));
592
+ }
593
+ const grouped = inFiles.reduce((items, name) => {
594
+ const key = name.endsWith(gzSuffix)
595
+ ? name.slice(0, -gzSuffix.length)
596
+ : name;
597
+ if (!items[key])
598
+ items[key] = [];
599
+ items[key].push(name);
600
+ return items;
601
+ }, {});
602
+ for (const key in grouped) {
603
+ const suffixFile = grouped[key].find((v) => v.endsWith(gzSuffix));
604
+ if (suffixFile) {
605
+ compressed[key] = suffixFile;
606
+ }
607
+ }
608
+ return [Object.keys(grouped), compressed];
609
+ }
610
+ exports.groupFiles = groupFiles;
@@ -0,0 +1,21 @@
1
+ /// <reference types="node" />
2
+ import { IncomingMessage, Server } from "http";
3
+ export declare function closeServer(server: Server): Promise<void>;
4
+ export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
5
+ export declare function fetchJson<T = any>(url: string, options?: {
6
+ headers?: Record<string, string>;
7
+ query?: Record<string, string>;
8
+ }): Promise<T>;
9
+ export declare function post(url: string, data: string, options?: {
10
+ headers?: Record<string, string>;
11
+ query?: Record<string, string>;
12
+ }): Promise<void>;
13
+ export declare function downloadFile(url: string, output: string, options?: {
14
+ headers?: Record<string, string>;
15
+ query?: Record<string, string>;
16
+ timeout?: number;
17
+ }): Promise<void>;
18
+ export declare function uploadFile(url: string, path: string, options?: {
19
+ headers?: Record<string, string>;
20
+ query?: Record<string, string>;
21
+ }): Promise<void>;
package/utils/http.js ADDED
@@ -0,0 +1,154 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.readRequestData = exports.closeServer = void 0;
4
+ const fs_1 = require("fs");
5
+ const promises_1 = require("fs/promises");
6
+ const http_1 = require("http");
7
+ const https_1 = require("https");
8
+ const request = (url, options, callback) => {
9
+ return url.startsWith("https://")
10
+ ? (0, https_1.request)(url, options, callback)
11
+ : (0, http_1.request)(url, options, callback);
12
+ };
13
+ function href(inUrl, query) {
14
+ const url = new URL(inUrl);
15
+ for (const key in query || {})
16
+ url.searchParams.set(key, query[key]);
17
+ return url.href;
18
+ }
19
+ async function closeServer(server) {
20
+ await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
21
+ }
22
+ exports.closeServer = closeServer;
23
+ function readRequestData(req) {
24
+ let data;
25
+ return new Promise((resolve, reject) => {
26
+ req
27
+ .on("error", reject)
28
+ .on("data", (chunk) => {
29
+ if (data === undefined)
30
+ data = "";
31
+ data += chunk;
32
+ })
33
+ .on("close", async () => {
34
+ resolve(data);
35
+ });
36
+ });
37
+ }
38
+ exports.readRequestData = readRequestData;
39
+ async function fetchJson(url, options = {}) {
40
+ return new Promise((resolve, reject) => {
41
+ let data;
42
+ request(href(url, options.query), {
43
+ method: "GET",
44
+ headers: options.headers,
45
+ }, (res) => {
46
+ if (res.statusCode !== 200)
47
+ return reject(new Error(`GET failed: ${res.statusCode} ${res.statusMessage}`));
48
+ res
49
+ .on("data", (chunk) => {
50
+ if (data === undefined)
51
+ data = "";
52
+ data += chunk;
53
+ })
54
+ .on("error", reject)
55
+ .on("close", () => {
56
+ if (data === undefined) {
57
+ resolve(undefined);
58
+ }
59
+ else {
60
+ try {
61
+ resolve(JSON.parse(data));
62
+ }
63
+ catch (error) {
64
+ reject(error);
65
+ }
66
+ }
67
+ });
68
+ })
69
+ .on("error", reject)
70
+ .end();
71
+ });
72
+ }
73
+ exports.fetchJson = fetchJson;
74
+ async function post(url, data, options = {}) {
75
+ await new Promise((resolve, reject) => {
76
+ const req = request(href(url, options.query), { method: "POST", headers: options.headers }, (res) => {
77
+ res.on("error", reject);
78
+ if (res.statusCode !== 200) {
79
+ reject(new Error(`Post failed: ${res.statusCode} ${res.statusMessage}`));
80
+ }
81
+ else {
82
+ resolve();
83
+ }
84
+ });
85
+ req.on("error", reject);
86
+ req.write(data);
87
+ req.end();
88
+ });
89
+ }
90
+ exports.post = post;
91
+ async function downloadFile(url, output, options = {}) {
92
+ const timeout = options.timeout ?? 3600 * 100;
93
+ const file = (0, fs_1.createWriteStream)(output);
94
+ await new Promise((resolve, reject) => {
95
+ const req = request(href(url, options.query), {
96
+ headers: options.headers,
97
+ }, (res) => {
98
+ if (res.statusCode === 200) {
99
+ res
100
+ .on("error", async (error) => {
101
+ file.destroy();
102
+ try {
103
+ await (0, promises_1.unlink)(output);
104
+ }
105
+ catch (_) { }
106
+ reject(error);
107
+ })
108
+ .pipe(file);
109
+ file.on("finish", () => {
110
+ file.close((error) => {
111
+ error ? reject(error) : resolve();
112
+ });
113
+ });
114
+ }
115
+ else {
116
+ reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
117
+ }
118
+ }).on("error", async (error) => {
119
+ try {
120
+ await (0, promises_1.unlink)(output);
121
+ }
122
+ catch (_) { }
123
+ reject(error);
124
+ });
125
+ req.setTimeout(timeout, () => {
126
+ req.destroy();
127
+ reject(new Error(`Request timeout after ${timeout / 1000}s`));
128
+ });
129
+ req.end();
130
+ });
131
+ }
132
+ exports.downloadFile = downloadFile;
133
+ async function uploadFile(url, path, options = {}) {
134
+ const { size } = await (0, promises_1.stat)(path);
135
+ const readStream = (0, fs_1.createReadStream)(path);
136
+ await new Promise((resolve, reject) => {
137
+ const req = request(href(url, options.query), {
138
+ method: "POST",
139
+ headers: {
140
+ ...options.headers,
141
+ "Content-length": size,
142
+ },
143
+ }, (res) => {
144
+ if (res.statusCode !== 200) {
145
+ reject(new Error(`Upload failed: ${res.statusCode} ${res.statusMessage}`));
146
+ }
147
+ else {
148
+ resolve();
149
+ }
150
+ }).on("error", reject);
151
+ readStream.on("error", reject).pipe(req);
152
+ });
153
+ }
154
+ exports.uploadFile = uploadFile;
@@ -0,0 +1,64 @@
1
+ import { Timer } from "./date";
2
+ import { ProgressManager } from "./progress";
3
+ import { Streams } from "./stream";
4
+ import { Listr, ListrGetRendererClassFromValue, ListrLogger, ListrTask, ListrTaskWrapper } from "listr2";
5
+ export declare class List3Logger<Levels extends string = string> extends ListrLogger<Levels> {
6
+ constructor(options?: {
7
+ streams?: Partial<Streams>;
8
+ });
9
+ }
10
+ export type Listr3Context = Record<string, Record<string, any>>;
11
+ type KeyIndex = number | string | (string | number)[];
12
+ type Listr3Task<T extends Listr3Context, K extends keyof T> = {
13
+ key: K;
14
+ keyIndex?: KeyIndex;
15
+ data: T[K];
16
+ title: string | {
17
+ initial: string;
18
+ started?: string;
19
+ failed?: string;
20
+ completed?: string;
21
+ };
22
+ runWrapper?: (cb: () => any) => any;
23
+ run: (task: ListrTaskWrapper<any, any, any>, data: T[K]) => Promise<void | ListrTask[] | Listr | undefined> | void | undefined | ListrTask[] | Listr;
24
+ exitOnError?: boolean;
25
+ enabled?: boolean;
26
+ skip?: boolean;
27
+ };
28
+ type List3TaskResultObject<K, D extends Record<string, any>> = {
29
+ key: K;
30
+ keyIndex?: string;
31
+ data: D;
32
+ elapsed: number;
33
+ error?: Error;
34
+ };
35
+ type SummaryResult = List3TaskResultObject<"summary", {
36
+ errors: number;
37
+ }>;
38
+ export type Listr3TaskResult<T extends Listr3Context> = {
39
+ [K in keyof T]: List3TaskResultObject<K, T[K]>;
40
+ }[keyof T];
41
+ export type Listr3TaskResultEnd<T extends Listr3Context> = Listr3TaskResult<T> | SummaryResult;
42
+ export declare class Listr3<T extends Listr3Context> extends Listr<void, "default", "simple"> {
43
+ readonly $options: {
44
+ streams?: Streams;
45
+ progressManager?: ProgressManager;
46
+ };
47
+ readonly resultMap: Record<string, Listr3TaskResult<T>>;
48
+ readonly resultList: Listr3TaskResult<T>[];
49
+ protected execTimer: Timer;
50
+ constructor($options: {
51
+ streams?: Streams;
52
+ progressManager?: ProgressManager;
53
+ });
54
+ private serializeKeyIndex;
55
+ private createResultIndex;
56
+ result(key: keyof T, keyIndex?: KeyIndex): Listr3TaskResult<T>;
57
+ $task<K extends keyof T>(item: Listr3Task<T, K>): ListrTask;
58
+ $tasks<K extends keyof T>(...items: (Listr3Task<T, K> | ListrTask | false)[]): ListrTask[];
59
+ add(tasks: ListrTask<void, ListrGetRendererClassFromValue<"default">> | ListrTask<void, ListrGetRendererClassFromValue<"default">>[]): this;
60
+ getSummaryResult(): SummaryResult;
61
+ getResult(): (SummaryResult | Listr3TaskResult<T>)[];
62
+ exec(): Promise<(Listr3TaskResult<T> | SummaryResult)[]>;
63
+ }
64
+ export {};