@datatruck/cli 0.27.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/Action/BackupAction.d.ts +75 -34
  2. package/Action/BackupAction.js +302 -246
  3. package/Action/CleanCacheAction.d.ts +8 -4
  4. package/Action/CleanCacheAction.js +8 -5
  5. package/Action/ConfigAction.d.ts +12 -5
  6. package/Action/ConfigAction.js +14 -18
  7. package/Action/CopyAction.d.ts +51 -0
  8. package/Action/CopyAction.js +165 -0
  9. package/Action/InitAction.d.ts +3 -3
  10. package/Action/InitAction.js +9 -9
  11. package/Action/PruneAction.d.ts +10 -16
  12. package/Action/PruneAction.js +37 -34
  13. package/Action/RestoreAction.d.ts +49 -24
  14. package/Action/RestoreAction.js +164 -195
  15. package/Action/SnapshotsAction.d.ts +8 -8
  16. package/Action/SnapshotsAction.js +9 -9
  17. package/CHANGELOG.md +513 -0
  18. package/Command/BackupCommand.d.ts +7 -4
  19. package/Command/BackupCommand.js +14 -26
  20. package/Command/CleanCacheCommand.d.ts +4 -4
  21. package/Command/CleanCacheCommand.js +26 -5
  22. package/Command/CommandAbstract.d.ts +10 -7
  23. package/Command/CommandAbstract.js +4 -1
  24. package/Command/ConfigCommand.d.ts +6 -9
  25. package/Command/ConfigCommand.js +13 -8
  26. package/Command/CopyCommand.d.ts +16 -0
  27. package/Command/CopyCommand.js +66 -0
  28. package/Command/InitCommand.d.ts +4 -4
  29. package/Command/InitCommand.js +13 -17
  30. package/Command/PruneCommand.d.ts +4 -10
  31. package/Command/PruneCommand.js +13 -12
  32. package/Command/RestoreCommand.d.ts +1 -1
  33. package/Command/RestoreCommand.js +13 -21
  34. package/Command/SnapshotsCommand.d.ts +4 -4
  35. package/Command/SnapshotsCommand.js +16 -15
  36. package/Command/StartServerCommand.d.ts +3 -3
  37. package/Config/Config.d.ts +11 -0
  38. package/Config/Config.js +18 -0
  39. package/Config/PrunePolicyConfig.d.ts +2 -2
  40. package/Factory/CommandFactory.d.ts +27 -34
  41. package/Factory/CommandFactory.js +27 -54
  42. package/Factory/RepositoryFactory.d.ts +1 -1
  43. package/Factory/RepositoryFactory.js +3 -3
  44. package/Factory/TaskFactory.d.ts +1 -1
  45. package/Factory/TaskFactory.js +3 -3
  46. package/Repository/DatatruckRepository.d.ts +10 -8
  47. package/Repository/DatatruckRepository.js +47 -25
  48. package/Repository/GitRepository.d.ts +9 -8
  49. package/Repository/GitRepository.js +22 -25
  50. package/Repository/RepositoryAbstract.d.ts +39 -37
  51. package/Repository/RepositoryAbstract.js +4 -5
  52. package/Repository/ResticRepository.d.ts +9 -8
  53. package/Repository/ResticRepository.js +30 -28
  54. package/Task/GitTask.d.ts +6 -7
  55. package/Task/GitTask.js +24 -30
  56. package/Task/MariadbTask.d.ts +4 -5
  57. package/Task/MariadbTask.js +26 -32
  58. package/Task/MssqlTask.d.ts +5 -3
  59. package/Task/MssqlTask.js +11 -12
  60. package/Task/MysqlDumpTask.d.ts +10 -3
  61. package/Task/MysqlDumpTask.js +102 -32
  62. package/Task/ScriptTask.d.ts +23 -18
  63. package/Task/ScriptTask.js +34 -24
  64. package/Task/SqlDumpTaskAbstract.d.ts +8 -3
  65. package/Task/SqlDumpTaskAbstract.js +32 -20
  66. package/Task/TaskAbstract.d.ts +24 -25
  67. package/Task/TaskAbstract.js +6 -10
  68. package/cli.js +13 -5
  69. package/config.schema.json +89 -1
  70. package/package.json +4 -5
  71. package/utils/DataFormat.d.ts +23 -12
  72. package/utils/DataFormat.js +36 -14
  73. package/utils/cli.d.ts +3 -9
  74. package/utils/cli.js +19 -55
  75. package/utils/crypto.d.ts +1 -0
  76. package/utils/crypto.js +15 -0
  77. package/utils/datatruck/client.d.ts +2 -0
  78. package/utils/datatruck/client.js +3 -0
  79. package/utils/datatruck/config.d.ts +2 -0
  80. package/utils/datatruck/config.js +18 -3
  81. package/utils/datatruck/paths.d.ts +5 -9
  82. package/utils/datatruck/paths.js +2 -2
  83. package/utils/datatruck/snapshot.d.ts +5 -2
  84. package/utils/datatruck/snapshot.js +12 -22
  85. package/utils/date.d.ts +21 -4
  86. package/utils/date.js +46 -24
  87. package/utils/fs.d.ts +16 -11
  88. package/utils/fs.js +81 -48
  89. package/utils/list.d.ts +64 -0
  90. package/utils/list.js +145 -0
  91. package/utils/mysql.d.ts +2 -0
  92. package/utils/mysql.js +21 -2
  93. package/utils/process.d.ts +1 -0
  94. package/utils/process.js +24 -31
  95. package/utils/progress.d.ts +33 -0
  96. package/utils/progress.js +113 -0
  97. package/utils/steps.d.ts +11 -0
  98. package/utils/steps.js +22 -10
  99. package/utils/stream.d.ts +7 -0
  100. package/utils/stream.js +10 -0
  101. package/utils/string.d.ts +0 -1
  102. package/utils/string.js +1 -13
  103. package/utils/tar.d.ts +10 -3
  104. package/utils/tar.js +73 -45
  105. package/utils/temp.d.ts +26 -0
  106. package/utils/temp.js +133 -0
  107. package/utils/virtual-fs.d.ts +6 -2
  108. package/utils/virtual-fs.js +6 -0
  109. package/Action/BackupSessionsAction.d.ts +0 -13
  110. package/Action/BackupSessionsAction.js +0 -18
  111. package/Action/RestoreSessionsAction.d.ts +0 -13
  112. package/Action/RestoreSessionsAction.js +0 -18
  113. package/Command/BackupSessionsCommand.d.ts +0 -12
  114. package/Command/BackupSessionsCommand.js +0 -92
  115. package/Command/RestoreSessionsCommand.d.ts +0 -12
  116. package/Command/RestoreSessionsCommand.js +0 -91
  117. package/Decorator/EntityDecorator.d.ts +0 -11
  118. package/Decorator/EntityDecorator.js +0 -17
  119. package/Entity/BackupSessionEntity.d.ts +0 -6
  120. package/Entity/BackupSessionEntity.js +0 -25
  121. package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
  122. package/Entity/BackupSessionRepositoryEntity.js +0 -25
  123. package/Entity/BackupSessionTaskEntity.d.ts +0 -5
  124. package/Entity/BackupSessionTaskEntity.js +0 -24
  125. package/Entity/CrudEntityAbstract.d.ts +0 -5
  126. package/Entity/CrudEntityAbstract.js +0 -9
  127. package/Entity/RestoreSessionEntity.d.ts +0 -5
  128. package/Entity/RestoreSessionEntity.js +0 -24
  129. package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
  130. package/Entity/RestoreSessionRepositoryEntity.js +0 -25
  131. package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
  132. package/Entity/RestoreSessionTaskEntity.js +0 -24
  133. package/Entity/StateEntityAbstract.d.ts +0 -9
  134. package/Entity/StateEntityAbstract.js +0 -12
  135. package/Factory/EntityFactory.d.ts +0 -6
  136. package/Factory/EntityFactory.js +0 -40
  137. package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
  138. package/SessionDriver/ConsoleSessionDriver.js +0 -208
  139. package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
  140. package/SessionDriver/SessionDriverAbstract.js +0 -28
  141. package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
  142. package/SessionDriver/SqliteSessionDriver.js +0 -173
  143. package/SessionManager/BackupSessionManager.d.ts +0 -45
  144. package/SessionManager/BackupSessionManager.js +0 -218
  145. package/SessionManager/RestoreSessionManager.d.ts +0 -47
  146. package/SessionManager/RestoreSessionManager.js +0 -218
  147. package/SessionManager/SessionManagerAbstract.d.ts +0 -18
  148. package/SessionManager/SessionManagerAbstract.js +0 -36
  149. package/migrations/001-initial.sql +0 -98
  150. package/utils/entity.d.ts +0 -4
  151. package/utils/entity.js +0 -10
package/utils/progress.js CHANGED
@@ -1,2 +1,115 @@
1
1
  "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
2
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.renderProgressStats = exports.renderProgress = exports.ProgressManager = void 0;
7
+ const cli_1 = require("./cli");
8
+ const date_1 = require("./date");
9
+ const bytes_1 = __importDefault(require("bytes"));
10
+ const chalk_1 = require("chalk");
11
+ const readline_1 = require("readline");
12
+ class ProgressManager {
13
+ options;
14
+ timer = (0, date_1.createTimer)();
15
+ interval = (0, date_1.createTimer)();
16
+ keydownListener;
17
+ tty;
18
+ enabled;
19
+ constructor(options) {
20
+ this.options = options;
21
+ this.tty =
22
+ options.tty === "auto"
23
+ ? options.verbose
24
+ ? false
25
+ : process.stdout.isTTY
26
+ : !!options.tty;
27
+ this.enabled =
28
+ options.enabled === "auto"
29
+ ? this.tty
30
+ ? true
31
+ : "interval"
32
+ : !!options.enabled;
33
+ }
34
+ elapsed() {
35
+ return this.timer.elapsed();
36
+ }
37
+ start() {
38
+ this.timer.reset();
39
+ (0, readline_1.emitKeypressEvents)(process.stdin);
40
+ process.stdin?.setRawMode?.(true);
41
+ process.stdin?.resume();
42
+ process.stdin?.setEncoding("utf8");
43
+ process.stdin?.on("keypress", (this.keydownListener = (inKey) => {
44
+ const key = inKey.toString();
45
+ if (key === "\u0003") {
46
+ process.stdin.setRawMode?.(false);
47
+ process.emit("SIGINT");
48
+ }
49
+ else if (/^(\r\n)|\r|\n$/.test(key)) {
50
+ this.interval = undefined;
51
+ }
52
+ }));
53
+ }
54
+ dispose() {
55
+ this.timer.stop();
56
+ if (this.keydownListener) {
57
+ process.stdin?.off("keypress", this.keydownListener);
58
+ this.keydownListener = undefined;
59
+ }
60
+ }
61
+ update(progress, cb) {
62
+ if (!this.enabled)
63
+ return;
64
+ if (this.enabled === "interval") {
65
+ if (this.interval) {
66
+ if (!this.interval.reset(this.options.interval ?? 5000))
67
+ return;
68
+ }
69
+ else {
70
+ this.interval = (0, date_1.createTimer)();
71
+ }
72
+ }
73
+ cb(renderProgress(progress, this.tty).join("\n"));
74
+ }
75
+ }
76
+ exports.ProgressManager = ProgressManager;
77
+ function renderProgress(progress, bar) {
78
+ return [
79
+ progress.absolute && renderProgressStats(progress.absolute, bar),
80
+ progress.relative && renderProgressStats(progress.relative, bar),
81
+ ].filter((v) => !!v);
82
+ }
83
+ exports.renderProgress = renderProgress;
84
+ function renderProgressStats(stats, progressBar) {
85
+ const text = [];
86
+ if (typeof stats.percent === "number") {
87
+ if (progressBar)
88
+ text.push((0, cli_1.renderProgressBar)(stats.percent));
89
+ text.push(`${stats.percent.toFixed(2)}%`);
90
+ }
91
+ if (typeof stats.current === "number" || typeof stats.total === "number") {
92
+ const format = (value) => stats.format === "size" ? (0, bytes_1.default)(value) : value;
93
+ if (typeof stats.current === "number" && typeof stats.total === "number") {
94
+ text.push(`${format(stats.current)}/${format(stats.total)}`);
95
+ }
96
+ else if (typeof stats.current === "number") {
97
+ text.push(`${format(stats.current)}`);
98
+ }
99
+ else if (typeof stats.total === "number") {
100
+ text.push(`?/${format(stats.total)}`);
101
+ }
102
+ }
103
+ if (stats.description && stats.payload) {
104
+ text.push(`${stats.description}: ${stats.payload}`);
105
+ }
106
+ else if (stats.description) {
107
+ text.push(stats.description);
108
+ }
109
+ else if (stats.payload) {
110
+ text.push(stats.payload);
111
+ }
112
+ const sep = (0, chalk_1.grey)(`|`);
113
+ return text.join(` ${sep} `);
114
+ }
115
+ exports.renderProgressStats = renderProgressStats;
package/utils/steps.d.ts CHANGED
@@ -9,12 +9,20 @@ export type NodeStepConfig = {
9
9
  vars?: Record<string, any>;
10
10
  code: string | string[];
11
11
  };
12
+ export type TelegramMessageStepConfig = {
13
+ bot: string;
14
+ chatId: number;
15
+ text?: string;
16
+ };
12
17
  export type Step = {
13
18
  type: "process";
14
19
  config: ProcessStepConfig;
15
20
  } | {
16
21
  type: "node";
17
22
  config: NodeStepConfig;
23
+ } | {
24
+ type: "telegram-message";
25
+ config: TelegramMessageStepConfig;
18
26
  };
19
27
  export type StepOptions = {
20
28
  env?: Record<string, string | undefined>;
@@ -25,6 +33,9 @@ export type StepOptions = {
25
33
  vars?: Record<string, any>;
26
34
  tempDir?: () => Promise<string>;
27
35
  };
36
+ telegram?: {
37
+ vars?: Record<string, string>;
38
+ };
28
39
  cwd?: string;
29
40
  onLine?: (p: string) => any;
30
41
  verbose?: boolean;
package/utils/steps.js CHANGED
@@ -1,9 +1,10 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.runSteps = void 0;
4
- const fs_1 = require("./fs");
4
+ const http_1 = require("./http");
5
5
  const process_1 = require("./process");
6
6
  const string_1 = require("./string");
7
+ const temp_1 = require("./temp");
7
8
  const promises_1 = require("fs/promises");
8
9
  const path_1 = require("path");
9
10
  async function runSteps(input, options) {
@@ -33,19 +34,19 @@ async function runSteps(input, options) {
33
34
  tempDir = await options.node.tempDir();
34
35
  }
35
36
  else {
36
- tempDir = await (0, fs_1.mkTmpDir)("node-step");
37
+ tempDir = await (0, temp_1.mkTmpDir)("node-step");
37
38
  }
38
39
  const scriptPath = (0, path_1.join)(tempDir, "script.js");
39
- const vars = Object.entries({
40
+ const vars = {
40
41
  ...step.config.vars,
41
42
  ...options.node?.vars,
42
- }).reduce((items, [name, value]) => {
43
- items.push(`let ${name} = ${JSON.stringify(value)}`);
44
- return items;
45
- }, []);
46
- await (0, promises_1.writeFile)(scriptPath, Array.isArray(step.config.code)
47
- ? [...vars, ...step.config.code].join(";\n")
48
- : `${vars.join(";\n")}\n${step.config.code}`);
43
+ };
44
+ const varKeys = Object.keys(vars);
45
+ const varJson = JSON.stringify(vars);
46
+ const code = Array.isArray(step.config.code)
47
+ ? [...step.config.code].join(";\n")
48
+ : step.config.code;
49
+ await (0, promises_1.writeFile)(scriptPath, `(async function({ ${varKeys} }) {\n${code};\n})(${varJson});`);
49
50
  await (0, process_1.exec)("node", [scriptPath], {
50
51
  cwd: options.cwd,
51
52
  env: {
@@ -63,6 +64,17 @@ async function runSteps(input, options) {
63
64
  }),
64
65
  });
65
66
  }
67
+ else if (step.type === "telegram-message") {
68
+ await (0, http_1.post)(`https://api.telegram.org/bot${step.config.bot}/sendMessage`, JSON.stringify({
69
+ chat_id: step.config.chatId.toString(),
70
+ text: (0, string_1.render)(step.config.text ?? `{TEXT}`, options.telegram?.vars || {}),
71
+ disable_notification: true,
72
+ }), {
73
+ headers: {
74
+ "Content-Type": "application/json",
75
+ },
76
+ });
77
+ }
66
78
  else {
67
79
  throw new Error(`Invalid step type: ${step.type}`);
68
80
  }
@@ -0,0 +1,7 @@
1
+ /// <reference types="node" />
2
+ import { Writable } from "stream";
3
+ export type Streams = {
4
+ stdout: Writable;
5
+ stderr: Writable;
6
+ };
7
+ export declare function createStreams(options?: Partial<Streams>): Streams;
@@ -0,0 +1,10 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createStreams = void 0;
4
+ function createStreams(options = {}) {
5
+ return {
6
+ stdout: options.stdout ?? process.stdout,
7
+ stderr: options.stderr ?? process.stderr,
8
+ };
9
+ }
10
+ exports.createStreams = createStreams;
package/utils/string.d.ts CHANGED
@@ -19,6 +19,5 @@ export declare function endsWith(input: string, patterns: string[]): boolean;
19
19
  export declare function createMatchFilter(include?: string[], exclude?: string[]): (input: string) => boolean;
20
20
  export declare function checkMatch(subject: string | undefined, patterns: string[]): boolean;
21
21
  export declare function formatDateTime(datetime: string): string;
22
- export declare function splitLines(input: string, satinize?: boolean): string[];
23
22
  export declare function undefIfEmpty(input: string): string | undefined;
24
23
  export {};
package/utils/string.js CHANGED
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.undefIfEmpty = exports.splitLines = exports.formatDateTime = exports.checkMatch = exports.createMatchFilter = exports.endsWith = exports.match = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
3
+ exports.undefIfEmpty = exports.formatDateTime = exports.checkMatch = exports.createMatchFilter = exports.endsWith = exports.match = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
4
4
  const AppError_1 = require("../Error/AppError");
5
5
  const micromatch_1 = require("micromatch");
6
6
  function serialize(message, data) {
@@ -121,18 +121,6 @@ function formatDateTime(datetime) {
121
121
  return result;
122
122
  }
123
123
  exports.formatDateTime = formatDateTime;
124
- function splitLines(input, satinize = true) {
125
- const lines = input.split(/\r?\n/);
126
- return satinize
127
- ? input.split(/\r?\n/).reduce((result, value) => {
128
- value = value.trim();
129
- if (value.length)
130
- result.push(value);
131
- return result;
132
- }, [])
133
- : lines;
134
- }
135
- exports.splitLines = splitLines;
136
124
  function undefIfEmpty(input) {
137
125
  return input.length ? input : undefined;
138
126
  }
package/utils/tar.d.ts CHANGED
@@ -24,14 +24,17 @@ export type DecompressOptions = {
24
24
  */
25
25
  cores?: CoresOptions;
26
26
  };
27
- export interface CreateTarOptions {
27
+ export type CreateTarOptions = {
28
28
  path: string;
29
29
  verbose?: boolean;
30
30
  output: string;
31
- includeList: string;
32
31
  compress?: boolean | CompressOptions;
33
32
  onEntry?: (entry: TarEntry) => void;
34
- }
33
+ } & ({
34
+ includeList: string;
35
+ } | {
36
+ include: string[];
37
+ });
35
38
  export interface ExtractOptions {
36
39
  input: string;
37
40
  output: string;
@@ -51,4 +54,8 @@ export type ListTarOptions = {
51
54
  export declare function listTar(options: ListTarOptions): Promise<number>;
52
55
  export declare function checkPigzLib(cache?: boolean): Promise<boolean>;
53
56
  export declare function createTar(options: CreateTarOptions): Promise<void>;
57
+ /**
58
+ * bsdtar (only windows?) fails if path ends with slash
59
+ */
60
+ export declare function normalizeTarPath(path: string): string;
54
61
  export declare function extractTar(options: ExtractOptions): Promise<void>;
package/utils/tar.js CHANGED
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.extractTar = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
3
+ exports.extractTar = exports.normalizeTarPath = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
4
4
  const cli_1 = require("./cli");
5
5
  const fs_1 = require("./fs");
6
6
  const math_1 = require("./math");
@@ -52,13 +52,18 @@ async function getTarVendor(cache = true, log = false) {
52
52
  }
53
53
  exports.getTarVendor = getTarVendor;
54
54
  async function listTar(options) {
55
+ const vendor = await getTarVendor(true, options.verbose);
55
56
  let total = 0;
56
- await (0, process_1.exec)("tar", ["-tf", toLocalPath(options.input), "--force-local"], {}, {
57
+ await (0, process_1.exec)("tar", [
58
+ "-tf",
59
+ toLocalPath(options.input),
60
+ ...(vendor === "bsdtar" ? [] : ["--force-local"]),
61
+ ], {}, {
57
62
  log: options.verbose,
58
63
  stdout: {
59
64
  parseLines: "skip-empty",
60
65
  onData: (path) => {
61
- options.onEntry?.({ path });
66
+ options.onEntry?.({ path: normalizeTarPath(path) });
62
67
  total++;
63
68
  },
64
69
  },
@@ -91,7 +96,9 @@ async function ifX(input, cb) {
91
96
  }
92
97
  async function createTar(options) {
93
98
  const vendor = await getTarVendor(true, options.verbose);
94
- const total = await (0, fs_1.countFileLines)(options.includeList);
99
+ const total = "include" in options
100
+ ? options.include.length
101
+ : await (0, fs_1.countFileLines)(options.includeList);
95
102
  const compress = await ifX(options.compress, async (compress) => ({
96
103
  ...compress,
97
104
  cores: await resolveCores(compress.cores),
@@ -103,16 +110,27 @@ async function createTar(options) {
103
110
  GZIP_OPT: compress.level.toString(),
104
111
  }),
105
112
  };
113
+ const onData = (line) => {
114
+ current++;
115
+ let path = vendor === "bsdtar" ? line.slice(2) : line;
116
+ options.onEntry?.({
117
+ path: normalizeTarPath(path),
118
+ progress: {
119
+ total,
120
+ current,
121
+ percent: (0, math_1.progressPercent)(total, current),
122
+ },
123
+ });
124
+ };
106
125
  await (0, process_1.exec)("tar", [
107
126
  "--no-recursion",
108
127
  "-C",
109
128
  toLocalPath(options.path),
110
129
  compress?.cores === 1 ? "-czvf" : "-cvf",
111
130
  toLocalPath(options.output),
112
- "-T",
113
- toLocalPath(options.includeList),
114
- "--ignore-failed-read",
115
- "--force-local",
131
+ // https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=172293
132
+ ...(vendor === "bsdtar" ? [] : ["--ignore-failed-read"]),
133
+ ...(vendor === "bsdtar" ? [] : ["--force-local"]),
116
134
  ...(compress && compress.cores > 1
117
135
  ? [
118
136
  "-I",
@@ -126,6 +144,9 @@ async function createTar(options) {
126
144
  .join(" ")}"`,
127
145
  ]
128
146
  : []),
147
+ ...("includeList" in options
148
+ ? ["-T", toLocalPath(options.includeList)]
149
+ : ["--", ...options.include]),
129
150
  ], {
130
151
  ...(compress &&
131
152
  compress.cores > 1 && {
@@ -136,25 +157,19 @@ async function createTar(options) {
136
157
  ...env,
137
158
  },
138
159
  }, {
139
- log: options.verbose ? { envNames: Object.keys(env) } : false,
140
- stderr: { toExitCode: true },
141
- stdout: options.onEntry
160
+ log: options.verbose
161
+ ? { envNames: Object.keys(env), exec: true, stderr: true, stdout: true }
162
+ : false,
163
+ ...(vendor === "bsdtar"
142
164
  ? {
143
- parseLines: "skip-empty",
144
- onData: (line) => {
145
- current++;
146
- const path = vendor === "bsdtar" ? line.slice(2) : line;
147
- options.onEntry?.({
148
- path,
149
- progress: {
150
- total,
151
- current,
152
- percent: (0, math_1.progressPercent)(total, current),
153
- },
154
- });
155
- },
165
+ stderr: options.onEntry
166
+ ? { toExitCode: true, parseLines: "skip-empty", onData }
167
+ : { toExitCode: true },
156
168
  }
157
- : undefined,
169
+ : {
170
+ stderr: { toExitCode: true },
171
+ stdout: { parseLines: "skip-empty", onData },
172
+ }),
158
173
  });
159
174
  }
160
175
  exports.createTar = createTar;
@@ -164,9 +179,18 @@ exports.createTar = createTar;
164
179
  function toLocalPath(path) {
165
180
  return (0, os_1.platform)() === "win32" ? path.replace(/\\/g, "/") : path;
166
181
  }
182
+ /**
183
+ * bsdtar (only windows?) fails if path ends with slash
184
+ */
185
+ function normalizeTarPath(path) {
186
+ return path.endsWith("/") ? path.slice(0, -1) : path;
187
+ }
188
+ exports.normalizeTarPath = normalizeTarPath;
167
189
  async function extractTar(options) {
168
- let total = options.total ??
169
- (await listTar({ input: options.input, verbose: options.verbose }));
190
+ let total = options.onEntry
191
+ ? options.total ??
192
+ (await listTar({ input: options.input, verbose: options.verbose }))
193
+ : undefined;
170
194
  if (!(await (0, fs_1.existsDir)(options.output))) {
171
195
  if (options.verbose)
172
196
  (0, cli_1.logExec)("mkdir", ["-p", options.output]);
@@ -177,12 +201,25 @@ async function extractTar(options) {
177
201
  cores: await resolveCores(decompress.cores),
178
202
  }));
179
203
  let current = 0;
204
+ const vendor = await getTarVendor(true, options.verbose);
205
+ const onData = (line) => {
206
+ const path = vendor === "bsdtar" ? line.slice(2) : line;
207
+ current++;
208
+ options.onEntry?.({
209
+ path: normalizeTarPath(path),
210
+ progress: {
211
+ total: total,
212
+ current,
213
+ percent: (0, math_1.progressPercent)(total, current),
214
+ },
215
+ });
216
+ };
180
217
  await (0, process_1.exec)("tar", [
181
218
  decompress?.cores === 1 ? "-xzvpf" : "-xvpf",
182
219
  toLocalPath(options.input),
183
220
  "-C",
184
221
  toLocalPath(options.output),
185
- "--force-local",
222
+ ...(vendor === "bsdtar" ? [] : ["--force-local"]),
186
223
  ...(decompress && decompress.cores > 1
187
224
  ? ["-I", `"pigz -p ${decompress.cores}"`]
188
225
  : []),
@@ -193,25 +230,16 @@ async function extractTar(options) {
193
230
  }),
194
231
  }, {
195
232
  log: options.verbose,
196
- stderr: {
197
- toExitCode: true,
198
- },
199
- stdout: options.onEntry
233
+ ...(vendor === "bsdtar"
200
234
  ? {
201
- parseLines: "skip-empty",
202
- onData: (path) => {
203
- current++;
204
- options.onEntry?.({
205
- path,
206
- progress: {
207
- total,
208
- current,
209
- percent: (0, math_1.progressPercent)(total, current),
210
- },
211
- });
212
- },
235
+ stderr: options.onEntry
236
+ ? { toExitCode: true, parseLines: "skip-empty", onData }
237
+ : { toExitCode: true },
213
238
  }
214
- : undefined,
239
+ : {
240
+ stderr: { toExitCode: true },
241
+ stdout: { parseLines: "skip-empty", onData },
242
+ }),
215
243
  });
216
244
  }
217
245
  exports.extractTar = extractTar;
@@ -0,0 +1,26 @@
1
+ /// <reference types="node" />
2
+ export declare function parentTmpDir(): string;
3
+ export declare function sessionTmpDir(): string;
4
+ export declare function ensureFreeDiskTempSpace(size: number | string): Promise<void>;
5
+ export declare function isTmpDir(path: string): boolean;
6
+ export declare function rmTmpDir(input: string | string[]): Promise<void>;
7
+ export declare function tmpDir(...keys: [string, ...string[]]): string;
8
+ export declare function mkTmpDir(...keys: [string, ...string[]]): Promise<string>;
9
+ export declare function useTempDir(...keys: [string, ...string[]]): Promise<AsyncDisposable & {
10
+ path: string;
11
+ }>;
12
+ export declare function useTempFile(path: string): AsyncDisposable & {
13
+ path: string;
14
+ };
15
+ export declare class CleanupListener {
16
+ readonly paths: string[];
17
+ stop(): void;
18
+ dispose(): Promise<void>;
19
+ }
20
+ export declare class GargabeCollector {
21
+ protected listeners: Set<CleanupListener>;
22
+ get pending(): boolean;
23
+ cleanup(cb?: () => any): Promise<void>;
24
+ cleanupOnFinish(cb: () => any): Promise<void>;
25
+ cleanupIfFail(cb: () => any): Promise<CleanupListener>;
26
+ }
package/utils/temp.js ADDED
@@ -0,0 +1,133 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.GargabeCollector = exports.CleanupListener = exports.useTempFile = exports.useTempDir = exports.mkTmpDir = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.ensureFreeDiskTempSpace = exports.sessionTmpDir = exports.parentTmpDir = void 0;
7
+ const globalData_1 = __importDefault(require("../globalData"));
8
+ const fs_1 = require("./fs");
9
+ const crypto_1 = require("crypto");
10
+ const promises_1 = require("fs/promises");
11
+ const path_1 = require("path");
12
+ Symbol.dispose ??= Symbol("Symbol.dispose");
13
+ Symbol.asyncDispose ??= Symbol("Symbol.asyncDispose");
14
+ function parentTmpDir() {
15
+ return (0, path_1.join)(globalData_1.default.tempDir, "datatruck-temp");
16
+ }
17
+ exports.parentTmpDir = parentTmpDir;
18
+ function sessionTmpDir() {
19
+ return (0, path_1.join)(parentTmpDir(), process.pid.toString());
20
+ }
21
+ exports.sessionTmpDir = sessionTmpDir;
22
+ async function ensureFreeDiskTempSpace(size) {
23
+ const path = sessionTmpDir();
24
+ await (0, fs_1.mkdirIfNotExists)(sessionTmpDir());
25
+ await (0, fs_1.ensureFreeDiskSpace)([path], size);
26
+ }
27
+ exports.ensureFreeDiskTempSpace = ensureFreeDiskTempSpace;
28
+ function isTmpDir(path) {
29
+ return path.startsWith(sessionTmpDir()) && path.includes("datatruck-temp");
30
+ }
31
+ exports.isTmpDir = isTmpDir;
32
+ async function rmTmpDir(input) {
33
+ if (typeof input === "string") {
34
+ if (!isTmpDir(input))
35
+ throw new Error(`Path is not a temp dir: ${input}`);
36
+ await (0, promises_1.rm)(input, { recursive: true });
37
+ }
38
+ else {
39
+ for (const path of input)
40
+ await rmTmpDir(path);
41
+ }
42
+ }
43
+ exports.rmTmpDir = rmTmpDir;
44
+ function tmpDir(...keys) {
45
+ const id = (0, crypto_1.randomUUID)().slice(0, 8);
46
+ const path = (0, path_1.join)(sessionTmpDir(), [...keys, id].map(encodeURIComponent).join("-"));
47
+ for (const listener of listeners)
48
+ listener.paths.push(path);
49
+ return path;
50
+ }
51
+ exports.tmpDir = tmpDir;
52
+ const listeners = new Set();
53
+ async function mkTmpDir(...keys) {
54
+ const path = tmpDir(...keys);
55
+ await (0, promises_1.mkdir)(path, { recursive: true });
56
+ return path;
57
+ }
58
+ exports.mkTmpDir = mkTmpDir;
59
+ async function useTempDir(...keys) {
60
+ const path = await mkTmpDir(...keys);
61
+ return {
62
+ path,
63
+ async [Symbol.asyncDispose]() {
64
+ try {
65
+ await rmTmpDir(path);
66
+ }
67
+ catch (_) { }
68
+ },
69
+ };
70
+ }
71
+ exports.useTempDir = useTempDir;
72
+ function useTempFile(path) {
73
+ return {
74
+ path,
75
+ async [Symbol.asyncDispose]() {
76
+ try {
77
+ await (0, promises_1.rm)(path, { recursive: true });
78
+ }
79
+ catch (_) { }
80
+ },
81
+ };
82
+ }
83
+ exports.useTempFile = useTempFile;
84
+ class CleanupListener {
85
+ paths = [];
86
+ stop() {
87
+ listeners.delete(this);
88
+ }
89
+ async dispose() {
90
+ this.stop();
91
+ await rmTmpDir(this.paths);
92
+ }
93
+ }
94
+ exports.CleanupListener = CleanupListener;
95
+ class GargabeCollector {
96
+ listeners = new Set();
97
+ get pending() {
98
+ return this.listeners.size > 0;
99
+ }
100
+ async cleanup(cb) {
101
+ try {
102
+ await cb?.();
103
+ }
104
+ finally {
105
+ for (const listener of this.listeners) {
106
+ this.listeners.delete(listener);
107
+ await listener.dispose();
108
+ }
109
+ }
110
+ }
111
+ async cleanupOnFinish(cb) {
112
+ const cleanup = new CleanupListener();
113
+ try {
114
+ await cb();
115
+ }
116
+ finally {
117
+ cleanup.dispose();
118
+ }
119
+ }
120
+ async cleanupIfFail(cb) {
121
+ const cleanup = new CleanupListener();
122
+ try {
123
+ await cb();
124
+ }
125
+ catch (error) {
126
+ await cleanup.dispose();
127
+ throw error;
128
+ }
129
+ this.listeners.add(cleanup);
130
+ return cleanup;
131
+ }
132
+ }
133
+ exports.GargabeCollector = GargabeCollector;
@@ -1,3 +1,4 @@
1
+ import { DiskStats } from "./fs";
1
2
  export declare function resolvePath(path: string): string;
2
3
  export type FsOptions = {
3
4
  backend: string;
@@ -7,6 +8,7 @@ export declare abstract class AbstractFs {
7
8
  constructor(options: FsOptions);
8
9
  resolvePath(path: string): string;
9
10
  abstract isLocal(): boolean;
11
+ isRemote(): boolean;
10
12
  abstract existsDir(path: string): Promise<boolean>;
11
13
  abstract mkdir(path: string): Promise<void>;
12
14
  abstract readFile(path: string): Promise<string>;
@@ -15,8 +17,9 @@ export declare abstract class AbstractFs {
15
17
  abstract readdir(path: string): Promise<string[]>;
16
18
  abstract ensureEmptyDir(path: string): Promise<void>;
17
19
  abstract writeFile(path: string, contents: string): Promise<void>;
18
- abstract upload(url: string, path: string): Promise<void>;
19
- abstract download(url: string, path: string): Promise<void>;
20
+ abstract upload(source: string, target: string): Promise<void>;
21
+ abstract download(source: string, target: string): Promise<void>;
22
+ abstract fetchDiskStats(source: string): Promise<DiskStats>;
20
23
  }
21
24
  export declare class LocalFs extends AbstractFs {
22
25
  isLocal(): boolean;
@@ -28,6 +31,7 @@ export declare class LocalFs extends AbstractFs {
28
31
  readdir(path: string): Promise<string[]>;
29
32
  writeFile(path: string, contents: string): Promise<void>;
30
33
  rmAll(path: string): Promise<void>;
34
+ fetchDiskStats(source: string): Promise<DiskStats>;
31
35
  upload(source: string, target: string): Promise<void>;
32
36
  download(source: string, target: string): Promise<void>;
33
37
  }