@datatruck/cli 0.27.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/Action/BackupAction.d.ts +75 -34
  2. package/Action/BackupAction.js +302 -246
  3. package/Action/CleanCacheAction.d.ts +8 -4
  4. package/Action/CleanCacheAction.js +8 -5
  5. package/Action/ConfigAction.d.ts +12 -5
  6. package/Action/ConfigAction.js +14 -18
  7. package/Action/CopyAction.d.ts +51 -0
  8. package/Action/CopyAction.js +165 -0
  9. package/Action/InitAction.d.ts +3 -3
  10. package/Action/InitAction.js +9 -9
  11. package/Action/PruneAction.d.ts +10 -16
  12. package/Action/PruneAction.js +37 -34
  13. package/Action/RestoreAction.d.ts +49 -24
  14. package/Action/RestoreAction.js +164 -195
  15. package/Action/SnapshotsAction.d.ts +8 -8
  16. package/Action/SnapshotsAction.js +9 -9
  17. package/CHANGELOG.md +513 -0
  18. package/Command/BackupCommand.d.ts +7 -4
  19. package/Command/BackupCommand.js +14 -26
  20. package/Command/CleanCacheCommand.d.ts +4 -4
  21. package/Command/CleanCacheCommand.js +26 -5
  22. package/Command/CommandAbstract.d.ts +10 -7
  23. package/Command/CommandAbstract.js +4 -1
  24. package/Command/ConfigCommand.d.ts +6 -9
  25. package/Command/ConfigCommand.js +13 -8
  26. package/Command/CopyCommand.d.ts +16 -0
  27. package/Command/CopyCommand.js +66 -0
  28. package/Command/InitCommand.d.ts +4 -4
  29. package/Command/InitCommand.js +13 -17
  30. package/Command/PruneCommand.d.ts +4 -10
  31. package/Command/PruneCommand.js +13 -12
  32. package/Command/RestoreCommand.d.ts +1 -1
  33. package/Command/RestoreCommand.js +13 -21
  34. package/Command/SnapshotsCommand.d.ts +4 -4
  35. package/Command/SnapshotsCommand.js +16 -15
  36. package/Command/StartServerCommand.d.ts +3 -3
  37. package/Config/Config.d.ts +11 -0
  38. package/Config/Config.js +18 -0
  39. package/Config/PrunePolicyConfig.d.ts +2 -2
  40. package/Factory/CommandFactory.d.ts +27 -34
  41. package/Factory/CommandFactory.js +27 -54
  42. package/Factory/RepositoryFactory.d.ts +1 -1
  43. package/Factory/RepositoryFactory.js +3 -3
  44. package/Factory/TaskFactory.d.ts +1 -1
  45. package/Factory/TaskFactory.js +3 -3
  46. package/Repository/DatatruckRepository.d.ts +10 -8
  47. package/Repository/DatatruckRepository.js +47 -25
  48. package/Repository/GitRepository.d.ts +9 -8
  49. package/Repository/GitRepository.js +22 -25
  50. package/Repository/RepositoryAbstract.d.ts +39 -37
  51. package/Repository/RepositoryAbstract.js +4 -5
  52. package/Repository/ResticRepository.d.ts +9 -8
  53. package/Repository/ResticRepository.js +30 -28
  54. package/Task/GitTask.d.ts +6 -7
  55. package/Task/GitTask.js +24 -30
  56. package/Task/MariadbTask.d.ts +4 -5
  57. package/Task/MariadbTask.js +26 -32
  58. package/Task/MssqlTask.d.ts +5 -3
  59. package/Task/MssqlTask.js +11 -12
  60. package/Task/MysqlDumpTask.d.ts +10 -3
  61. package/Task/MysqlDumpTask.js +102 -32
  62. package/Task/ScriptTask.d.ts +23 -18
  63. package/Task/ScriptTask.js +34 -24
  64. package/Task/SqlDumpTaskAbstract.d.ts +8 -3
  65. package/Task/SqlDumpTaskAbstract.js +32 -20
  66. package/Task/TaskAbstract.d.ts +24 -25
  67. package/Task/TaskAbstract.js +6 -10
  68. package/cli.js +13 -5
  69. package/config.schema.json +89 -1
  70. package/package.json +4 -5
  71. package/utils/DataFormat.d.ts +23 -12
  72. package/utils/DataFormat.js +36 -14
  73. package/utils/cli.d.ts +3 -9
  74. package/utils/cli.js +19 -55
  75. package/utils/crypto.d.ts +1 -0
  76. package/utils/crypto.js +15 -0
  77. package/utils/datatruck/client.d.ts +2 -0
  78. package/utils/datatruck/client.js +3 -0
  79. package/utils/datatruck/config.d.ts +2 -0
  80. package/utils/datatruck/config.js +18 -3
  81. package/utils/datatruck/paths.d.ts +5 -9
  82. package/utils/datatruck/paths.js +2 -2
  83. package/utils/datatruck/snapshot.d.ts +5 -2
  84. package/utils/datatruck/snapshot.js +12 -22
  85. package/utils/date.d.ts +21 -4
  86. package/utils/date.js +46 -24
  87. package/utils/fs.d.ts +16 -11
  88. package/utils/fs.js +81 -48
  89. package/utils/list.d.ts +64 -0
  90. package/utils/list.js +145 -0
  91. package/utils/mysql.d.ts +2 -0
  92. package/utils/mysql.js +21 -2
  93. package/utils/process.d.ts +1 -0
  94. package/utils/process.js +24 -31
  95. package/utils/progress.d.ts +33 -0
  96. package/utils/progress.js +113 -0
  97. package/utils/steps.d.ts +11 -0
  98. package/utils/steps.js +22 -10
  99. package/utils/stream.d.ts +7 -0
  100. package/utils/stream.js +10 -0
  101. package/utils/string.d.ts +0 -1
  102. package/utils/string.js +1 -13
  103. package/utils/tar.d.ts +10 -3
  104. package/utils/tar.js +73 -45
  105. package/utils/temp.d.ts +26 -0
  106. package/utils/temp.js +133 -0
  107. package/utils/virtual-fs.d.ts +6 -2
  108. package/utils/virtual-fs.js +6 -0
  109. package/Action/BackupSessionsAction.d.ts +0 -13
  110. package/Action/BackupSessionsAction.js +0 -18
  111. package/Action/RestoreSessionsAction.d.ts +0 -13
  112. package/Action/RestoreSessionsAction.js +0 -18
  113. package/Command/BackupSessionsCommand.d.ts +0 -12
  114. package/Command/BackupSessionsCommand.js +0 -92
  115. package/Command/RestoreSessionsCommand.d.ts +0 -12
  116. package/Command/RestoreSessionsCommand.js +0 -91
  117. package/Decorator/EntityDecorator.d.ts +0 -11
  118. package/Decorator/EntityDecorator.js +0 -17
  119. package/Entity/BackupSessionEntity.d.ts +0 -6
  120. package/Entity/BackupSessionEntity.js +0 -25
  121. package/Entity/BackupSessionRepositoryEntity.d.ts +0 -6
  122. package/Entity/BackupSessionRepositoryEntity.js +0 -25
  123. package/Entity/BackupSessionTaskEntity.d.ts +0 -5
  124. package/Entity/BackupSessionTaskEntity.js +0 -24
  125. package/Entity/CrudEntityAbstract.d.ts +0 -5
  126. package/Entity/CrudEntityAbstract.js +0 -9
  127. package/Entity/RestoreSessionEntity.d.ts +0 -5
  128. package/Entity/RestoreSessionEntity.js +0 -24
  129. package/Entity/RestoreSessionRepositoryEntity.d.ts +0 -6
  130. package/Entity/RestoreSessionRepositoryEntity.js +0 -25
  131. package/Entity/RestoreSessionTaskEntity.d.ts +0 -5
  132. package/Entity/RestoreSessionTaskEntity.js +0 -24
  133. package/Entity/StateEntityAbstract.d.ts +0 -9
  134. package/Entity/StateEntityAbstract.js +0 -12
  135. package/Factory/EntityFactory.d.ts +0 -6
  136. package/Factory/EntityFactory.js +0 -40
  137. package/SessionDriver/ConsoleSessionDriver.d.ts +0 -42
  138. package/SessionDriver/ConsoleSessionDriver.js +0 -208
  139. package/SessionDriver/SessionDriverAbstract.d.ts +0 -77
  140. package/SessionDriver/SessionDriverAbstract.js +0 -28
  141. package/SessionDriver/SqliteSessionDriver.d.ts +0 -20
  142. package/SessionDriver/SqliteSessionDriver.js +0 -173
  143. package/SessionManager/BackupSessionManager.d.ts +0 -45
  144. package/SessionManager/BackupSessionManager.js +0 -218
  145. package/SessionManager/RestoreSessionManager.d.ts +0 -47
  146. package/SessionManager/RestoreSessionManager.js +0 -218
  147. package/SessionManager/SessionManagerAbstract.d.ts +0 -18
  148. package/SessionManager/SessionManagerAbstract.js +0 -36
  149. package/migrations/001-initial.sql +0 -98
  150. package/utils/entity.d.ts +0 -4
  151. package/utils/entity.js +0 -10
@@ -1,52 +1,45 @@
1
- import { CleanCacheActionOptionsType } from "../Action/CleanCacheAction";
2
- import { BackupCommandOptionsType } from "../Command/BackupCommand";
3
- import { BackupSessionsCommandOptionsType } from "../Command/BackupSessionsCommand";
4
- import { CleanCacheCommand } from "../Command/CleanCacheCommand";
5
- import { GlobalOptionsType } from "../Command/CommandAbstract";
6
- import { ConfigCommandLogType, ConfigCommandOptionsType } from "../Command/ConfigCommand";
7
- import { InitCommandLogType, InitCommandOptionsType } from "../Command/InitCommand";
8
- import { PruneCommandOptionsType } from "../Command/PruneCommand";
1
+ import { CleanCacheActionOptions } from "../Action/CleanCacheAction";
2
+ import { BackupCommandOptions, BackupCommandResult } from "../Command/BackupCommand";
3
+ import { GlobalOptions } from "../Command/CommandAbstract";
4
+ import { ConfigCommandResult, ConfigCommandOptions } from "../Command/ConfigCommand";
5
+ import { CopyCommandOptionsType } from "../Command/CopyCommand";
6
+ import { InitCommandResult, InitCommandOptions } from "../Command/InitCommand";
7
+ import { PruneCommandOptions } from "../Command/PruneCommand";
9
8
  import { RestoreCommandOptionsType } from "../Command/RestoreCommand";
10
- import { RestoreSessionsCommandOptionsType } from "../Command/RestoreSessionsCommand";
11
- import { SnapshotsCommandLogType, SnapshotsCommandOptionsType } from "../Command/SnapshotsCommand";
12
- import { StartServerCommandOptionsType } from "../Command/StartServerCommand";
9
+ import { SnapshotsCommandResult, SnapshotsCommandOptions } from "../Command/SnapshotsCommand";
10
+ import { StartServerCommand, StartServerCommandOptions } from "../Command/StartServerCommand";
11
+ import { Streams } from "../utils/stream";
13
12
  export declare enum CommandEnum {
14
13
  config = "config",
15
14
  init = "init",
16
15
  snapshots = "snapshots",
17
16
  prune = "prune",
18
17
  backup = "backup",
19
- backupSessions = "backup-sessions",
20
18
  restore = "restore",
21
- restoreSessions = "restore-sessions",
19
+ copy = "copy",
22
20
  cleanCache = "clean-cache",
23
21
  startServer = "start-server"
24
22
  }
25
23
  export type OptionsMapType = {
26
- [CommandEnum.config]: ConfigCommandOptionsType;
27
- [CommandEnum.init]: InitCommandOptionsType;
28
- [CommandEnum.snapshots]: SnapshotsCommandOptionsType;
29
- [CommandEnum.prune]: PruneCommandOptionsType;
30
- [CommandEnum.backup]: BackupCommandOptionsType;
31
- [CommandEnum.backupSessions]: BackupSessionsCommandOptionsType;
24
+ [CommandEnum.config]: ConfigCommandOptions;
25
+ [CommandEnum.init]: InitCommandOptions;
26
+ [CommandEnum.snapshots]: SnapshotsCommandOptions;
27
+ [CommandEnum.prune]: PruneCommandOptions;
28
+ [CommandEnum.backup]: BackupCommandOptions;
32
29
  [CommandEnum.restore]: RestoreCommandOptionsType;
33
- [CommandEnum.restoreSessions]: RestoreSessionsCommandOptionsType;
34
- [CommandEnum.cleanCache]: CleanCacheActionOptionsType;
35
- [CommandEnum.startServer]: StartServerCommandOptionsType;
30
+ [CommandEnum.copy]: CopyCommandOptionsType;
31
+ [CommandEnum.cleanCache]: CleanCacheActionOptions;
32
+ [CommandEnum.startServer]: StartServerCommandOptions;
36
33
  };
37
34
  export type LogMapType = {
38
- [CommandEnum.config]: ConfigCommandLogType;
39
- [CommandEnum.init]: InitCommandLogType;
40
- [CommandEnum.snapshots]: SnapshotsCommandLogType;
35
+ [CommandEnum.config]: ConfigCommandResult;
36
+ [CommandEnum.init]: InitCommandResult;
37
+ [CommandEnum.snapshots]: SnapshotsCommandResult;
38
+ [CommandEnum.backup]: BackupCommandResult;
41
39
  };
42
- export declare function CommandFactory<TCommand extends keyof OptionsMapType>(type: TCommand, globalOptions: GlobalOptionsType<false>, options: OptionsMapType[TCommand]): CleanCacheCommand;
43
- export declare function exec<TCommand extends keyof OptionsMapType>(type: TCommand, globalOptions: GlobalOptionsType<false>, options: OptionsMapType[TCommand]): Promise<number>;
44
- export declare function createActionInterface(globalOptions: GlobalOptionsType<false>): {
40
+ export declare function CommandFactory<TCommand extends keyof OptionsMapType>(type: TCommand, globalOptions: GlobalOptions<true>, options: OptionsMapType[TCommand], streams?: Partial<Streams>): StartServerCommand;
41
+ export declare function exec<TCommand extends keyof OptionsMapType>(type: TCommand, globalOptions: GlobalOptions<true>, options: OptionsMapType[TCommand], streams?: Partial<Streams>): Promise<number>;
42
+ export declare function createActionInterface(globalOptions: GlobalOptions<true>): {
45
43
  [K in keyof OptionsMapType as `${K}`]: (options: OptionsMapType[K]) => Promise<K extends keyof LogMapType ? LogMapType[K] : never>;
46
44
  };
47
- export declare function runAndParse<TCommand extends keyof LogMapType>(type: TCommand, run: () => Promise<any>): Promise<{
48
- exitCode: any;
49
- log: LogMapType[TCommand];
50
- }>;
51
- export declare function makeParseLog<TCommand extends keyof LogMapType>(type: TCommand): () => LogMapType[TCommand];
52
- export declare function CommandConstructorFactory(type: CommandEnum): typeof CleanCacheCommand;
45
+ export declare function CommandConstructorFactory(type: CommandEnum): typeof StartServerCommand;
@@ -1,17 +1,17 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.CommandConstructorFactory = exports.makeParseLog = exports.runAndParse = exports.createActionInterface = exports.exec = exports.CommandFactory = exports.CommandEnum = void 0;
3
+ exports.CommandConstructorFactory = exports.createActionInterface = exports.exec = exports.CommandFactory = exports.CommandEnum = void 0;
4
4
  const BackupCommand_1 = require("../Command/BackupCommand");
5
- const BackupSessionsCommand_1 = require("../Command/BackupSessionsCommand");
6
5
  const CleanCacheCommand_1 = require("../Command/CleanCacheCommand");
7
6
  const ConfigCommand_1 = require("../Command/ConfigCommand");
7
+ const CopyCommand_1 = require("../Command/CopyCommand");
8
8
  const InitCommand_1 = require("../Command/InitCommand");
9
9
  const PruneCommand_1 = require("../Command/PruneCommand");
10
10
  const RestoreCommand_1 = require("../Command/RestoreCommand");
11
- const RestoreSessionsCommand_1 = require("../Command/RestoreSessionsCommand");
12
11
  const SnapshotsCommand_1 = require("../Command/SnapshotsCommand");
13
12
  const StartServerCommand_1 = require("../Command/StartServerCommand");
14
13
  const AppError_1 = require("../Error/AppError");
14
+ const stream_1 = require("stream");
15
15
  var CommandEnum;
16
16
  (function (CommandEnum) {
17
17
  CommandEnum["config"] = "config";
@@ -19,72 +19,48 @@ var CommandEnum;
19
19
  CommandEnum["snapshots"] = "snapshots";
20
20
  CommandEnum["prune"] = "prune";
21
21
  CommandEnum["backup"] = "backup";
22
- CommandEnum["backupSessions"] = "backup-sessions";
23
22
  CommandEnum["restore"] = "restore";
24
- CommandEnum["restoreSessions"] = "restore-sessions";
23
+ CommandEnum["copy"] = "copy";
25
24
  CommandEnum["cleanCache"] = "clean-cache";
26
25
  CommandEnum["startServer"] = "start-server";
27
26
  })(CommandEnum || (exports.CommandEnum = CommandEnum = {}));
28
- function CommandFactory(type, globalOptions, options) {
27
+ function CommandFactory(type, globalOptions, options, streams) {
29
28
  const constructor = CommandConstructorFactory(type);
30
- return new constructor(globalOptions, options);
29
+ return new constructor(globalOptions, options, streams);
31
30
  }
32
31
  exports.CommandFactory = CommandFactory;
33
- async function exec(type, globalOptions, options) {
34
- return await CommandFactory(type, globalOptions, options).onExec();
32
+ async function exec(type, globalOptions, options, streams) {
33
+ return await CommandFactory(type, globalOptions, options, streams).onExec();
35
34
  }
36
35
  exports.exec = exec;
37
36
  function createActionInterface(globalOptions) {
38
37
  const object = {};
39
38
  for (const type of Object.values(CommandEnum)) {
40
39
  object[type] = async (options) => {
41
- const run = () => exec(type, { ...globalOptions, outputFormat: "json", verbose: 1 }, options);
42
- let exitCode;
43
- let log;
44
- if (["config", "init", "snapshots"].includes(type)) {
45
- const parsed = await runAndParse(type, run);
46
- exitCode = parsed.exitCode;
47
- log = parsed.log;
40
+ let stdoutData = "";
41
+ const stdout = new stream_1.Writable({
42
+ write(chunk, encoding, callback) {
43
+ stdoutData += chunk.toString();
44
+ process.stdout.write(chunk, encoding, callback);
45
+ },
46
+ }).on("data", (chunk) => (stdoutData += chunk.toString()));
47
+ const end = () => !stdout.closed &&
48
+ new Promise((resolve) => stdout.end().on("close", resolve));
49
+ try {
50
+ const exitCode = await exec(type, { ...globalOptions, outputFormat: "json", verbose: 1 }, options, { stdout });
51
+ if (exitCode !== 0)
52
+ throw new Error(`Invalid exit code: ${exitCode}`);
53
+ await end();
54
+ return JSON.parse(stdoutData);
48
55
  }
49
- else {
50
- exitCode = await run();
56
+ finally {
57
+ await end();
51
58
  }
52
- if (exitCode !== 0)
53
- throw new Error(`Invalid exit code: ${exitCode}`);
54
- return log;
55
59
  };
56
60
  }
57
61
  return object;
58
62
  }
59
63
  exports.createActionInterface = createActionInterface;
60
- async function runAndParse(type, run) {
61
- const parseLog = makeParseLog(type);
62
- try {
63
- const exitCode = await run();
64
- return { exitCode, log: parseLog() };
65
- }
66
- catch (error) {
67
- try {
68
- parseLog();
69
- }
70
- catch (_) { }
71
- throw error;
72
- }
73
- }
74
- exports.runAndParse = runAndParse;
75
- function makeParseLog(type) {
76
- const data = [];
77
- const consoleLog = console.log;
78
- console.log = console.info = (...items) => {
79
- consoleLog.bind(console)(...items);
80
- data.push(...items);
81
- };
82
- return function parseLog() {
83
- console.log = console.info = consoleLog;
84
- return JSON.parse(data.flat().join("\n"));
85
- };
86
- }
87
- exports.makeParseLog = makeParseLog;
88
64
  function CommandConstructorFactory(type) {
89
65
  if (type === CommandEnum.config) {
90
66
  return ConfigCommand_1.ConfigCommand;
@@ -101,14 +77,11 @@ function CommandConstructorFactory(type) {
101
77
  else if (type === CommandEnum.backup) {
102
78
  return BackupCommand_1.BackupCommand;
103
79
  }
104
- else if (type === CommandEnum.backupSessions) {
105
- return BackupSessionsCommand_1.BackupSessionsCommand;
106
- }
107
80
  else if (type === CommandEnum.restore) {
108
81
  return RestoreCommand_1.RestoreCommand;
109
82
  }
110
- else if (type === CommandEnum.restoreSessions) {
111
- return RestoreSessionsCommand_1.RestoreSessionsCommand;
83
+ else if (type === CommandEnum.copy) {
84
+ return CopyCommand_1.CopyCommand;
112
85
  }
113
86
  else if (type === CommandEnum.cleanCache) {
114
87
  return CleanCacheCommand_1.CleanCacheCommand;
@@ -1,3 +1,3 @@
1
1
  import type { RepositoryConfigType } from "../Config/RepositoryConfig";
2
2
  import type { RepositoryAbstract } from "../Repository/RepositoryAbstract";
3
- export declare function RepositoryFactory(repository: RepositoryConfigType): RepositoryAbstract<any>;
3
+ export declare function createRepo(repository: RepositoryConfigType): RepositoryAbstract<any>;
@@ -1,11 +1,11 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.RepositoryFactory = void 0;
3
+ exports.createRepo = void 0;
4
4
  const AppError_1 = require("../Error/AppError");
5
5
  const DatatruckRepository_1 = require("../Repository/DatatruckRepository");
6
6
  const GitRepository_1 = require("../Repository/GitRepository");
7
7
  const ResticRepository_1 = require("../Repository/ResticRepository");
8
- function RepositoryFactory(repository) {
8
+ function createRepo(repository) {
9
9
  const type = repository.type;
10
10
  if (type === GitRepository_1.gitRepositoryName) {
11
11
  return new GitRepository_1.GitRepository(repository);
@@ -20,4 +20,4 @@ function RepositoryFactory(repository) {
20
20
  throw new AppError_1.AppError(`Invalid repository type: ${type}`);
21
21
  }
22
22
  }
23
- exports.RepositoryFactory = RepositoryFactory;
23
+ exports.createRepo = createRepo;
@@ -1,3 +1,3 @@
1
1
  import { TaskConfigType } from "../Config/TaskConfig";
2
2
  import type { TaskAbstract } from "../Task/TaskAbstract";
3
- export declare function TaskFactory(task: TaskConfigType): TaskAbstract;
3
+ export declare function createTask(task: TaskConfigType): TaskAbstract;
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.TaskFactory = void 0;
3
+ exports.createTask = void 0;
4
4
  const AppError_1 = require("../Error/AppError");
5
5
  const GitTask_1 = require("../Task/GitTask");
6
6
  const MariadbTask_1 = require("../Task/MariadbTask");
@@ -8,7 +8,7 @@ const MssqlTask_1 = require("../Task/MssqlTask");
8
8
  const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
9
9
  const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
10
10
  const ScriptTask_1 = require("../Task/ScriptTask");
11
- function TaskFactory(task) {
11
+ function createTask(task) {
12
12
  if (task.name === GitTask_1.gitTaskName) {
13
13
  return new GitTask_1.GitTask(task.config ?? {});
14
14
  }
@@ -31,4 +31,4 @@ function TaskFactory(task) {
31
31
  throw new AppError_1.AppError(`Invalid task name: ${task["name"]}`);
32
32
  }
33
33
  }
34
- exports.TaskFactory = TaskFactory;
34
+ exports.createTask = createTask;
@@ -1,5 +1,5 @@
1
1
  import { CompressOptions } from "../utils/tar";
2
- import { RepositoryAbstract, BackupDataType, InitDataType, RestoreDataType, SnapshotsDataType, SnapshotResultType, PruneDataType, CopyBackupType } from "./RepositoryAbstract";
2
+ import { RepositoryAbstract, RepoBackupData, RepoInitData, RepoRestoreData, RepoFetchSnapshotsData, Snapshot, RepoPruneData, RepoCopyData } from "./RepositoryAbstract";
3
3
  import type { JSONSchema7 } from "json-schema";
4
4
  export type MetaDataType = {
5
5
  id: string;
@@ -12,6 +12,7 @@ export type MetaDataType = {
12
12
  tarStats?: Record<string, {
13
13
  files: number;
14
14
  size: number;
15
+ checksum: string;
15
16
  }>;
16
17
  };
17
18
  export type DatatruckRepositoryConfigType = {
@@ -47,12 +48,13 @@ export declare class DatatruckRepository extends RepositoryAbstract<DatatruckRep
47
48
  sourcePath: string;
48
49
  } | null;
49
50
  static parseMetaData(data: string): Promise<MetaDataType>;
50
- onGetSource(): string;
51
- onInit(data: InitDataType): Promise<void>;
52
- onPrune(data: PruneDataType): Promise<void>;
53
- onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
54
- onBackup(data: BackupDataType<DatatruckPackageRepositoryConfigType>): Promise<void>;
55
- onCopyBackup(data: CopyBackupType<DatatruckRepositoryConfigType>): Promise<void>;
56
- onRestore(data: RestoreDataType<DatatruckPackageRepositoryConfigType>): Promise<void>;
51
+ getSource(): string;
52
+ fetchDiskStats(config: DatatruckRepositoryConfigType): Promise<import("../utils/fs").DiskStats>;
53
+ init(data: RepoInitData): Promise<void>;
54
+ prune(data: RepoPruneData): Promise<void>;
55
+ fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
56
+ backup(data: RepoBackupData<DatatruckPackageRepositoryConfigType>): Promise<void>;
57
+ copy(data: RepoCopyData<DatatruckRepositoryConfigType>): Promise<void>;
58
+ restore(data: RepoRestoreData<DatatruckPackageRepositoryConfigType>): Promise<void>;
57
59
  }
58
60
  export {};
@@ -4,11 +4,14 @@ exports.DatatruckRepository = exports.datatruckPackageRepositoryDefinition = exp
4
4
  const AppError_1 = require("../Error/AppError");
5
5
  const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
6
6
  const cli_1 = require("../utils/cli");
7
+ const crypto_1 = require("../utils/crypto");
7
8
  const client_1 = require("../utils/datatruck/client");
8
9
  const paths_1 = require("../utils/datatruck/paths");
9
10
  const fs_1 = require("../utils/fs");
11
+ const math_1 = require("../utils/math");
10
12
  const string_1 = require("../utils/string");
11
13
  const tar_1 = require("../utils/tar");
14
+ const temp_1 = require("../utils/temp");
12
15
  const RepositoryAbstract_1 = require("./RepositoryAbstract");
13
16
  const assert_1 = require("assert");
14
17
  const promises_1 = require("fs/promises");
@@ -73,14 +76,18 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
73
76
  static async parseMetaData(data) {
74
77
  return JSON.parse(data.toString());
75
78
  }
76
- onGetSource() {
79
+ getSource() {
77
80
  return this.config.backend;
78
81
  }
79
- async onInit(data) {
82
+ fetchDiskStats(config) {
83
+ const fs = (0, client_1.createFs)(config.backend);
84
+ return fs.fetchDiskStats(".");
85
+ }
86
+ async init(data) {
80
87
  const fs = (0, client_1.createFs)(this.config.backend);
81
88
  await fs.mkdir(".");
82
89
  }
83
- async onPrune(data) {
90
+ async prune(data) {
84
91
  const fs = (0, client_1.createFs)(this.config.backend);
85
92
  const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, {
86
93
  name: data.snapshot.packageName,
@@ -90,7 +97,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
90
97
  if (await fs.existsDir(snapshotName))
91
98
  await fs.rmAll(snapshotName);
92
99
  }
93
- async onSnapshots(data) {
100
+ async fetchSnapshots(data) {
94
101
  const fs = (0, client_1.createFs)(this.config.backend);
95
102
  if (!(await fs.existsDir(".")))
96
103
  throw new Error(`Repository (${this.repository.name}) out path does not exist: ${fs.resolvePath(".")}`);
@@ -132,26 +139,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
132
139
  }
133
140
  return snapshots;
134
141
  }
135
- async onBackup(data) {
142
+ async backup(data) {
136
143
  const fs = (0, client_1.createFs)(this.config.backend);
137
144
  const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
138
145
  const outPath = fs.isLocal()
139
146
  ? fs.resolvePath(snapshotName)
140
- : await this.mkTmpDir("datatruck-backup");
147
+ : await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "fs-remote");
141
148
  const pkg = data.package;
142
- const sourcePath = data.targetPath ?? pkg.path;
143
- (0, assert_1.ok)(sourcePath);
149
+ const path = pkg.path;
144
150
  await fs.mkdir(snapshotName);
145
151
  const backupPathsOptions = {
146
152
  package: data.package,
147
153
  snapshot: data.snapshot,
148
- targetPath: sourcePath,
154
+ path: path,
149
155
  verbose: data.options.verbose,
150
156
  };
151
157
  const scanner = await (0, fs_1.createFileScanner)({
152
158
  onProgress: data.onProgress,
153
159
  glob: {
154
- cwd: sourcePath,
160
+ cwd: path,
155
161
  onlyFiles: false,
156
162
  include: await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions),
157
163
  ignore: pkg.exclude
@@ -170,7 +176,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
170
176
  const packs = [defaultsPack, ...configPacks];
171
177
  const defaultsPackIndex = packs.findIndex((p) => p === defaultsPack);
172
178
  const stream = (0, fs_1.createWriteStreamPool)({
173
- path: await this.mkTmpDir("files"),
179
+ path: await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "backup", "stream-pool"),
174
180
  onStreamPath: (key) => `files-${key}.txt`,
175
181
  });
176
182
  scanner.total++;
@@ -192,7 +198,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
192
198
  scanner.total++;
193
199
  stream.writeLine(packIndex, ".");
194
200
  }
195
- stream.writeLine(packIndex, entry.path);
201
+ stream.writeLine(packIndex, (0, tar_1.normalizeTarPath)(entry.path));
196
202
  return true;
197
203
  });
198
204
  await stream.end();
@@ -208,25 +214,29 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
208
214
  tarStats[packBasename] = {
209
215
  files: stream.lines(packIndex),
210
216
  size: 0,
217
+ checksum: "",
211
218
  };
212
219
  const tarPath = (0, path_1.join)(outPath, packBasename);
213
220
  await (0, tar_1.createTar)({
214
221
  compress: pack.compress,
215
222
  verbose: data.options.verbose,
216
223
  includeList,
217
- path: sourcePath,
224
+ path: path,
218
225
  output: tarPath,
219
- onEntry: async (data) => await scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
226
+ onEntry: async (data) => scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
220
227
  });
228
+ scanner.progress("Fetching tar stats", (0, path_1.basename)(tarPath));
229
+ tarStats[packBasename].checksum = await (0, crypto_1.calcFileHash)(tarPath, "sha1");
221
230
  tarStats[packBasename].size = (await (0, promises_1.stat)(tarPath)).size;
222
231
  if (!fs.isLocal()) {
232
+ scanner.progress("Uploading tar", (0, path_1.basename)(tarPath));
223
233
  await fs.upload(tarPath, `${snapshotName}/${packBasename}`);
224
234
  await (0, promises_1.rm)(tarPath);
225
235
  }
226
236
  }
227
237
  packIndex++;
228
238
  }
229
- await scanner.end();
239
+ scanner.end();
230
240
  // Meta
231
241
  const metaPath = `${snapshotName}/meta.json`;
232
242
  const nodePkg = (0, fs_1.parsePackageFile)();
@@ -244,7 +254,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
244
254
  (0, cli_1.logExec)(`Writing metadata into ${fs.resolvePath(metaPath)}`);
245
255
  await fs.writeFile(`${snapshotName}/meta.json`, JSON.stringify(meta));
246
256
  }
247
- async onCopyBackup(data) {
257
+ async copy(data) {
248
258
  const sourceFs = (0, client_1.createFs)(this.config.backend);
249
259
  const targetFs = (0, client_1.createFs)(data.mirrorRepositoryConfig.backend);
250
260
  const snapshotName = DatatruckRepository.buildSnapshotName(data.snapshot, data.package);
@@ -253,13 +263,25 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
253
263
  await targetFs.mkdir(snapshotName);
254
264
  await targetFs.ensureEmptyDir(snapshotName);
255
265
  const entries = await sourceFs.readdir(snapshotName);
266
+ const total = entries.length;
267
+ let current = 0;
256
268
  for (const entry of entries) {
269
+ data.onProgress({
270
+ absolute: {
271
+ current,
272
+ description: "Copying",
273
+ payload: entry,
274
+ total,
275
+ percent: (0, math_1.progressPercent)(total, current),
276
+ },
277
+ });
278
+ current++;
257
279
  const sourceEntry = `${snapshotName}/${entry}`;
258
280
  if (targetFs.isLocal()) {
259
281
  await sourceFs.download(sourceEntry, targetFs.resolvePath(sourceEntry));
260
282
  }
261
283
  else {
262
- const tempDir = await this.mkTmpDir("remote-copy", entry);
284
+ const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "remote-copy", entry);
263
285
  const tempFile = (0, path_1.join)(tempDir, entry);
264
286
  try {
265
287
  await sourceFs.download(sourceEntry, tempFile);
@@ -271,12 +293,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
271
293
  }
272
294
  }
273
295
  }
274
- async onRestore(data) {
296
+ async restore(data) {
275
297
  const fs = (0, client_1.createFs)(this.config.backend);
276
- const relRestorePath = data.targetPath ?? data.package.restorePath;
298
+ const relRestorePath = data.snapshotPath;
277
299
  (0, assert_1.ok)(relRestorePath);
278
300
  const restorePath = (0, path_1.resolve)(relRestorePath);
279
- const [snapshot] = await this.onSnapshots({
301
+ const [snapshot] = await this.fetchSnapshots({
280
302
  options: {
281
303
  ids: [data.options.snapshotId],
282
304
  },
@@ -286,12 +308,12 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
286
308
  const snapshotName = DatatruckRepository.buildSnapshotName(snapshot, data.package);
287
309
  const meta = await DatatruckRepository.parseMetaData(await fs.readFile(`${snapshotName}/meta.json`));
288
310
  const progress = (0, fs_1.createProgress)({ onProgress: data.onProgress });
289
- await progress.update("Scanning files");
311
+ progress.update("Scanning files");
290
312
  const entries = (await fs.readdir(snapshotName)).filter((v) => v.endsWith(".tar") || v.endsWith(".tar.gz"));
291
313
  const tarStats = meta?.tarStats || {};
292
314
  for (const file in tarStats)
293
315
  progress.total += tarStats[file].files;
294
- await progress.update(`Scanned files: ${progress.total}`);
316
+ progress.update(`Scanned files: ${progress.total}`);
295
317
  if (data.options.verbose)
296
318
  (0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
297
319
  for (const entry of entries) {
@@ -299,7 +321,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
299
321
  try {
300
322
  const sourceEntry = `${snapshotName}/${entry}`;
301
323
  if (!fs.isLocal()) {
302
- const tempDir = await this.mkTmpDir("remote-restore", entry);
324
+ const tempDir = await (0, temp_1.mkTmpDir)(exports.datatruckRepositoryName, "repo", "restore", "remote-fs", entry);
303
325
  tempEntry = `${tempDir}/${entry}`;
304
326
  await fs.download(sourceEntry, tempEntry);
305
327
  }
@@ -309,7 +331,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
309
331
  output: restorePath,
310
332
  decompress: entry.endsWith(".tar.gz"),
311
333
  verbose: data.options.verbose,
312
- onEntry: async (data) => await progress.update(entry.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
334
+ onEntry: (data) => progress.update(entry.endsWith(".tar.gz") ? "Extracting" : "Unpacking", data.path),
313
335
  });
314
336
  }
315
337
  finally {
@@ -317,7 +339,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
317
339
  await (0, fs_1.tryRm)(tempEntry);
318
340
  }
319
341
  }
320
- await progress.update("Finished");
342
+ progress.update("Finished");
321
343
  }
322
344
  }
323
345
  exports.DatatruckRepository = DatatruckRepository;
@@ -1,4 +1,4 @@
1
- import { RepositoryAbstract, BackupDataType, InitDataType, RestoreDataType, SnapshotsDataType, SnapshotResultType, SnapshotTagEnum, SnapshotTagObjectType, PruneDataType, CopyBackupType } from "./RepositoryAbstract";
1
+ import { RepositoryAbstract, RepoBackupData, RepoInitData, RepoRestoreData, RepoFetchSnapshotsData, Snapshot, SnapshotTagEnum, SnapshotTagObjectType, RepoPruneData, RepoCopyData } from "./RepositoryAbstract";
2
2
  import { JSONSchema7 } from "json-schema";
3
3
  export type GitRepositoryConfigType = {
4
4
  repo: string;
@@ -10,7 +10,8 @@ export declare const gitRepositoryDefinition: JSONSchema7;
10
10
  export declare const gitPackageRepositoryDefinition: JSONSchema7;
11
11
  export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfigType> {
12
12
  static refPrefix: string;
13
- onGetSource(): string;
13
+ getSource(): string;
14
+ fetchDiskStats(config: GitRepositoryConfigType): Promise<import("../utils/fs").DiskStats | undefined>;
14
15
  static buildSnapshotTagName(tag: Pick<SnapshotTagObjectType, SnapshotTagEnum.PACKAGE | SnapshotTagEnum.ID>): string;
15
16
  static buildSnapshotTag(tag: SnapshotTagObjectType): {
16
17
  name: string;
@@ -21,10 +22,10 @@ export declare class GitRepository extends RepositoryAbstract<GitRepositoryConfi
21
22
  tags: string[];
22
23
  }) | null;
23
24
  static buildBranchName(packageName: string): string;
24
- onInit(data: InitDataType): Promise<void>;
25
- onPrune(data: PruneDataType): Promise<void>;
26
- onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
27
- onBackup(data: BackupDataType<GitPackageRepositoryConfigType>): Promise<void>;
28
- onCopyBackup(data: CopyBackupType<GitRepositoryConfigType>): Promise<void>;
29
- onRestore(data: RestoreDataType<GitPackageRepositoryConfigType>): Promise<void>;
25
+ init(data: RepoInitData): Promise<void>;
26
+ prune(data: RepoPruneData): Promise<void>;
27
+ fetchSnapshots(data: RepoFetchSnapshotsData): Promise<Snapshot[]>;
28
+ backup(data: RepoBackupData<GitPackageRepositoryConfigType>): Promise<void>;
29
+ copy(data: RepoCopyData<GitRepositoryConfigType>): Promise<void>;
30
+ restore(data: RepoRestoreData<GitPackageRepositoryConfigType>): Promise<void>;
30
31
  }