@datatruck/cli 0.2.0 → 0.3.2

package/Action/BackupAction.js

@@ -26,6 +26,7 @@ class BackupAction {
         });
         packages = (0, config_util_1.resolvePackages)(packages, {
             snapshotId: snapshot.id,
+            snapshotDate: snapshot.date,
             action: "backup",
         });
         for (const pkg of packages) {

package/Action/RestoreAction.js

@@ -206,6 +206,7 @@ class RestoreAction {
         let packages = (0, config_util_1.filterPackages)(this.config, this.options);
         packages = (0, config_util_1.resolvePackages)(packages, {
             snapshotId: this.options.snapshotId,
+            snapshotDate: snapshots[0].date,
             action: "restore",
         });
         const snapshotAndConfigs = this.assocConfigs(packages, snapshots);

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # @datatruck/cli
 
+## 0.3.2
+
+### Patch Changes
+
+- [`8957c3b`](https://github.com/swordev/datatruck/commit/8957c3b5846606db8b825fef357445210f2a3ac3) Thanks [@juanrgm](https://github.com/juanrgm)! - Fix restic progress parser
+
+* [`2989718`](https://github.com/swordev/datatruck/commit/29897185e3d6659359d51ab2212351005137f86c) Thanks [@juanrgm](https://github.com/juanrgm)! - Show closing reason
+
+- [`b9e0843`](https://github.com/swordev/datatruck/commit/b9e0843c7970944cfd30a7d2a543f515adfa60e4) Thanks [@juanrgm](https://github.com/juanrgm)! - Show restic progress in megabytes
+
+## 0.3.1
+
+### Patch Changes
+
+- [`c3bb4c6`](https://github.com/swordev/datatruck/commit/c3bb4c609887c5525cf35487ea237750addb6e75) Thanks [@juanrgm](https://github.com/juanrgm)! - Fix restic stdout parser
+
+## 0.3.0
+
+### Minor Changes
+
+- [`d63fd25`](https://github.com/swordev/datatruck/commit/d63fd25ffa8d2e539d2125dfd6a3f55020086804) Thanks [@juanrgm](https://github.com/juanrgm)! - Add `snapshotDate` param
+
+* [`486ef4a`](https://github.com/swordev/datatruck/commit/486ef4add27ae1dbfd166b16c257522f43537ecd) Thanks [@juanrgm](https://github.com/juanrgm)! - Resolve params in `include` and `exclude`
+
+- [`617dae2`](https://github.com/swordev/datatruck/commit/617dae2c8ed90e6e65e8109f03cfad0e64bd7c02) Thanks [@juanrgm](https://github.com/juanrgm)! - Add `script` task
+
+### Patch Changes
+
+- [`d1b3ea9`](https://github.com/swordev/datatruck/commit/d1b3ea9c9540d30898c00490963523a4fbc68193) Thanks [@juanrgm](https://github.com/juanrgm)! - Avoid use gitignore if is not necessary in restic repository
+
 ## 0.2.0
 
 ### Minor Changes

package/Config/TaskConfig.d.ts

@@ -3,6 +3,7 @@ import { MariadbTaskConfigType, mariadbTaskName } from "../Task/MariadbTask";
 import { MssqlTaskConfigType, mssqlTaskName } from "../Task/MssqlTask";
 import { MysqlDumpTaskConfigType, mysqlDumpTaskName } from "../Task/MysqlDumpTask";
 import { PostgresqlDumpTaskConfigType, postgresqlDumpTaskName } from "../Task/PostgresqlDumpTask";
+import { ScriptTaskConfigType, scriptTaskName } from "../Task/ScriptTask";
 import { JSONSchema7 } from "json-schema";
 export declare const taskConfigDefinition: JSONSchema7;
 export declare type TaskConfigType = {
@@ -20,4 +21,7 @@ export declare type TaskConfigType = {
 } | {
     name: typeof postgresqlDumpTaskName;
     config: PostgresqlDumpTaskConfigType;
+} | {
+    name: typeof scriptTaskName;
+    config: ScriptTaskConfigType;
 };

package/Config/TaskConfig.js

@@ -7,12 +7,14 @@ const MariadbTask_1 = require("../Task/MariadbTask");
 const MssqlTask_1 = require("../Task/MssqlTask");
 const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
 const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
+const ScriptTask_1 = require("../Task/ScriptTask");
 const names = {
     [GitTask_1.gitTaskName]: DefinitionEnum_1.DefinitionEnum.gitTask,
     [MariadbTask_1.mariadbTaskName]: DefinitionEnum_1.DefinitionEnum.mariadbTask,
     [MssqlTask_1.mssqlTaskName]: DefinitionEnum_1.DefinitionEnum.mssqlTask,
     [MysqlDumpTask_1.mysqlDumpTaskName]: DefinitionEnum_1.DefinitionEnum.mysqlDumpTask,
     [PostgresqlDumpTask_1.postgresqlDumpTaskName]: DefinitionEnum_1.DefinitionEnum.postgresqlDumpTask,
+    [ScriptTask_1.scriptTaskName]: DefinitionEnum_1.DefinitionEnum.scriptTask,
 };
 exports.taskConfigDefinition = {
     type: "object",

package/Factory/TaskFactory.js

@@ -7,6 +7,7 @@ const MariadbTask_1 = require("../Task/MariadbTask");
 const MssqlTask_1 = require("../Task/MssqlTask");
 const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
 const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
+const ScriptTask_1 = require("../Task/ScriptTask");
 function TaskFactory(task) {
     if (task.name === GitTask_1.gitTaskName) {
         return new GitTask_1.GitTask(task.config ?? {});
@@ -23,6 +24,9 @@ function TaskFactory(task) {
     else if (task.name === MssqlTask_1.mssqlTaskName) {
         return new MssqlTask_1.MssqlTask(task.config ?? {});
     }
+    else if (task.name === ScriptTask_1.scriptTaskName) {
+        return new ScriptTask_1.ScriptTask(task.config ?? {});
+    }
     else {
         throw new AppError_1.AppError(`Invalid task name: ${task["name"]}`);
     }

package/JsonSchema/DefinitionEnum.d.ts

@@ -11,6 +11,7 @@ export declare enum DefinitionEnum {
     gitRepository = "git-repository",
     gitPackageRepository = "git-package-repository",
     gitTask = "git-task",
+    scriptTask = "script-task",
     mariadbTask = "mariadb-task",
     mssqlTask = "mssql-task",
     mysqlDumpTask = "mysql-dump-task",
@@ -20,6 +21,6 @@ export declare enum DefinitionEnum {
     prunePolicy = "prune-policy",
     pathsObject = "paths-object"
 }
-export declare function makeRef(type: DefinitionEnum): {
+export declare function makeRef(type: DefinitionEnum, subType?: string): {
     $ref: string;
 };

package/JsonSchema/DefinitionEnum.js

@@ -15,6 +15,7 @@ var DefinitionEnum;
     DefinitionEnum["gitRepository"] = "git-repository";
     DefinitionEnum["gitPackageRepository"] = "git-package-repository";
     DefinitionEnum["gitTask"] = "git-task";
+    DefinitionEnum["scriptTask"] = "script-task";
     DefinitionEnum["mariadbTask"] = "mariadb-task";
     DefinitionEnum["mssqlTask"] = "mssql-task";
     DefinitionEnum["mysqlDumpTask"] = "mysql-dump-task";
@@ -24,9 +25,9 @@ var DefinitionEnum;
     DefinitionEnum["prunePolicy"] = "prune-policy";
     DefinitionEnum["pathsObject"] = "paths-object";
 })(DefinitionEnum = exports.DefinitionEnum || (exports.DefinitionEnum = {}));
-function makeRef(type) {
+function makeRef(type, subType) {
     return {
-        $ref: `#/definitions/${type}`,
+        $ref: `#/definitions/${type}` + (subType ? `_${subType}` : ""),
     };
 }
 exports.makeRef = makeRef;

package/JsonSchema/JsonSchema.js

@@ -15,6 +15,7 @@ const MariadbTask_1 = require("../Task/MariadbTask");
 const MssqlTask_1 = require("../Task/MssqlTask");
 const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
 const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
+const ScriptTask_1 = require("../Task/ScriptTask");
 const SqlDumpTaskAbstract_1 = require("../Task/SqlDumpTaskAbstract");
 const DefinitionEnum_1 = require("./DefinitionEnum");
 exports.definitions = {
@@ -35,6 +36,7 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.resticRepository]: ResticRepository_1.resticRepositoryDefinition,
     [DefinitionEnum_1.DefinitionEnum.resticPackageRepository]: ResticRepository_1.resticPackageRepositoryDefinition,
     [DefinitionEnum_1.DefinitionEnum.gitTask]: GitTask_1.gitTaskDefinition,
+    [DefinitionEnum_1.DefinitionEnum.scriptTask]: ScriptTask_1.scriptTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.sqlDumpTask]: SqlDumpTaskAbstract_1.sqlDumpTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.mariadbTask]: MariadbTask_1.mariadbTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.mssqlTask]: MssqlTask_1.mssqlTaskDefinition,
@@ -44,6 +46,18 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.prunePolicy]: PrunePolicyConfig_1.prunePolicyConfigDefinition,
     [DefinitionEnum_1.DefinitionEnum.pathsObject]: PackageConfig_1.pathsObjectDefinition,
 };
+for (const key in exports.definitions) {
+    const schemaKey = key;
+    const schema = exports.definitions[schemaKey];
+    for (const defName in schema.definitions || {}) {
+        exports.definitions[`${schemaKey}_${defName}`] =
+            schema.definitions[defName];
+    }
+    exports.definitions[schemaKey] = {
+        ...schema,
+    };
+    delete exports.definitions[schemaKey].definitions;
+}
 exports.schema = {
     definitions: exports.definitions,
     ...(0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.config),

package/Repository/LocalRepository.js

@@ -205,6 +205,8 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
             paths: stream,
             packs: compress?.packs,
         });
+        if (data.options.verbose)
+            (0, cli_util_1.logExec)(`Path lists: ${pathLists.path}`);
         let currentFiles = 0;
         if (compress?.packs) {
             let packIndex = 0;

package/Repository/ResticRepository.js

@@ -155,28 +155,31 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const pkg = data.package;
         const sourcePath = data.targetPath ?? data.package.path;
         (0, assert_1.ok)(sourcePath);
-        const include = await (0, paths_util_1.parsePaths)(pkg.include ?? ["**"], {
-            cwd: sourcePath,
-            verbose: data.options.verbose,
-        });
-        const exclude = pkg.exclude
-            ? await (0, paths_util_1.parsePaths)(pkg.exclude, {
+        let gitignorePath;
+        if (pkg.include || pkg.exclude) {
+            const include = await (0, paths_util_1.parsePaths)(pkg.include ?? ["**"], {
                 cwd: sourcePath,
                 verbose: data.options.verbose,
-            })
-            : undefined;
-        const stream = fast_glob_1.default.stream(include, {
-            cwd: sourcePath,
-            ignore: exclude,
-            dot: true,
-            onlyFiles: true,
-            markDirectories: true,
-        });
-        if (data.options.verbose)
-            (0, cli_util_1.logExec)(`Writing paths lists`);
-        const gitignorePath = await (0, fs_util_1.writeGitIgnoreList)({
-            paths: stream,
-        });
+            });
+            const exclude = pkg.exclude
+                ? await (0, paths_util_1.parsePaths)(pkg.exclude, {
+                    cwd: sourcePath,
+                    verbose: data.options.verbose,
+                })
+                : undefined;
+            const stream = fast_glob_1.default.stream(include, {
+                cwd: sourcePath,
+                ignore: exclude,
+                dot: true,
+                onlyFiles: true,
+                markDirectories: true,
+            });
+            if (data.options.verbose)
+                (0, cli_util_1.logExec)(`Writing paths lists`);
+            gitignorePath = await (0, fs_util_1.writeGitIgnoreList)({
+                paths: stream,
+            });
+        }
         if (data.options.tags?.some((tag) => tag.startsWith(ResticRepository.refPrefix)))
             throw new AppError_1.AppError(`Tag prefix is not allowed`);
         const packageTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.PACKAGE, data.package.name);
@@ -190,7 +193,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
             cwd: sourcePath,
             paths: ["."],
             allowEmptySnapshot: true,
-            excludeFile: [gitignorePath],
+            excludeFile: gitignorePath ? [gitignorePath] : undefined,
             parent: lastSnapshot?.id,
             // https://github.com/restic/restic/pull/3200
             ...((await restic.checkBackupSetPathSupport()) && {
@@ -207,8 +210,10 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
             onStream: async (streamData) => {
                 if (streamData.message_type === "status") {
                     await data.onProgress({
-                        total: streamData.total_bytes,
-                        current: streamData.bytes_done ?? 0,
+                        total: Number((streamData.total_bytes / 1024 / 1024).toFixed(2)),
+                        current: streamData.bytes_done
+                            ? Number((streamData.bytes_done / 1024 / 1024).toFixed(2))
+                            : 0,
                         percent: Number((streamData.percent_done * 100).toFixed(2)),
                         step: streamData.current_files?.join(", ") ?? "-",
                     });

package/Task/MssqlTask.js

@@ -96,6 +96,7 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
                     database: databaseName,
                     packageName: data.package.name,
                     snapshotId: data.options.snapshotId,
+                    snapshotDate: data.snapshot.date,
                 });
             const databasePath = (0, path_1.join)(restorePath, file);
             const exists = await this.fetchDatabaseNames(databaseName);

package/Task/ScriptTask.d.ts

@@ -0,0 +1,42 @@
+import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
+import { JSONSchema7 } from "json-schema";
+export declare type ProcessStepConfig = {
+    command: string;
+    env?: Record<string, string>;
+    args?: string[];
+};
+export declare type NodeStepConfig = {
+    env?: Record<string, string>;
+    code: string | string[];
+};
+export declare type Step = {
+    type: "process";
+    config: ProcessStepConfig;
+} | {
+    type: "node";
+    config: NodeStepConfig;
+};
+export declare type ScriptTaskConfigType = {
+    env?: Record<string, string | undefined>;
+    backupSteps: Step[];
+    restoreSteps: Step[];
+};
+export declare const scriptTaskName = "script";
+export declare const scriptTaskDefinition: JSONSchema7;
+export declare class ScriptTask extends TaskAbstract<ScriptTaskConfigType> {
+    protected verbose?: boolean;
+    onBeforeBackup(): Promise<{
+        targetPath: string;
+    }>;
+    protected getVars(data: BackupDataType | RestoreDataType): Record<string, string | undefined>;
+    static processSteps(input: Step[] | Step, options: {
+        env?: Record<string, string | undefined>;
+        vars: Record<string, string | undefined>;
+        verbose?: boolean;
+    }): Promise<void>;
+    onBackup(data: BackupDataType): Promise<void>;
+    onBeforeRestore(): Promise<{
+        targetPath: string;
+    }>;
+    onRestore(data: RestoreDataType): Promise<void>;
+}

package/Task/ScriptTask.js

@@ -0,0 +1,180 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScriptTask = exports.scriptTaskDefinition = exports.scriptTaskName = void 0;
+const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const fs_util_1 = require("../util/fs-util");
+const process_util_1 = require("../util/process-util");
+const string_util_1 = require("../util/string-util");
+const TaskAbstract_1 = require("./TaskAbstract");
+const assert_1 = require("assert");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+var ScriptTaskDefinitionEnum;
+(function (ScriptTaskDefinitionEnum) {
+    ScriptTaskDefinitionEnum["step"] = "step";
+    ScriptTaskDefinitionEnum["processStepConfig"] = "processStepConfig";
+    ScriptTaskDefinitionEnum["nodeStepConfig"] = "nodeStepConfig";
+})(ScriptTaskDefinitionEnum || (ScriptTaskDefinitionEnum = {}));
+const stepTypes = {
+    process: ScriptTaskDefinitionEnum.processStepConfig,
+    node: ScriptTaskDefinitionEnum.nodeStepConfig,
+};
+exports.scriptTaskName = "script";
+exports.scriptTaskDefinition = {
+    definitions: {
+        step: {
+            type: "object",
+            required: ["type"],
+            properties: {
+                type: { enum: Object.keys(stepTypes) },
+                config: {},
+            },
+            anyOf: Object.keys(stepTypes).map((name) => ({
+                if: {
+                    type: "object",
+                    properties: {
+                        type: { const: name },
+                    },
+                },
+                then: {
+                    type: "object",
+                    properties: {
+                        config: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.scriptTask, stepTypes[name]),
+                    },
+                },
+                else: false,
+            })),
+        },
+        processStepConfig: {
+            type: "object",
+            required: ["command"],
+            properties: {
+                command: { type: "string" },
+                env: {
+                    type: "object",
+                    patternProperties: { ".+": { type: "string" } },
+                },
+                args: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+            },
+        },
+        nodeStepConfig: {
+            type: "object",
+            required: ["code"],
+            properties: {
+                code: {
+                    anyOf: [{ type: "string" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil)],
+                },
+                env: {
+                    type: "object",
+                    patternProperties: { ".+": { type: "string" } },
+                },
+            },
+        },
+    },
+    type: "object",
+    additionalProperties: false,
+    required: ["backupSteps", "restoreSteps"],
+    properties: {
+        env: {
+            type: "object",
+            patternProperties: {
+                ".+": { type: "string" },
+            },
+        },
+        backupSteps: {
+            type: "array",
+            items: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.scriptTask, ScriptTaskDefinitionEnum.step),
+        },
+        restoreSteps: {
+            type: "array",
+            items: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.scriptTask, ScriptTaskDefinitionEnum.step),
+        },
+    },
+};
+class ScriptTask extends TaskAbstract_1.TaskAbstract {
+    async onBeforeBackup() {
+        return {
+            targetPath: await (0, fs_util_1.mkTmpDir)(ScriptTask.name),
+        };
+    }
+    getVars(data) {
+        return {
+            DTT_SNAPSHOT_ID: data.snapshot.id,
+            DTT_SNAPSHOT_DATE: data.snapshot.date,
+            DTT_PACKAGE_NAME: data.package.name,
+            DTT_PACKAGE_PATH: data.package.path,
+            DTT_TARGET_PATH: data.targetPath,
+        };
+    }
+    static async processSteps(input, options) {
+        const steps = Array.isArray(input) ? input : [input];
+        for (const step of steps) {
+            if (step.type === "process") {
+                await (0, process_util_1.exec)(step.config.command, (step.config.args || []).map((v) => (0, string_util_1.render)(v, options.vars)), {
+                    env: {
+                        ...process.env,
+                        ...options.vars,
+                        ...options.env,
+                        ...step.config.env,
+                    },
+                }, {
+                    log: options.verbose,
+                });
+            }
+            else if (step.type === "node") {
+                const tempDir = await (0, fs_util_1.mkTmpDir)("script-task-node-step");
+                const scriptPath = (0, path_1.join)(tempDir, "script.js");
+                await (0, promises_1.writeFile)(scriptPath, Array.isArray(step.config.code)
+                    ? step.config.code.join("\n")
+                    : step.config.code);
+                await (0, process_util_1.exec)("node", [scriptPath], {
+                    env: {
+                        ...process.env,
+                        ...options.vars,
+                        ...options.env,
+                        ...step.config.env,
+                    },
+                }, {
+                    log: options.verbose,
+                });
+            }
+            else {
+                throw new Error(`Invalid step type: ${step.type}`);
+            }
+        }
+    }
+    async onBackup(data) {
+        this.verbose = data.options.verbose;
+        const config = this.config;
+        const path = data.package.path;
+        const targetPath = data.targetPath;
+        (0, assert_1.ok)(typeof path === "string");
+        (0, assert_1.ok)(typeof targetPath === "string");
+        await ScriptTask.processSteps(config.backupSteps, {
+            env: config.env,
+            vars: this.getVars(data),
+            verbose: this.verbose,
+        });
+    }
+    async onBeforeRestore() {
+        return {
+            targetPath: await (0, fs_util_1.mkTmpDir)(ScriptTask.name),
+        };
+    }
+    async onRestore(data) {
+        this.verbose = data.options.verbose;
+        const config = this.config;
+        const restorePath = data.package.restorePath;
+        const targetPath = data.targetPath;
+        (0, assert_1.ok)(typeof restorePath === "string");
+        (0, assert_1.ok)(typeof targetPath === "string");
+        await (0, fs_util_1.mkdirIfNotExists)(restorePath);
+        await (0, fs_util_1.ensureEmptyDir)(restorePath);
+        await ScriptTask.processSteps(config.restoreSteps, {
+            env: config.env,
+            vars: this.getVars(data),
+            verbose: this.verbose,
+        });
+    }
+}
+exports.ScriptTask = ScriptTask;

package/Task/SqlDumpTaskAbstract.js

@@ -111,6 +111,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
             name: (0, config_util_1.resolveDatabaseName)(this.config.database, {
                 packageName: data.package.name,
                 snapshotId: data.options.snapshotId,
+                snapshotDate: data.snapshot.date,
                 action: "restore",
                 database: undefined,
             }),
@@ -119,6 +120,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
             database.name = (0, config_util_1.resolveDatabaseName)(this.config.targetDatabase.name, {
                 packageName: data.package.name,
                 snapshotId: data.options.snapshotId,
+                snapshotDate: data.snapshot.date,
                 action: "restore",
                 database: database.name,
             });

package/cli.js

@@ -84,10 +84,12 @@ exports.buildArgs = buildArgs;
 function parseArgs(args) {
     program.parse(args);
     const verbose = getGlobalOptions().verbose;
-    (0, process_util_1.onExit)((eventName) => {
+    (0, process_util_1.onExit)((eventName, error) => {
         if (eventName !== "exit") {
             process.stdout.write(cli_util_1.showCursorCommand);
-            console.log("\nClosing...");
+            console.log(`\nClosing... (reason: ${eventName})`);
+            if (error instanceof Error)
+                console.error((0, chalk_1.red)(error.stack));
         }
         if (!verbose)
             try {

package/config.schema.json

@@ -261,7 +261,8 @@
             "mariadb",
             "mssql",
             "mysql-dump",
-            "postgresql-dump"
+            "postgresql-dump",
+            "script"
           ]
         },
         "config": {}
@@ -361,6 +362,25 @@
             }
           },
           "else": false
+        },
+        {
+          "if": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "const": "script"
+              }
+            }
+          },
+          "then": {
+            "type": "object",
+            "properties": {
+              "config": {
+                "$ref": "#/definitions/script-task"
+              }
+            }
+          },
+          "else": false
         }
       ]
     },
@@ -551,6 +571,36 @@
         }
       }
     },
+    "script-task": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "backupSteps",
+        "restoreSteps"
+      ],
+      "properties": {
+        "env": {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": "string"
+            }
+          }
+        },
+        "backupSteps": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/script-task_step"
+          }
+        },
+        "restoreSteps": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/script-task_step"
+          }
+        }
+      }
+    },
     "sqldump-task": {
       "type": "object",
       "required": [
@@ -810,6 +860,109 @@
           "else": false
         }
       ]
+    },
+    "script-task_step": {
+      "type": "object",
+      "required": [
+        "type"
+      ],
+      "properties": {
+        "type": {
+          "enum": [
+            "process",
+            "node"
+          ]
+        },
+        "config": {}
+      },
+      "anyOf": [
+        {
+          "if": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "process"
+              }
+            }
+          },
+          "then": {
+            "type": "object",
+            "properties": {
+              "config": {
+                "$ref": "#/definitions/script-task_processStepConfig"
+              }
+            }
+          },
+          "else": false
+        },
+        {
+          "if": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "node"
+              }
+            }
+          },
+          "then": {
+            "type": "object",
+            "properties": {
+              "config": {
+                "$ref": "#/definitions/script-task_nodeStepConfig"
+              }
+            }
+          },
+          "else": false
+        }
+      ]
+    },
+    "script-task_processStepConfig": {
+      "type": "object",
+      "required": [
+        "command"
+      ],
+      "properties": {
+        "command": {
+          "type": "string"
+        },
+        "env": {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": "string"
+            }
+          }
+        },
+        "args": {
+          "$ref": "#/definitions/stringlist-util"
+        }
+      }
+    },
+    "script-task_nodeStepConfig": {
+      "type": "object",
+      "required": [
+        "code"
+      ],
+      "properties": {
+        "code": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "$ref": "#/definitions/stringlist-util"
+            }
+          ]
+        },
+        "env": {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": "string"
+            }
+          }
+        }
+      }
     }
   },
   "$ref": "#/definitions/config"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.2.0",
+  "version": "0.3.2",
   "dependencies": {
     "ajv": "^8.11.0",
     "chalk": "^4.1.2",

package/util/ResticUtil.js

@@ -117,8 +117,17 @@ class ResticUtil {
             stdout: {
                 ...(options.onStream && {
                     onData: async (data) => {
-                        if (data.startsWith("{") && data.endsWith("}")) {
-                            await options.onStream?.(JSON.parse(data));
+                        for (const rawLine of data.split("\n")) {
+                            const line = rawLine.trim();
+                            if (line.startsWith("{") && line.endsWith("}")) {
+                                let parsedLine;
+                                try {
+                                    parsedLine = JSON.parse(line);
+                                }
+                                catch (error) { }
+                                if (parsedLine)
+                                    await options.onStream?.(parsedLine);
+                            }
                         }
                     },
                 }),

package/util/datatruck/config-util.d.ts

@@ -18,6 +18,7 @@ declare type ResolveDatabaseNameParamsType = ResolvePackageParamsType & {
 export declare function resolveDatabaseName(value: string, params: ResolveDatabaseNameParamsType): string;
 declare type ResolvePackageParamsType = {
     snapshotId: string;
+    snapshotDate: string;
     action: "backup" | "restore";
 };
 export declare function resolvePackage(pkg: PackageConfigType, params: ResolvePackageParamsType): PackageConfigType;
@@ -25,6 +26,20 @@ export declare function resolvePackages(packages: PackageConfigType[], params: R
 export declare const pkgPathParams: {
     [name in "temp" | Exclude<keyof ResolvePackagePathParamsType, "path">]: string;
 };
+export declare const pkgIncludeParams: {
+    snapshotDate: string;
+    packageName: string;
+    temp: string;
+    snapshotId: string;
+    action: string;
+};
+export declare const pkgExcludeParams: {
+    snapshotDate: string;
+    packageName: string;
+    temp: string;
+    snapshotId: string;
+    action: string;
+};
 export declare const pkgRestorePathParams: {
     [name in "temp" | keyof ResolvePackagePathParamsType]: string;
 };
@@ -33,20 +48,37 @@ export declare const dbNameParams: {
 };
 export declare const params: {
     pkgPath: {
+        snapshotDate: string;
         packageName: string;
         temp: string;
         snapshotId: string;
         action: string;
     };
     pkgRestorePath: {
+        snapshotDate: string;
         packageName: string;
         path: string;
         temp: string;
         snapshotId: string;
         action: string;
     };
+    pkgInclude: {
+        snapshotDate: string;
+        packageName: string;
+        temp: string;
+        snapshotId: string;
+        action: string;
+    };
+    pkgExclude: {
+        snapshotDate: string;
+        packageName: string;
+        temp: string;
+        snapshotId: string;
+        action: string;
+    };
     dbName: {
         snapshotId: string;
+        snapshotDate: string;
         action: string;
         packageName: string;
         database: string;

package/util/datatruck/config-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.params = exports.dbNameParams = exports.pkgRestorePathParams = exports.pkgPathParams = exports.resolvePackages = exports.resolvePackage = exports.resolveDatabaseName = exports.resolvePackagePath = exports.filterPackages = exports.findRepositoryOrFail = void 0;
+exports.params = exports.dbNameParams = exports.pkgRestorePathParams = exports.pkgExcludeParams = exports.pkgIncludeParams = exports.pkgPathParams = exports.resolvePackages = exports.resolvePackage = exports.resolveDatabaseName = exports.resolvePackagePath = exports.filterPackages = exports.findRepositoryOrFail = void 0;
 const AppError_1 = require("../../Error/AppError");
 const fs_util_1 = require("../fs-util");
 const string_util_1 = require("../string-util");
@@ -53,6 +53,10 @@ function resolvePackage(pkg, params) {
         packageName: pkg.name,
         path: undefined,
     };
+    if (pkg.include)
+        pkg.include = pkg.include.map((v) => typeof v === "string" ? (0, string_util_1.render)(v, pkgParams) : v);
+    if (pkg.exclude)
+        pkg.exclude = pkg.exclude.map((v) => typeof v === "string" ? (0, string_util_1.render)(v, pkgParams) : v);
     if (pkg.path)
         pkg.path = resolvePackagePath(pkg.path, pkgParams);
     if (pkg.restorePath)
@@ -71,23 +75,30 @@ exports.pkgPathParams = {
     action: "{action}",
     packageName: "{packageName}",
     snapshotId: "{snapshotId}",
+    snapshotDate: "{snapshotDate}",
     temp: "{temp}",
 };
+exports.pkgIncludeParams = exports.pkgPathParams;
+exports.pkgExcludeParams = exports.pkgPathParams;
 exports.pkgRestorePathParams = {
     action: "{action}",
     packageName: "{packageName}",
     path: "{path}",
     snapshotId: "{snapshotId}",
+    snapshotDate: "{snapshotDate}",
     temp: "{temp}",
 };
 exports.dbNameParams = {
     action: "{action}",
     packageName: "{packageName}",
     snapshotId: "{snapshotId}",
+    snapshotDate: "{snapshotDate}",
     database: "{database}",
 };
 exports.params = {
     pkgPath: exports.pkgPathParams,
     pkgRestorePath: exports.pkgRestorePathParams,
+    pkgInclude: exports.pkgIncludeParams,
+    pkgExclude: exports.pkgExcludeParams,
     dbName: exports.dbNameParams,
 };

package/util/process-util.d.ts

@@ -46,5 +46,5 @@ export declare type ExecResultType = {
 };
 export declare function exec(command: string, argv?: string[], options?: SpawnOptions | null, settings?: ExecSettingsInterface): Promise<ExecResultType>;
 declare type EventNameType = "exit" | "SIGINT" | "SIGUSR1" | "SIGUSR2" | "SIGTERM" | "uncaughtException";
-export declare function onExit(cb: (eventName: EventNameType) => void): void;
+export declare function onExit(cb: (eventName: EventNameType, ...args: any[]) => void): void;
 export {};

package/util/process-util.js

@@ -175,7 +175,7 @@ const eventNames = [
 ];
 function onExit(cb) {
     for (const eventName of eventNames) {
-        process.on(eventName, cb.bind(null, eventName));
+        process.on(eventName, (...args) => cb(eventName, ...args));
     }
 }
 exports.onExit = onExit;

package/util/string-util.js

@@ -21,7 +21,13 @@ function snakeCase(value, char = "_") {
 }
 exports.snakeCase = snakeCase;
 function render(subject, vars) {
-    return subject.replace(/{(\w+)}/g, function (match, name) {
+    return subject.replace(/{([\w/]*)}/g, function (match, name) {
+        if (!name.length) {
+            return "{";
+        }
+        else if (name === "/") {
+            return "}";
+        }
         const value = vars[name];
         if (typeof value === "undefined")
             throw new AppError_1.AppError(`Variable is not defined: '${subject}' (${name})`);