@datatruck/cli 0.1.0 → 0.3.1

package/Action/BackupAction.js

@@ -26,6 +26,7 @@ class BackupAction {
         });
         packages = (0, config_util_1.resolvePackages)(packages, {
             snapshotId: snapshot.id,
+            snapshotDate: snapshot.date,
             action: "backup",
         });
         for (const pkg of packages) {

package/Action/RestoreAction.js

@@ -206,6 +206,7 @@ class RestoreAction {
         let packages = (0, config_util_1.filterPackages)(this.config, this.options);
         packages = (0, config_util_1.resolvePackages)(packages, {
             snapshotId: this.options.snapshotId,
+            snapshotDate: snapshots[0].date,
             action: "restore",
         });
         const snapshotAndConfigs = this.assocConfigs(packages, snapshots);

package/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @datatruck/cli
 
+## 0.3.1
+
+### Patch Changes
+
+- [`c3bb4c6`](https://github.com/swordev/datatruck/commit/c3bb4c609887c5525cf35487ea237750addb6e75) Thanks [@juanrgm](https://github.com/juanrgm)! - Fix restic stdout parser
+
+## 0.3.0
+
+### Minor Changes
+
+- [`d63fd25`](https://github.com/swordev/datatruck/commit/d63fd25ffa8d2e539d2125dfd6a3f55020086804) Thanks [@juanrgm](https://github.com/juanrgm)! - Add `snapshotDate` param
+
+* [`486ef4a`](https://github.com/swordev/datatruck/commit/486ef4add27ae1dbfd166b16c257522f43537ecd) Thanks [@juanrgm](https://github.com/juanrgm)! - Resolve params in `include` and `exclude`
+
+- [`617dae2`](https://github.com/swordev/datatruck/commit/617dae2c8ed90e6e65e8109f03cfad0e64bd7c02) Thanks [@juanrgm](https://github.com/juanrgm)! - Add `script` task
+
+### Patch Changes
+
+- [`d1b3ea9`](https://github.com/swordev/datatruck/commit/d1b3ea9c9540d30898c00490963523a4fbc68193) Thanks [@juanrgm](https://github.com/juanrgm)! - Avoid use gitignore if is not necessary in restic repository
+
+## 0.2.0
+
+### Minor Changes
+
+- [`120460c`](https://github.com/swordev/datatruck/commit/120460c8824cef4184e43f571a4cc0798b899b66) Thanks [@juanrgm](https://github.com/juanrgm)! - Enable `include` option in restic repository
+
+### Patch Changes
+
+- [`e30ede3`](https://github.com/swordev/datatruck/commit/e30ede371bc7ab3fc1cd47758fdac7a28e8e2705) Thanks [@juanrgm](https://github.com/juanrgm)! - Resolve `RESTIC_PASSWORD_FILE` path
+
+* [`8539d28`](https://github.com/swordev/datatruck/commit/8539d285b2c51d700aa811cd772d573fa0d613eb) Thanks [@juanrgm](https://github.com/juanrgm)! - Allow empty backup in restic repository
+
 ## 0.1.0
 
 ### Minor Changes

package/Config/TaskConfig.d.ts

@@ -3,6 +3,7 @@ import { MariadbTaskConfigType, mariadbTaskName } from "../Task/MariadbTask";
 import { MssqlTaskConfigType, mssqlTaskName } from "../Task/MssqlTask";
 import { MysqlDumpTaskConfigType, mysqlDumpTaskName } from "../Task/MysqlDumpTask";
 import { PostgresqlDumpTaskConfigType, postgresqlDumpTaskName } from "../Task/PostgresqlDumpTask";
+import { ScriptTaskConfigType, scriptTaskName } from "../Task/ScriptTask";
 import { JSONSchema7 } from "json-schema";
 export declare const taskConfigDefinition: JSONSchema7;
 export declare type TaskConfigType = {
@@ -20,4 +21,7 @@ export declare type TaskConfigType = {
 } | {
     name: typeof postgresqlDumpTaskName;
     config: PostgresqlDumpTaskConfigType;
+} | {
+    name: typeof scriptTaskName;
+    config: ScriptTaskConfigType;
 };

package/Config/TaskConfig.js

@@ -7,12 +7,14 @@ const MariadbTask_1 = require("../Task/MariadbTask");
 const MssqlTask_1 = require("../Task/MssqlTask");
 const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
 const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
+const ScriptTask_1 = require("../Task/ScriptTask");
 const names = {
     [GitTask_1.gitTaskName]: DefinitionEnum_1.DefinitionEnum.gitTask,
     [MariadbTask_1.mariadbTaskName]: DefinitionEnum_1.DefinitionEnum.mariadbTask,
     [MssqlTask_1.mssqlTaskName]: DefinitionEnum_1.DefinitionEnum.mssqlTask,
     [MysqlDumpTask_1.mysqlDumpTaskName]: DefinitionEnum_1.DefinitionEnum.mysqlDumpTask,
     [PostgresqlDumpTask_1.postgresqlDumpTaskName]: DefinitionEnum_1.DefinitionEnum.postgresqlDumpTask,
+    [ScriptTask_1.scriptTaskName]: DefinitionEnum_1.DefinitionEnum.scriptTask,
 };
 exports.taskConfigDefinition = {
     type: "object",

package/Factory/TaskFactory.js

@@ -7,6 +7,7 @@ const MariadbTask_1 = require("../Task/MariadbTask");
 const MssqlTask_1 = require("../Task/MssqlTask");
 const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
 const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
+const ScriptTask_1 = require("../Task/ScriptTask");
 function TaskFactory(task) {
     if (task.name === GitTask_1.gitTaskName) {
         return new GitTask_1.GitTask(task.config ?? {});
@@ -23,6 +24,9 @@ function TaskFactory(task) {
     else if (task.name === MssqlTask_1.mssqlTaskName) {
         return new MssqlTask_1.MssqlTask(task.config ?? {});
     }
+    else if (task.name === ScriptTask_1.scriptTaskName) {
+        return new ScriptTask_1.ScriptTask(task.config ?? {});
+    }
     else {
         throw new AppError_1.AppError(`Invalid task name: ${task["name"]}`);
     }

package/JsonSchema/DefinitionEnum.d.ts

@@ -11,6 +11,7 @@ export declare enum DefinitionEnum {
     gitRepository = "git-repository",
     gitPackageRepository = "git-package-repository",
     gitTask = "git-task",
+    scriptTask = "script-task",
     mariadbTask = "mariadb-task",
     mssqlTask = "mssql-task",
     mysqlDumpTask = "mysql-dump-task",
@@ -20,6 +21,6 @@ export declare enum DefinitionEnum {
     prunePolicy = "prune-policy",
     pathsObject = "paths-object"
 }
-export declare function makeRef(type: DefinitionEnum): {
+export declare function makeRef(type: DefinitionEnum, subType?: string): {
     $ref: string;
 };

package/JsonSchema/DefinitionEnum.js

@@ -15,6 +15,7 @@ var DefinitionEnum;
     DefinitionEnum["gitRepository"] = "git-repository";
     DefinitionEnum["gitPackageRepository"] = "git-package-repository";
     DefinitionEnum["gitTask"] = "git-task";
+    DefinitionEnum["scriptTask"] = "script-task";
     DefinitionEnum["mariadbTask"] = "mariadb-task";
     DefinitionEnum["mssqlTask"] = "mssql-task";
     DefinitionEnum["mysqlDumpTask"] = "mysql-dump-task";
@@ -24,9 +25,9 @@ var DefinitionEnum;
     DefinitionEnum["prunePolicy"] = "prune-policy";
     DefinitionEnum["pathsObject"] = "paths-object";
 })(DefinitionEnum = exports.DefinitionEnum || (exports.DefinitionEnum = {}));
-function makeRef(type) {
+function makeRef(type, subType) {
     return {
-        $ref: `#/definitions/${type}`,
+        $ref: `#/definitions/${type}` + (subType ? `_${subType}` : ""),
     };
 }
 exports.makeRef = makeRef;

package/JsonSchema/JsonSchema.js

@@ -15,6 +15,7 @@ const MariadbTask_1 = require("../Task/MariadbTask");
 const MssqlTask_1 = require("../Task/MssqlTask");
 const MysqlDumpTask_1 = require("../Task/MysqlDumpTask");
 const PostgresqlDumpTask_1 = require("../Task/PostgresqlDumpTask");
+const ScriptTask_1 = require("../Task/ScriptTask");
 const SqlDumpTaskAbstract_1 = require("../Task/SqlDumpTaskAbstract");
 const DefinitionEnum_1 = require("./DefinitionEnum");
 exports.definitions = {
@@ -35,6 +36,7 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.resticRepository]: ResticRepository_1.resticRepositoryDefinition,
     [DefinitionEnum_1.DefinitionEnum.resticPackageRepository]: ResticRepository_1.resticPackageRepositoryDefinition,
     [DefinitionEnum_1.DefinitionEnum.gitTask]: GitTask_1.gitTaskDefinition,
+    [DefinitionEnum_1.DefinitionEnum.scriptTask]: ScriptTask_1.scriptTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.sqlDumpTask]: SqlDumpTaskAbstract_1.sqlDumpTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.mariadbTask]: MariadbTask_1.mariadbTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.mssqlTask]: MssqlTask_1.mssqlTaskDefinition,
@@ -44,6 +46,18 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.prunePolicy]: PrunePolicyConfig_1.prunePolicyConfigDefinition,
     [DefinitionEnum_1.DefinitionEnum.pathsObject]: PackageConfig_1.pathsObjectDefinition,
 };
+for (const key in exports.definitions) {
+    const schemaKey = key;
+    const schema = exports.definitions[schemaKey];
+    for (const defName in schema.definitions || {}) {
+        exports.definitions[`${schemaKey}_${defName}`] =
+            schema.definitions[defName];
+    }
+    exports.definitions[schemaKey] = {
+        ...schema,
+    };
+    delete exports.definitions[schemaKey].definitions;
+}
 exports.schema = {
     definitions: exports.definitions,
     ...(0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.config),

package/Repository/LocalRepository.js

@@ -205,6 +205,8 @@ class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
             paths: stream,
             packs: compress?.packs,
         });
+        if (data.options.verbose)
+            (0, cli_util_1.logExec)(`Path lists: ${pathLists.path}`);
         let currentFiles = 0;
         if (compress?.packs) {
             let packIndex = 0;

package/Repository/ResticRepository.js

@@ -1,13 +1,18 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ResticRepository = exports.resticPackageRepositoryDefinition = exports.resticRepositoryDefinition = exports.resticRepositoryName = void 0;
 const AppError_1 = require("../Error/AppError");
 const ResticUtil_1 = require("../util/ResticUtil");
+const cli_util_1 = require("../util/cli-util");
 const paths_util_1 = require("../util/datatruck/paths-util");
 const fs_util_1 = require("../util/fs-util");
 const string_util_1 = require("../util/string-util");
 const RepositoryAbstract_1 = require("./RepositoryAbstract");
 const assert_1 = require("assert");
+const fast_glob_1 = __importDefault(require("fast-glob"));
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
 exports.resticRepositoryName = "restic";
@@ -54,7 +59,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         if (this.env)
             return this.env;
         return (this.env = {
-            RESTIC_PASSWORD_FILE: this.config.passwordFile,
+            RESTIC_PASSWORD_FILE: (0, path_1.resolve)(this.config.passwordFile),
             RESTIC_REPOSITORY: await ResticUtil_1.ResticUtil.formatRepository(this.config.repository),
         });
     }
@@ -150,14 +155,31 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const pkg = data.package;
         const sourcePath = data.targetPath ?? data.package.path;
         (0, assert_1.ok)(sourcePath);
-        const include = (await (0, paths_util_1.parsePaths)(pkg.include ?? [], {
-            cwd: sourcePath,
-            verbose: data.options.verbose,
-        })).map(path_1.normalize);
-        // https://github.com/restic/restic/issues/233
-        // https://github.com/restic/restic/pull/2311
-        if (include.length)
-            throw new AppError_1.AppError(`Include is not supported`);
+        let gitignorePath;
+        if (pkg.include || pkg.exclude) {
+            const include = await (0, paths_util_1.parsePaths)(pkg.include ?? ["**"], {
+                cwd: sourcePath,
+                verbose: data.options.verbose,
+            });
+            const exclude = pkg.exclude
+                ? await (0, paths_util_1.parsePaths)(pkg.exclude, {
+                    cwd: sourcePath,
+                    verbose: data.options.verbose,
+                })
+                : undefined;
+            const stream = fast_glob_1.default.stream(include, {
+                cwd: sourcePath,
+                ignore: exclude,
+                dot: true,
+                onlyFiles: true,
+                markDirectories: true,
+            });
+            if (data.options.verbose)
+                (0, cli_util_1.logExec)(`Writing paths lists`);
+            gitignorePath = await (0, fs_util_1.writeGitIgnoreList)({
+                paths: stream,
+            });
+        }
         if (data.options.tags?.some((tag) => tag.startsWith(ResticRepository.refPrefix)))
             throw new AppError_1.AppError(`Tag prefix is not allowed`);
         const packageTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.PACKAGE, data.package.name);
@@ -170,15 +192,13 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         await restic.backup({
             cwd: sourcePath,
             paths: ["."],
+            allowEmptySnapshot: true,
+            excludeFile: gitignorePath ? [gitignorePath] : undefined,
             parent: lastSnapshot?.id,
             // https://github.com/restic/restic/pull/3200
             ...((await restic.checkBackupSetPathSupport()) && {
                 setPaths: [`/datatruck/${data.package.name}`],
             }),
-            exclude: (await (0, paths_util_1.parsePaths)(pkg.exclude ?? [], {
-                cwd: sourcePath,
-                verbose: data.options.verbose,
-            })).map(path_1.normalize),
             tags: [
                 ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.ID, data.snapshot.id),
                 ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.SHORT_ID, data.snapshot.id.slice(0, 8)),

package/Task/MssqlTask.js

@@ -96,6 +96,7 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
                     database: databaseName,
                     packageName: data.package.name,
                     snapshotId: data.options.snapshotId,
+                    snapshotDate: data.snapshot.date,
                 });
             const databasePath = (0, path_1.join)(restorePath, file);
             const exists = await this.fetchDatabaseNames(databaseName);

package/Task/ScriptTask.d.ts

@@ -0,0 +1,42 @@
+import { BackupDataType, RestoreDataType, TaskAbstract } from "./TaskAbstract";
+import { JSONSchema7 } from "json-schema";
+export declare type ProcessStepConfig = {
+    command: string;
+    env?: Record<string, string>;
+    args?: string[];
+};
+export declare type NodeStepConfig = {
+    env?: Record<string, string>;
+    code: string | string[];
+};
+export declare type Step = {
+    type: "process";
+    config: ProcessStepConfig;
+} | {
+    type: "node";
+    config: NodeStepConfig;
+};
+export declare type ScriptTaskConfigType = {
+    env?: Record<string, string | undefined>;
+    backupSteps: Step[];
+    restoreSteps: Step[];
+};
+export declare const scriptTaskName = "script";
+export declare const scriptTaskDefinition: JSONSchema7;
+export declare class ScriptTask extends TaskAbstract<ScriptTaskConfigType> {
+    protected verbose?: boolean;
+    onBeforeBackup(): Promise<{
+        targetPath: string;
+    }>;
+    protected getVars(data: BackupDataType | RestoreDataType): Record<string, string | undefined>;
+    static processSteps(input: Step[] | Step, options: {
+        env?: Record<string, string | undefined>;
+        vars: Record<string, string | undefined>;
+        verbose?: boolean;
+    }): Promise<void>;
+    onBackup(data: BackupDataType): Promise<void>;
+    onBeforeRestore(): Promise<{
+        targetPath: string;
+    }>;
+    onRestore(data: RestoreDataType): Promise<void>;
+}

package/Task/ScriptTask.js

@@ -0,0 +1,180 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScriptTask = exports.scriptTaskDefinition = exports.scriptTaskName = void 0;
+const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const fs_util_1 = require("../util/fs-util");
+const process_util_1 = require("../util/process-util");
+const string_util_1 = require("../util/string-util");
+const TaskAbstract_1 = require("./TaskAbstract");
+const assert_1 = require("assert");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+var ScriptTaskDefinitionEnum;
+(function (ScriptTaskDefinitionEnum) {
+    ScriptTaskDefinitionEnum["step"] = "step";
+    ScriptTaskDefinitionEnum["processStepConfig"] = "processStepConfig";
+    ScriptTaskDefinitionEnum["nodeStepConfig"] = "nodeStepConfig";
+})(ScriptTaskDefinitionEnum || (ScriptTaskDefinitionEnum = {}));
+const stepTypes = {
+    process: ScriptTaskDefinitionEnum.processStepConfig,
+    node: ScriptTaskDefinitionEnum.nodeStepConfig,
+};
+exports.scriptTaskName = "script";
+exports.scriptTaskDefinition = {
+    definitions: {
+        step: {
+            type: "object",
+            required: ["type"],
+            properties: {
+                type: { enum: Object.keys(stepTypes) },
+                config: {},
+            },
+            anyOf: Object.keys(stepTypes).map((name) => ({
+                if: {
+                    type: "object",
+                    properties: {
+                        type: { const: name },
+                    },
+                },
+                then: {
+                    type: "object",
+                    properties: {
+                        config: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.scriptTask, stepTypes[name]),
+                    },
+                },
+                else: false,
+            })),
+        },
+        processStepConfig: {
+            type: "object",
+            required: ["command"],
+            properties: {
+                command: { type: "string" },
+                env: {
+                    type: "object",
+                    patternProperties: { ".+": { type: "string" } },
+                },
+                args: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+            },
+        },
+        nodeStepConfig: {
+            type: "object",
+            required: ["code"],
+            properties: {
+                code: {
+                    anyOf: [{ type: "string" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil)],
+                },
+                env: {
+                    type: "object",
+                    patternProperties: { ".+": { type: "string" } },
+                },
+            },
+        },
+    },
+    type: "object",
+    additionalProperties: false,
+    required: ["backupSteps", "restoreSteps"],
+    properties: {
+        env: {
+            type: "object",
+            patternProperties: {
+                ".+": { type: "string" },
+            },
+        },
+        backupSteps: {
+            type: "array",
+            items: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.scriptTask, ScriptTaskDefinitionEnum.step),
+        },
+        restoreSteps: {
+            type: "array",
+            items: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.scriptTask, ScriptTaskDefinitionEnum.step),
+        },
+    },
+};
+class ScriptTask extends TaskAbstract_1.TaskAbstract {
+    async onBeforeBackup() {
+        return {
+            targetPath: await (0, fs_util_1.mkTmpDir)(ScriptTask.name),
+        };
+    }
+    getVars(data) {
+        return {
+            DTT_SNAPSHOT_ID: data.snapshot.id,
+            DTT_SNAPSHOT_DATE: data.snapshot.date,
+            DTT_PACKAGE_NAME: data.package.name,
+            DTT_PACKAGE_PATH: data.package.path,
+            DTT_TARGET_PATH: data.targetPath,
+        };
+    }
+    static async processSteps(input, options) {
+        const steps = Array.isArray(input) ? input : [input];
+        for (const step of steps) {
+            if (step.type === "process") {
+                await (0, process_util_1.exec)(step.config.command, (step.config.args || []).map((v) => (0, string_util_1.render)(v, options.vars)), {
+                    env: {
+                        ...process.env,
+                        ...options.vars,
+                        ...options.env,
+                        ...step.config.env,
+                    },
+                }, {
+                    log: options.verbose,
+                });
+            }
+            else if (step.type === "node") {
+                const tempDir = await (0, fs_util_1.mkTmpDir)("script-task-node-step");
+                const scriptPath = (0, path_1.join)(tempDir, "script.js");
+                await (0, promises_1.writeFile)(scriptPath, Array.isArray(step.config.code)
+                    ? step.config.code.join("\n")
+                    : step.config.code);
+                await (0, process_util_1.exec)("node", [scriptPath], {
+                    env: {
+                        ...process.env,
+                        ...options.vars,
+                        ...options.env,
+                        ...step.config.env,
+                    },
+                }, {
+                    log: options.verbose,
+                });
+            }
+            else {
+                throw new Error(`Invalid step type: ${step.type}`);
+            }
+        }
+    }
+    async onBackup(data) {
+        this.verbose = data.options.verbose;
+        const config = this.config;
+        const path = data.package.path;
+        const targetPath = data.targetPath;
+        (0, assert_1.ok)(typeof path === "string");
+        (0, assert_1.ok)(typeof targetPath === "string");
+        await ScriptTask.processSteps(config.backupSteps, {
+            env: config.env,
+            vars: this.getVars(data),
+            verbose: this.verbose,
+        });
+    }
+    async onBeforeRestore() {
+        return {
+            targetPath: await (0, fs_util_1.mkTmpDir)(ScriptTask.name),
+        };
+    }
+    async onRestore(data) {
+        this.verbose = data.options.verbose;
+        const config = this.config;
+        const restorePath = data.package.restorePath;
+        const targetPath = data.targetPath;
+        (0, assert_1.ok)(typeof restorePath === "string");
+        (0, assert_1.ok)(typeof targetPath === "string");
+        await (0, fs_util_1.mkdirIfNotExists)(restorePath);
+        await (0, fs_util_1.ensureEmptyDir)(restorePath);
+        await ScriptTask.processSteps(config.restoreSteps, {
+            env: config.env,
+            vars: this.getVars(data),
+            verbose: this.verbose,
+        });
+    }
+}
+exports.ScriptTask = ScriptTask;

package/Task/SqlDumpTaskAbstract.js

@@ -111,6 +111,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
             name: (0, config_util_1.resolveDatabaseName)(this.config.database, {
                 packageName: data.package.name,
                 snapshotId: data.options.snapshotId,
+                snapshotDate: data.snapshot.date,
                 action: "restore",
                 database: undefined,
             }),
@@ -119,6 +120,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
             database.name = (0, config_util_1.resolveDatabaseName)(this.config.targetDatabase.name, {
                 packageName: data.package.name,
                 snapshotId: data.options.snapshotId,
+                snapshotDate: data.snapshot.date,
                 action: "restore",
                 database: database.name,
             });

package/config.schema.json

@@ -261,7 +261,8 @@
             "mariadb",
             "mssql",
             "mysql-dump",
-            "postgresql-dump"
+            "postgresql-dump",
+            "script"
           ]
         },
         "config": {}
@@ -361,6 +362,25 @@
             }
           },
           "else": false
+        },
+        {
+          "if": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "const": "script"
+              }
+            }
+          },
+          "then": {
+            "type": "object",
+            "properties": {
+              "config": {
+                "$ref": "#/definitions/script-task"
+              }
+            }
+          },
+          "else": false
         }
       ]
     },
@@ -551,6 +571,36 @@
         }
       }
     },
+    "script-task": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "backupSteps",
+        "restoreSteps"
+      ],
+      "properties": {
+        "env": {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": "string"
+            }
+          }
+        },
+        "backupSteps": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/script-task_step"
+          }
+        },
+        "restoreSteps": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/script-task_step"
+          }
+        }
+      }
+    },
     "sqldump-task": {
       "type": "object",
       "required": [
@@ -810,6 +860,109 @@
           "else": false
         }
       ]
+    },
+    "script-task_step": {
+      "type": "object",
+      "required": [
+        "type"
+      ],
+      "properties": {
+        "type": {
+          "enum": [
+            "process",
+            "node"
+          ]
+        },
+        "config": {}
+      },
+      "anyOf": [
+        {
+          "if": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "process"
+              }
+            }
+          },
+          "then": {
+            "type": "object",
+            "properties": {
+              "config": {
+                "$ref": "#/definitions/script-task_processStepConfig"
+              }
+            }
+          },
+          "else": false
+        },
+        {
+          "if": {
+            "type": "object",
+            "properties": {
+              "type": {
+                "const": "node"
+              }
+            }
+          },
+          "then": {
+            "type": "object",
+            "properties": {
+              "config": {
+                "$ref": "#/definitions/script-task_nodeStepConfig"
+              }
+            }
+          },
+          "else": false
+        }
+      ]
+    },
+    "script-task_processStepConfig": {
+      "type": "object",
+      "required": [
+        "command"
+      ],
+      "properties": {
+        "command": {
+          "type": "string"
+        },
+        "env": {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": "string"
+            }
+          }
+        },
+        "args": {
+          "$ref": "#/definitions/stringlist-util"
+        }
+      }
+    },
+    "script-task_nodeStepConfig": {
+      "type": "object",
+      "required": [
+        "code"
+      ],
+      "properties": {
+        "code": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "$ref": "#/definitions/stringlist-util"
+            }
+          ]
+        },
+        "env": {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": "string"
+            }
+          }
+        }
+      }
     }
   },
   "$ref": "#/definitions/config"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.1.0",
+  "version": "0.3.1",
   "dependencies": {
     "ajv": "^8.11.0",
     "chalk": "^4.1.2",

package/util/ResticUtil.d.ts

@@ -1,4 +1,4 @@
-import { ExecSettingsInterface } from "./process-util";
+import { ExecResultType, ExecSettingsInterface } from "./process-util";
 import { UriType } from "./string-util";
 export declare type RepositoryType = {
     name?: string;
@@ -43,7 +43,7 @@ export declare class ResticUtil {
     static formatRepository(input: RepositoryType, hidePassword?: boolean): Promise<string>;
     exec(args: string[], settings?: ExecSettingsInterface, options?: {
         cwd?: string;
-    }): Promise<import("./process-util").ExecResultType>;
+    }): Promise<ExecResultType>;
     checkRepository(): Promise<boolean>;
     forget(options: {
         snapshotId?: string;
@@ -81,12 +81,14 @@ export declare class ResticUtil {
         paths: string[];
         setPaths?: string[];
         exclude?: string[];
+        excludeFile?: string[];
         parent?: string;
+        allowEmptySnapshot?: boolean;
         onStream?: (data: BackupStreamType) => Promise<void>;
-    }): Promise<import("./process-util").ExecResultType>;
+    }): Promise<ExecResultType>;
     restore(options: {
         id: string;
         target: string;
         onStream?: (data: BackupStreamType) => Promise<void>;
-    }): Promise<import("./process-util").ExecResultType>;
+    }): Promise<ExecResultType>;
 }

package/util/ResticUtil.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ResticUtil = void 0;
+const fs_util_1 = require("./fs-util");
 const process_util_1 = require("./process-util");
 const string_util_1 = require("./string-util");
 const fs_extra_1 = require("fs-extra");
+const promises_1 = require("fs/promises");
 const path_1 = require("path");
 class ResticUtil {
     constructor(options) {
@@ -13,7 +15,7 @@ class ResticUtil {
         if (input.backend === "local") {
             if (typeof input.path !== "string")
                 throw new Error(`Invalid path at "${input.name}" repository: ${input.path}`);
-            return (0, path_1.normalize)(input.path);
+            return (0, path_1.resolve)(input.path);
         }
         if (input.passwordFile)
             input = {
@@ -36,7 +38,7 @@ class ResticUtil {
                     stderr: true,
                     colorize: true,
                     allToStderr: true,
-                    envNames: ["RESTIC_REPOSITORY"],
+                    envNames: ["RESTIC_REPOSITORY", "RESTIC_PASSWORD_FILE"],
                 }
                 : {},
             ...(settings ?? {}),
@@ -99,10 +101,11 @@ class ResticUtil {
         return result.stderr.includes("flag needs an argument");
     }
     async backup(options) {
-        return await this.exec([
+        const exec = async () => await this.exec([
             "backup",
             "--json",
             ...(options.exclude?.flatMap((v) => ["-e", v]) ?? []),
+            ...(options.excludeFile?.flatMap((v) => ["--exclude-file", v]) ?? []),
             ...(options.tags?.flatMap((v) => ["--tag", v]) ?? []),
             ...(options.setPaths?.flatMap((v) => ["--set-path", v]) ?? []),
             ...(options.parent ? ["--parent", options.parent] : []),
@@ -114,6 +117,7 @@ class ResticUtil {
             stdout: {
                 ...(options.onStream && {
                     onData: async (data) => {
+                        data = data.trim();
                         if (data.startsWith("{") && data.endsWith("}")) {
                             await options.onStream?.(JSON.parse(data));
                         }
@@ -123,6 +127,25 @@ class ResticUtil {
         }, {
             cwd: options.cwd,
         });
+        try {
+            return await exec();
+        }
+        catch (error) {
+            if (options.allowEmptySnapshot &&
+                error.message.includes("unable to save snapshot: snapshot is empty")) {
+                const emptyPath = await (0, fs_util_1.mkTmpDir)("empty");
+                await (0, promises_1.writeFile)(`${emptyPath}/.empty`, "");
+                return await this.backup({
+                    ...options,
+                    cwd: emptyPath,
+                    allowEmptySnapshot: false,
+                    paths: ["."],
+                    exclude: [],
+                    excludeFile: [],
+                });
+            }
+            throw error;
+        }
     }
     async restore(options) {
         return await this.exec(["restore", "--json", options.id, "--target", options.target], {

package/util/datatruck/config-util.d.ts

@@ -18,6 +18,7 @@ declare type ResolveDatabaseNameParamsType = ResolvePackageParamsType & {
 export declare function resolveDatabaseName(value: string, params: ResolveDatabaseNameParamsType): string;
 declare type ResolvePackageParamsType = {
     snapshotId: string;
+    snapshotDate: string;
     action: "backup" | "restore";
 };
 export declare function resolvePackage(pkg: PackageConfigType, params: ResolvePackageParamsType): PackageConfigType;
@@ -25,6 +26,20 @@ export declare function resolvePackages(packages: PackageConfigType[], params: R
 export declare const pkgPathParams: {
     [name in "temp" | Exclude<keyof ResolvePackagePathParamsType, "path">]: string;
 };
+export declare const pkgIncludeParams: {
+    snapshotDate: string;
+    packageName: string;
+    temp: string;
+    snapshotId: string;
+    action: string;
+};
+export declare const pkgExcludeParams: {
+    snapshotDate: string;
+    packageName: string;
+    temp: string;
+    snapshotId: string;
+    action: string;
+};
 export declare const pkgRestorePathParams: {
     [name in "temp" | keyof ResolvePackagePathParamsType]: string;
 };
@@ -33,20 +48,37 @@ export declare const dbNameParams: {
 };
 export declare const params: {
     pkgPath: {
+        snapshotDate: string;
         packageName: string;
         temp: string;
         snapshotId: string;
         action: string;
     };
     pkgRestorePath: {
+        snapshotDate: string;
         packageName: string;
         path: string;
         temp: string;
         snapshotId: string;
         action: string;
     };
+    pkgInclude: {
+        snapshotDate: string;
+        packageName: string;
+        temp: string;
+        snapshotId: string;
+        action: string;
+    };
+    pkgExclude: {
+        snapshotDate: string;
+        packageName: string;
+        temp: string;
+        snapshotId: string;
+        action: string;
+    };
     dbName: {
         snapshotId: string;
+        snapshotDate: string;
         action: string;
         packageName: string;
         database: string;

package/util/datatruck/config-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.params = exports.dbNameParams = exports.pkgRestorePathParams = exports.pkgPathParams = exports.resolvePackages = exports.resolvePackage = exports.resolveDatabaseName = exports.resolvePackagePath = exports.filterPackages = exports.findRepositoryOrFail = void 0;
+exports.params = exports.dbNameParams = exports.pkgRestorePathParams = exports.pkgExcludeParams = exports.pkgIncludeParams = exports.pkgPathParams = exports.resolvePackages = exports.resolvePackage = exports.resolveDatabaseName = exports.resolvePackagePath = exports.filterPackages = exports.findRepositoryOrFail = void 0;
 const AppError_1 = require("../../Error/AppError");
 const fs_util_1 = require("../fs-util");
 const string_util_1 = require("../string-util");
@@ -53,6 +53,10 @@ function resolvePackage(pkg, params) {
         packageName: pkg.name,
         path: undefined,
     };
+    if (pkg.include)
+        pkg.include = pkg.include.map((v) => typeof v === "string" ? (0, string_util_1.render)(v, pkgParams) : v);
+    if (pkg.exclude)
+        pkg.exclude = pkg.exclude.map((v) => typeof v === "string" ? (0, string_util_1.render)(v, pkgParams) : v);
     if (pkg.path)
         pkg.path = resolvePackagePath(pkg.path, pkgParams);
     if (pkg.restorePath)
@@ -71,23 +75,30 @@ exports.pkgPathParams = {
     action: "{action}",
     packageName: "{packageName}",
     snapshotId: "{snapshotId}",
+    snapshotDate: "{snapshotDate}",
     temp: "{temp}",
 };
+exports.pkgIncludeParams = exports.pkgPathParams;
+exports.pkgExcludeParams = exports.pkgPathParams;
 exports.pkgRestorePathParams = {
     action: "{action}",
     packageName: "{packageName}",
     path: "{path}",
     snapshotId: "{snapshotId}",
+    snapshotDate: "{snapshotDate}",
     temp: "{temp}",
 };
 exports.dbNameParams = {
     action: "{action}",
     packageName: "{packageName}",
     snapshotId: "{snapshotId}",
+    snapshotDate: "{snapshotDate}",
     database: "{database}",
 };
 exports.params = {
     pkgPath: exports.pkgPathParams,
     pkgRestorePath: exports.pkgRestorePathParams,
+    pkgInclude: exports.pkgIncludeParams,
+    pkgExclude: exports.pkgExcludeParams,
     dbName: exports.dbNameParams,
 };

package/util/fs-util.d.ts

@@ -23,6 +23,9 @@ export declare function readPartialFile(path: string, positions: [number, number
 export declare function checkFile(path: string): Promise<boolean>;
 export declare function checkDir(path: string): Promise<boolean>;
 export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
+export declare function writeGitIgnoreList(options: {
+    paths: NodeJS.ReadableStream | string[];
+}): Promise<string>;
 export declare function writePathLists(options: {
     paths: NodeJS.ReadableStream | string[];
     packs?: {

package/util/fs-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.writePathLists = exports.forEachFile = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.tmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = void 0;
+exports.writePathLists = exports.writeGitIgnoreList = exports.forEachFile = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.tmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = void 0;
 const path_util_1 = require("./path-util");
 const crypto_1 = require("crypto");
 const fs_1 = require("fs");
@@ -190,6 +190,40 @@ async function forEachFile(dirPath, cb, includeDir) {
     }
 }
 exports.forEachFile = forEachFile;
+async function writeGitIgnoreList(options) {
+    const tempDir = await mkTmpDir("gitignore-list");
+    const path = (0, path_1.join)(tempDir, `.gitignore`);
+    const stream = (0, fs_2.createWriteStream)(path);
+    const dirs = new Set();
+    stream.write("*\n");
+    for await (const value of options.paths) {
+        const dir = (0, path_1.dirname)(value.toString());
+        if (dir !== ".") {
+            let parentPath;
+            for (const value of dir.split("/")) {
+                if (!parentPath) {
+                    parentPath = `!${value}`;
+                }
+                else {
+                    parentPath += `/${value}`;
+                }
+                if (!dirs.has(parentPath)) {
+                    stream.write(`${parentPath}\n`);
+                    stream.write(`${parentPath.slice(1)}/*\n`);
+                    dirs.add(parentPath);
+                }
+            }
+        }
+        stream.write(`!${value}\n`);
+    }
+    await new Promise((resolve, reject) => {
+        stream.close();
+        stream.on("close", resolve);
+        stream.on("error", reject);
+    });
+    return path;
+}
+exports.writeGitIgnoreList = writeGitIgnoreList;
 async function writePathLists(options) {
     const tempDir = await mkTmpDir("path-lists");
     const includedPaths = [];

package/util/string-util.js

@@ -21,7 +21,13 @@ function snakeCase(value, char = "_") {
 }
 exports.snakeCase = snakeCase;
 function render(subject, vars) {
-    return subject.replace(/{(\w+)}/g, function (match, name) {
+    return subject.replace(/{([\w/]*)}/g, function (match, name) {
+        if (!name.length) {
+            return "{";
+        }
+        else if (name === "/") {
+            return "}";
+        }
         const value = vars[name];
         if (typeof value === "undefined")
             throw new AppError_1.AppError(`Variable is not defined: '${subject}' (${name})`);