@datatruck/cli 0.24.0 → 0.25.0

package/Action/RestoreAction.d.ts

@@ -1,7 +1,6 @@
 import type { ConfigType } from "../Config/Config";
 import { PackageConfigType } from "../Config/PackageConfig";
 import { RepositoryConfigType } from "../Config/RepositoryConfig";
-import { AppError } from "../Error/AppError";
 import { SnapshotResultType } from "../Repository/RepositoryAbstract";
 import { RestoreSessionManager } from "../SessionManager/RestoreSessionManager";
 import { TaskAbstract } from "../Task/TaskAbstract";
@@ -38,7 +37,7 @@ export declare class RestoreAction<TRequired extends boolean = true> {
         error: boolean;
         tmpDirs: string[];
     }>;
-    protected getError(pkg: PackageConfigType): AppError | null;
+    protected getError(pkg: PackageConfigType): Error | undefined;
     exec(session: RestoreSessionManager): Promise<{
         errors: Error[];
     }>;

package/Action/RestoreAction.js

@@ -190,20 +190,16 @@ class RestoreAction {
         };
     }
     getError(pkg) {
-        const taskErrors = this.taskErrors[pkg.name]?.length;
-        const repoErrors = this.repoErrors[pkg.name]?.length;
-        if (taskErrors && repoErrors) {
-            return new AppError_1.AppError("Task and repository failed");
-        }
-        else if (taskErrors && !repoErrors) {
-            return new AppError_1.AppError("Task failed");
-        }
-        else if (!taskErrors && repoErrors) {
-            return new AppError_1.AppError("Repository failed");
-        }
-        else {
-            return null;
-        }
+        const taskErrors = this.taskErrors[pkg.name] || [];
+        const repoErrors = this.repoErrors[pkg.name] || [];
+        const errors = [...taskErrors, ...repoErrors];
+        if (!errors.length)
+            return;
+        return AppError_1.AppError.create(taskErrors.length && repoErrors.length
+            ? "Task and repository failed"
+            : taskErrors.length && !repoErrors.length
+                ? "Task failed"
+                : "Repository failed", errors);
     }
     async exec(session) {
         if (!this.options.snapshotId)

package/Command/BackupCommand.d.ts

@@ -12,5 +12,5 @@ export type BackupCommandOptionsType<TResolved = false> = {
 };
 export declare class BackupCommand extends CommandAbstract<BackupCommandOptionsType<false>, BackupCommandOptionsType<true>> {
     onOptions(): import("../utils/cli").OptionsType<BackupCommandOptionsType<false>, BackupCommandOptionsType<true>>;
-    onExec(): Promise<1 | 0>;
+    onExec(): Promise<0 | 1>;
 }

package/Command/RestoreCommand.d.ts

@@ -12,5 +12,5 @@ export type RestoreCommandOptionsType<TResolved = false> = {
 };
 export declare class RestoreCommand extends CommandAbstract<RestoreCommandOptionsType<false>, RestoreCommandOptionsType<true>> {
     onOptions(): import("../utils/cli").OptionsType<RestoreCommandOptionsType<false>, RestoreCommandOptionsType<true>>;
-    onExec(): Promise<1 | 0>;
+    onExec(): Promise<0 | 1>;
 }

package/Config/PackageConfig.d.ts

@@ -1,14 +1,9 @@
+import { Step } from "../utils/steps";
 import { PackageRepositoryConfigType } from "./PackageRepositoryConfig";
 import { PrunePolicyConfigType } from "./PrunePolicyConfig";
 import type { TaskConfigType } from "./TaskConfig";
 import { JSONSchema7 } from "json-schema";
 export declare const packageConfigDefinition: JSONSchema7;
-export declare const pathsObjectDefinition: JSONSchema7;
-export type PathsObjectType = {
-    type: "spawn";
-    command: string;
-    args?: string[];
-};
 export type PackageConfigType = {
     name: string;
     enabled?: boolean;
@@ -19,8 +14,8 @@ export type PackageConfigType = {
         uid: string | number;
         gid: string | number;
     };
-    include?: (string | PathsObjectType)[];
-    exclude?: (string | PathsObjectType)[];
+    include?: (string | Step)[];
+    exclude?: (string | Step)[];
     repositoryNames?: string[];
     prunePolicy?: PrunePolicyConfigType;
     repositoryConfigs?: PackageRepositoryConfigType[];

package/Config/PackageConfig.js

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.pathsObjectDefinition = exports.packageConfigDefinition = void 0;
+exports.packageConfigDefinition = void 0;
 const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const ScriptTask_1 = require("../Task/ScriptTask");
 exports.packageConfigDefinition = {
     type: "object",
     required: ["name"],
@@ -24,13 +25,19 @@ exports.packageConfigDefinition = {
         include: {
             type: "array",
             items: {
-                anyOf: [{ type: "string" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.pathsObject)],
+                anyOf: [
+                    { type: "string" },
+                    (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.scriptTask, ScriptTask_1.ScriptTaskDefinitionEnum.step),
+                ],
             },
         },
         exclude: {
             type: "array",
             items: {
-                anyOf: [{ type: "string" }, (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.pathsObject)],
+                anyOf: [
+                    { type: "string" },
+                    (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.scriptTask, ScriptTask_1.ScriptTaskDefinitionEnum.step),
+                ],
             },
         },
         repositoryNames: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
@@ -41,29 +48,3 @@ exports.packageConfigDefinition = {
         prunePolicy: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.prunePolicy),
     },
 };
-exports.pathsObjectDefinition = {
-    type: "object",
-    required: ["type"],
-    properties: {
-        type: { type: "string" },
-    },
-    anyOf: [
-        {
-            if: {
-                type: "object",
-                properties: {
-                    type: { const: "spawn" },
-                },
-            },
-            then: {
-                type: "object",
-                required: ["command"],
-                properties: {
-                    command: { type: "string" },
-                    args: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
-                },
-            },
-            else: false,
-        },
-    ],
-};

package/JsonSchema/DefinitionEnum.d.ts

@@ -19,7 +19,6 @@ export declare enum DefinitionEnum {
     sqlDumpTask = "sqldump-task",
     stringListUtil = "stringlist-util",
     prunePolicy = "prune-policy",
-    pathsObject = "paths-object",
     compressUtil = "compress-util"
 }
 export declare function makeRef(type: DefinitionEnum, subType?: string): {

package/JsonSchema/DefinitionEnum.js

@@ -23,7 +23,6 @@ var DefinitionEnum;
     DefinitionEnum["sqlDumpTask"] = "sqldump-task";
     DefinitionEnum["stringListUtil"] = "stringlist-util";
     DefinitionEnum["prunePolicy"] = "prune-policy";
-    DefinitionEnum["pathsObject"] = "paths-object";
     DefinitionEnum["compressUtil"] = "compress-util";
 })(DefinitionEnum || (exports.DefinitionEnum = DefinitionEnum = {}));
 function makeRef(type, subType) {

package/JsonSchema/JsonSchema.js

@@ -45,7 +45,6 @@ exports.definitions = {
     [DefinitionEnum_1.DefinitionEnum.postgresqlDumpTask]: PostgresqlDumpTask_1.postgresqlDumpTaskDefinition,
     [DefinitionEnum_1.DefinitionEnum.config]: Config_1.configDefinition,
     [DefinitionEnum_1.DefinitionEnum.prunePolicy]: PrunePolicyConfig_1.prunePolicyConfigDefinition,
-    [DefinitionEnum_1.DefinitionEnum.pathsObject]: PackageConfig_1.pathsObjectDefinition,
     [DefinitionEnum_1.DefinitionEnum.compressUtil]: tar_1.compressDefinition,
 };
 for (const key in exports.definitions) {

package/Repository/DatatruckRepository.js

@@ -159,20 +159,20 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const sourcePath = data.targetPath ?? pkg.path;
         (0, assert_1.ok)(sourcePath);
         await (0, promises_1.mkdir)(outPath, { recursive: true });
+        const backupPathsOptions = {
+            package: data.package,
+            snapshot: data.snapshot,
+            targetPath: sourcePath,
+            verbose: data.options.verbose,
+        };
         const scanner = await (0, fs_1.createFileScanner)({
             onProgress: data.onProgress,
             glob: {
                 cwd: sourcePath,
                 onlyFiles: false,
-                include: await (0, paths_1.parsePaths)(pkg.include ?? ["**"], {
-                    cwd: sourcePath,
-                    verbose: data.options.verbose,
-                }),
+                include: await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions),
                 ignore: pkg.exclude
-                    ? await (0, paths_1.parsePaths)(pkg.exclude, {
-                        cwd: sourcePath,
-                        verbose: data.options.verbose,
-                    })
+                    ? await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions)
                     : undefined,
             },
         });

package/Repository/GitRepository.js

@@ -166,15 +166,15 @@ class GitRepository extends RepositoryAbstract_1.RepositoryAbstract {
         });
         await git.removeAll();
         const createdPaths = [];
-        const include = await (0, paths_1.parsePaths)(pkg.include ?? ["**"], {
-            cwd: sourcePath,
+        const backupPathsOptions = {
+            package: data.package,
+            snapshot: data.snapshot,
+            targetPath: sourcePath,
             verbose: data.options.verbose,
-        });
+        };
+        const include = await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions);
         const exclude = pkg.exclude
-            ? await (0, paths_1.parsePaths)(pkg.exclude, {
-                cwd: sourcePath,
-                verbose: data.options.verbose,
-            })
+            ? await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions)
             : undefined;
         const stream = await (0, fast_glob_1.default)(include, {
             cwd: sourcePath,

package/Repository/ResticRepository.js

@@ -191,11 +191,14 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const sourcePath = data.targetPath ?? data.package.path;
         (0, assert_1.ok)(sourcePath);
         let gitignorePath;
+        const backupPathsOptions = {
+            package: data.package,
+            snapshot: data.snapshot,
+            targetPath: sourcePath,
+            verbose: data.options.verbose,
+        };
         if (!pkg.include && pkg.exclude) {
-            const exclude = await (0, paths_1.parsePaths)(pkg.exclude, {
-                cwd: sourcePath,
-                verbose: data.options.verbose,
-            });
+            const exclude = await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions);
             await data.onProgress({
                 relative: {
                     description: "Writing excluded paths list",
@@ -207,15 +210,9 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
             await (0, promises_1.writeFile)(gitignorePath, ignoredContents);
         }
         else if (pkg.include || pkg.exclude) {
-            const include = await (0, paths_1.parsePaths)(pkg.include ?? ["**"], {
-                cwd: sourcePath,
-                verbose: data.options.verbose,
-            });
+            const include = await (0, paths_1.parseBackupPaths)(pkg.include ?? ["**"], backupPathsOptions);
             const exclude = pkg.exclude
-                ? await (0, paths_1.parsePaths)(pkg.exclude, {
-                    cwd: sourcePath,
-                    verbose: data.options.verbose,
-                })
+                ? await (0, paths_1.parseBackupPaths)(pkg.exclude, backupPathsOptions)
                 : undefined;
             const stream = fast_glob_1.default.stream(include, {
                 cwd: sourcePath,

package/Task/ScriptTask.d.ts

@@ -6,6 +6,11 @@ export type ScriptTaskConfigType = {
     backupSteps: Step[];
     restoreSteps: Step[];
 };
+export declare enum ScriptTaskDefinitionEnum {
+    step = "step",
+    processStepConfig = "processStepConfig",
+    nodeStepConfig = "nodeStepConfig"
+}
 export declare const scriptTaskName = "script";
 export declare const scriptTaskDefinition: JSONSchema7;
 export declare class ScriptTask extends TaskAbstract<ScriptTaskConfigType> {

package/Task/ScriptTask.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ScriptTask = exports.scriptTaskDefinition = exports.scriptTaskName = void 0;
+exports.ScriptTask = exports.scriptTaskDefinition = exports.scriptTaskName = exports.ScriptTaskDefinitionEnum = void 0;
 const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
 const fs_1 = require("../utils/fs");
 const steps_1 = require("../utils/steps");
@@ -11,7 +11,7 @@ var ScriptTaskDefinitionEnum;
     ScriptTaskDefinitionEnum["step"] = "step";
     ScriptTaskDefinitionEnum["processStepConfig"] = "processStepConfig";
     ScriptTaskDefinitionEnum["nodeStepConfig"] = "nodeStepConfig";
-})(ScriptTaskDefinitionEnum || (ScriptTaskDefinitionEnum = {}));
+})(ScriptTaskDefinitionEnum || (exports.ScriptTaskDefinitionEnum = ScriptTaskDefinitionEnum = {}));
 const stepTypes = {
     process: ScriptTaskDefinitionEnum.processStepConfig,
     node: ScriptTaskDefinitionEnum.nodeStepConfig,

package/config.schema.json

@@ -176,7 +176,7 @@
                 "type": "string"
               },
               {
-                "$ref": "#/definitions/paths-object"
+                "$ref": "#/definitions/script-task_step"
               }
             ]
           }
@@ -189,7 +189,7 @@
                 "type": "string"
               },
               {
-                "$ref": "#/definitions/paths-object"
+                "$ref": "#/definitions/script-task_step"
               }
             ]
           }
@@ -1036,44 +1036,6 @@
         }
       }
     },
-    "paths-object": {
-      "type": "object",
-      "required": [
-        "type"
-      ],
-      "properties": {
-        "type": {
-          "type": "string"
-        }
-      },
-      "anyOf": [
-        {
-          "if": {
-            "type": "object",
-            "properties": {
-              "type": {
-                "const": "spawn"
-              }
-            }
-          },
-          "then": {
-            "type": "object",
-            "required": [
-              "command"
-            ],
-            "properties": {
-              "command": {
-                "type": "string"
-              },
-              "args": {
-                "$ref": "#/definitions/stringlist-util"
-              }
-            }
-          },
-          "else": false
-        }
-      ]
-    },
     "compress-util": {
       "type": "object",
       "additionalProperties": false,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.24.0",
+  "version": "0.25.0",
   "dependencies": {
     "@supercharge/promise-pool": "^3.1.0",
     "ajv": "^8.12.0",

package/utils/datatruck/config.d.ts

@@ -32,15 +32,15 @@ export declare const pkgPathParams: {
 };
 export declare const pkgIncludeParams: {
     packageName: string;
-    snapshotId: string;
     temp: string;
+    snapshotId: string;
     snapshotDate: string;
     action: string;
 };
 export declare const pkgExcludeParams: {
     packageName: string;
-    snapshotId: string;
     temp: string;
+    snapshotId: string;
     snapshotDate: string;
     action: string;
 };
@@ -53,30 +53,30 @@ export declare const dbNameParams: {
 export declare const params: {
     pkgPath: {
         packageName: string;
-        snapshotId: string;
         temp: string;
+        snapshotId: string;
         snapshotDate: string;
         action: string;
     };
     pkgRestorePath: {
         path: string;
         packageName: string;
-        snapshotId: string;
         temp: string;
+        snapshotId: string;
         snapshotDate: string;
         action: string;
     };
     pkgInclude: {
         packageName: string;
-        snapshotId: string;
         temp: string;
+        snapshotId: string;
         snapshotDate: string;
         action: string;
     };
     pkgExclude: {
         packageName: string;
-        snapshotId: string;
         temp: string;
+        snapshotId: string;
         snapshotDate: string;
         action: string;
     };

package/utils/datatruck/paths.d.ts

@@ -1,5 +1,21 @@
-import { PathsObjectType } from "../../Config/PackageConfig";
-export declare function parsePaths(values: (string | PathsObjectType)[], options: {
+import { PackageConfigType } from "../../Config/PackageConfig";
+import { SnapshotType } from "../../Repository/RepositoryAbstract";
+import { Step } from "../steps";
+export type ParsePathsOptions = {
     cwd?: string;
     verbose?: boolean;
+    vars?: Record<string, any>;
+};
+export declare function parsePaths(values: (string | Step)[], options: {
+    cwd?: string;
+    verbose?: boolean;
+    vars?: Record<string, any>;
+    tempDir?: () => Promise<string>;
 }): Promise<string[]>;
+export type BackupPathsOptions = {
+    package: PackageConfigType;
+    snapshot: SnapshotType;
+    targetPath: string;
+    verbose?: boolean;
+};
+export declare function parseBackupPaths(paths: (string | Step)[], options: BackupPathsOptions): Promise<string[]>;

package/utils/datatruck/paths.js

@@ -1,26 +1,35 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.parsePaths = void 0;
-const process_1 = require("../process");
+exports.parseBackupPaths = exports.parsePaths = void 0;
+const steps_1 = require("../steps");
 async function parsePaths(values, options) {
     let paths = [];
     for (const value of values) {
         if (typeof value === "string") {
             paths.push(value);
         }
-        else if (value.type === "spawn") {
-            const spawnResult = await (0, process_1.exec)(value.command, value.args, { cwd: options.cwd }, {
-                log: options.verbose,
-                stderr: { save: true },
-                stdout: { save: true },
+        else {
+            await (0, steps_1.runSteps)(value, {
+                node: { tempDir: options.tempDir },
+                verbose: options.verbose,
+                onLine: (path) => paths.push(path),
             });
-            const spawnFiles = [spawnResult.stderr, spawnResult.stdout].flatMap((text) => text
-                .split(/\r?\n/)
-                .map((v) => v.trim())
-                .filter((v) => !!v.length));
-            paths.push(...spawnFiles);
         }
     }
     return paths;
 }
 exports.parsePaths = parsePaths;
+async function parseBackupPaths(paths, options) {
+    return parsePaths(paths, {
+        cwd: options.targetPath,
+        verbose: options.verbose,
+        vars: {
+            dtt: {
+                package: options.package,
+                snapshot: options.snapshot,
+                targetPath: options.targetPath,
+            },
+        },
+    });
+}
+exports.parseBackupPaths = parseBackupPaths;

package/utils/steps.d.ts

@@ -25,6 +25,8 @@ export type StepOptions = {
         vars?: Record<string, any>;
         tempDir?: () => Promise<string>;
     };
+    cwd?: string;
+    onLine?: (p: string) => any;
     verbose?: boolean;
 };
 export declare function runSteps(input: Step[] | Step, options: StepOptions): Promise<void>;

package/utils/steps.js

@@ -1,17 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runSteps = void 0;
+const fs_1 = require("./fs");
 const process_1 = require("./process");
 const string_1 = require("./string");
-const crypto_1 = require("crypto");
 const promises_1 = require("fs/promises");
-const os_1 = require("os");
 const path_1 = require("path");
 async function runSteps(input, options) {
     const steps = Array.isArray(input) ? input : [input];
     for (const step of steps) {
         if (step.type === "process") {
             await (0, process_1.exec)(step.config.command, (step.config.args || []).map((v) => (0, string_1.render)(v, options.process?.vars || {})), {
+                cwd: options.cwd,
                 env: {
                     ...process.env,
                     ...options.env,
@@ -19,6 +19,12 @@ async function runSteps(input, options) {
                 },
             }, {
                 log: options.verbose,
+                ...(options.onLine && {
+                    stdout: {
+                        parseLines: "skip-empty",
+                        onData: (line) => options.onLine(line),
+                    },
+                }),
             });
         }
         else if (step.type === "node") {
@@ -27,8 +33,7 @@ async function runSteps(input, options) {
                 tempDir = await options.node.tempDir();
             }
             else {
-                tempDir = (0, path_1.join)((0, os_1.tmpdir)(), (0, crypto_1.randomBytes)(8).toString("hex"));
-                await (0, promises_1.mkdir)(tempDir, { recursive: true });
+                tempDir = await (0, fs_1.mkTmpDir)("node-step");
             }
             const scriptPath = (0, path_1.join)(tempDir, "script.js");
             const vars = Object.entries({
@@ -42,6 +47,7 @@ async function runSteps(input, options) {
                 ? [...vars, ...step.config.code].join(";\n")
                 : `${vars.join(";\n")}\n${step.config.code}`);
             await (0, process_1.exec)("node", [scriptPath], {
+                cwd: options.cwd,
                 env: {
                     ...process.env,
                     ...options.env,
@@ -49,6 +55,12 @@ async function runSteps(input, options) {
                 },
             }, {
                 log: options.verbose,
+                ...(options.onLine && {
+                    stdout: {
+                        parseLines: "skip-empty",
+                        onData: (line) => options.onLine(line),
+                    },
+                }),
             });
         }
         else {