@datatruck/cli 0.12.1 → 0.14.0

package/SessionDriver/ConsoleSessionDriver.js

@@ -16,6 +16,11 @@ class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract
         this.chron = (0, date_util_1.createChron)();
     }
     async onInit() {
+        this.tty = this.options.verbose
+            ? false
+            : this.options.progress === "auto"
+                ? process.stdout.isTTY
+                : this.options.progress === "tty";
         this.chron.start();
         this.renderInterval = setInterval(() => {
             if (this.lastMessage)
@@ -24,7 +29,7 @@ class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract
     }
     async onEnd(data) {
         clearInterval(this.renderInterval);
-        if (!this.options.verbose)
+        if (this.tty)
             process.stdout.write(cli_util_1.showCursorCommand);
         (0, cli_util_1.logVars)({
             ...data,
@@ -36,10 +41,10 @@ class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract
     }
     printMessage(message, endMessage) {
         const text = this.renderMessage(message);
-        if (this.options.verbose && this.lastMessageText === text) {
+        if (!this.tty && this.lastMessageText === text) {
             return;
         }
-        if (this.options.verbose) {
+        if (!this.tty) {
             process.stdout.write(`${this.renderSpinner(text)}\n`);
         }
         else {
@@ -58,7 +63,7 @@ class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract
         this.lastMessageText = text;
     }
     renderSpinner(text) {
-        return text.replace("{spinner}", (0, chalk_1.grey)(this.options.verbose ? "?" : (0, cli_util_1.renderSpinner)(this.prints)));
+        return text.replace("{spinner}", (0, chalk_1.grey)(this.tty ? (0, cli_util_1.renderSpinner)(this.prints) : "?"));
     }
     renderMessage(message) {
         const badges = renderBadges([
@@ -66,7 +71,6 @@ class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract
             ...(message.errorBadge ? [message.errorBadge] : []),
         ]);
         const padding = "   ".repeat(message.level ?? 0);
-        const haveProgressBar = typeof message.progressPercent === "number";
         const sessionId = message.sessionId.toString().padStart(2, "0");
         const parts = [
             `${padding}${message.textPrefix} [${(0, chalk_1.grey)(sessionId)}] ${message.text}`,
@@ -89,6 +93,13 @@ class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract
     async onWrite(data) {
         if (data.action === SessionDriverAbstract_1.ActionEnum.Init)
             return;
+        if (data.action === SessionDriverAbstract_1.ActionEnum.Progress && this.options.progressInterval) {
+            const skip = this.lastProgressDate &&
+                Date.now() - this.lastProgressDate < this.options.progressInterval;
+            if (skip)
+                return;
+            this.lastProgressDate = Date.now();
+        }
         const message = {
             sessionId: "sessionId" in data.data ? data.data.sessionId : data.data.id,
             badges: [],
@@ -113,9 +124,7 @@ class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract
             if (data.data.error)
                 message.errorBadge = {
                     name: "error",
-                    value: this.options.verbose
-                        ? data.data.error
-                        : data.data.error.split("\n")[0],
+                    value: this.tty ? data.data.error.split("\n")[0] : data.data.error,
                     color: chalk_1.red,
                 };
         }
@@ -159,8 +168,7 @@ class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract
         if (isHeader && data.action === SessionDriverAbstract_1.ActionEnum.End) {
             return;
         }
-        const endMessage = !this.options.verbose &&
-            (!hasProgress || data.action === SessionDriverAbstract_1.ActionEnum.End || isHeader);
+        const endMessage = this.tty && (!hasProgress || data.action === SessionDriverAbstract_1.ActionEnum.End || isHeader);
         this.printMessage(message, endMessage);
         if (endMessage)
             process.stdout.write("\n");

package/SessionDriver/SessionDriverAbstract.d.ts

@@ -64,13 +64,12 @@ export declare type ReadResultType = {
     repositoryType: string;
     error: string | null;
 };
-export declare abstract class SessionDriverAbstract {
-    readonly options: {
-        verbose?: boolean;
-    };
-    constructor(options: {
-        verbose?: boolean;
-    });
+export declare type SessionDriverOptions = {
+    verbose?: boolean;
+};
+export declare abstract class SessionDriverAbstract<TOptions extends SessionDriverOptions = SessionDriverOptions> {
+    readonly options: TOptions;
+    constructor(options: TOptions);
     onInit(): Promise<void>;
     abstract onWrite(data: WriteDataType): Promise<void>;
     onEnd(data?: Record<string, any>): Promise<void>;

package/Task/GitTask.js

@@ -169,6 +169,7 @@ class GitTask extends TaskAbstract_1.TaskAbstract {
                     basePath: path,
                 },
                 targetPath: outPath,
+                skipNotFoundError: true,
                 concurrency: this.config.fileCopyConcurrency,
                 onPath: async ({ entryPath }) => {
                     currentFiles++;

package/Task/SqlDumpTaskAbstract.js

@@ -15,6 +15,7 @@ const path_1 = require("path");
 exports.sqlDumpTaskDefinition = {
     type: "object",
     required: ["password", "hostname", "username", "database"],
+    additionalProperties: false,
     properties: {
         password: {
             anyOf: [

package/cli.js

@@ -80,6 +80,8 @@ program.description(description);
 program.usage("dtt");
 program.option("-v,--verbose", "Verbose", (_, previous) => previous + 1, 0);
 program.option("-c,--config <path>", "Config path", process.env["DATATRUCK_CONFIG"] ?? (cwd.endsWith(path_1.sep) ? cwd : `${cwd}${path_1.sep}`));
+program.option("--progress <value>", "Progress type (auto, plain, tty)", "auto");
+program.option("--progress-interval <ms>", "Progress interval");
 program.option("-o,--output-format <format>", "Output format (json, pjson, yaml, table, custom=$, tpl=name)", "table");
 makeCommand(CommandFactory_1.CommandEnum.config).alias("c");
 makeCommand(CommandFactory_1.CommandEnum.init).alias("i");

package/config.schema.json

@@ -81,7 +81,7 @@
             "type": "object",
             "properties": {
               "type": {
-                "const": "local"
+                "const": "datatruck"
               }
             }
           },
@@ -89,7 +89,7 @@
             "type": "object",
             "properties": {
               "config": {
-                "$ref": "#/definitions/local-repository"
+                "$ref": "#/definitions/datatruck-repository"
               }
             }
           },
@@ -248,7 +248,7 @@
             "type": "object",
             "properties": {
               "type": {
-                "const": "local"
+                "const": "datatruck"
               }
             }
           },
@@ -256,7 +256,7 @@
             "type": "object",
             "properties": {
               "config": {
-                "$ref": "#/definitions/local-package-repository"
+                "$ref": "#/definitions/datatruck-package-repository"
               }
             }
           },
@@ -438,7 +438,7 @@
       "additionalProperties": false,
       "properties": {}
     },
-    "local-repository": {
+    "datatruck-repository": {
       "type": "object",
       "required": [
         "outPath"
@@ -453,7 +453,7 @@
         }
       }
     },
-    "local-package-repository": {
+    "datatruck-package-repository": {
       "type": "object",
       "additionalProperties": false,
       "properties": {
@@ -475,6 +475,9 @@
                       "include"
                     ],
                     "properties": {
+                      "name": {
+                        "type": "string"
+                      },
                       "include": {
                         "$ref": "#/definitions/stringlist-util"
                       },
@@ -683,6 +686,7 @@
         "username",
         "database"
       ],
+      "additionalProperties": false,
       "properties": {
         "password": {
           "anyOf": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.12.1",
+  "version": "0.14.0",
   "dependencies": {
     "ajv": "^8.11.0",
     "async": "^3.2.4",
@@ -8,8 +8,10 @@
     "cli-table3": "^0.6.2",
     "commander": "^9.4.0",
     "dayjs": "^1.11.5",
+    "fast-folder-size": "^1.7.1",
     "fast-glob": "^3.2.11",
     "micromatch": "^4.0.5",
+    "pretty-bytes": "^5.6.0",
     "sqlite": "^4.1.2",
     "sqlite3": "^5.0.11"
   },

package/util/ResticUtil.d.ts

@@ -10,13 +10,16 @@ export declare type RepositoryType = {
 } & Omit<UriType, "password">;
 export declare type BackupStreamType = {
     message_type: "status";
-    seconds_elapsed: number;
+    seconds_elapsed?: number;
     percent_done: number;
-    total_files: number;
+    total_files?: number;
     files_done?: number;
     total_bytes: number;
     bytes_done?: number;
     current_files?: string[];
+} | {
+    message_type: "restore-status";
+    total_bytes: number;
 } | {
     message_type: "summary";
     files_new: number;
@@ -95,6 +98,10 @@ export declare class ResticUtil {
     restore(options: {
         id: string;
         target: string;
+        /**
+         * @default 30_000
+         */
+        progressInterval?: number | false;
         onStream?: (data: BackupStreamType) => Promise<void>;
     }): Promise<ExecResultType>;
 }

package/util/ResticUtil.js

@@ -29,7 +29,7 @@ class ResticUtil {
     async exec(args, settings, options) {
         return await (0, process_util_1.exec)("restic", args, {
             stdio: ["ignore", "pipe", "pipe"],
-            env: { ...this.options.env },
+            env: { ...process.env, ...this.options.env },
             cwd: options?.cwd,
         }, {
             stderr: { toExitCode: true },
@@ -40,7 +40,11 @@ class ResticUtil {
                     stderr: true,
                     colorize: true,
                     allToStderr: true,
-                    envNames: ["RESTIC_REPOSITORY", "RESTIC_PASSWORD_FILE"],
+                    envNames: [
+                        "RESTIC_REPOSITORY",
+                        "RESTIC_PASSWORD_FILE",
+                        "RESTIC_PASSWORD",
+                    ],
                 }
                 : {},
             ...(settings ?? {}),
@@ -96,11 +100,12 @@ class ResticUtil {
         return JSON.parse(result.stdout);
     }
     async checkBackupSetPathSupport() {
-        const result = await this.exec(["backup", "--set-path"], {
-            onExitCodeError: () => false,
-            stderr: { save: true },
+        /*const result = await this.exec(["backup", "--set-path"], {
+          onExitCodeError: () => false,
+          stderr: { save: true },
         });
-        return result.stderr.includes("flag needs an argument");
+        return result.stderr.includes("flag needs an argument");*/
+        return false;
     }
     async backup(options) {
         const exec = async () => await this.exec([
@@ -174,20 +179,42 @@ class ResticUtil {
         });
     }
     async restore(options) {
-        return await this.exec(["restore", "--json", options.id, "--target", options.target], {
-            stderr: {
-                toExitCode: true,
-            },
-            stdout: {
-                ...(options.onStream && {
-                    onData: async (data) => {
-                        if (data.startsWith("{") && data.endsWith("}")) {
-                            await options.onStream?.(JSON.parse(data));
-                        }
-                    },
-                }),
-            },
-        });
+        let progressTimeout;
+        const progressInterval = options.progressInterval ?? 30000;
+        async function progressRutine() {
+            try {
+                const total_bytes = await (0, fs_util_1.fastFolderSizeAsync)(options.target);
+                options.onStream?.({
+                    message_type: "restore-status",
+                    total_bytes,
+                });
+            }
+            finally {
+                if (typeof progressInterval === "number")
+                    progressTimeout = setTimeout(progressRutine, progressInterval);
+            }
+        }
+        if (typeof progressInterval === "number")
+            progressRutine();
+        try {
+            return await this.exec(["restore", "--json", options.id, "--target", options.target], {
+                stderr: {
+                    toExitCode: true,
+                },
+                stdout: {
+                    ...(options.onStream && {
+                        onData: async (data) => {
+                            if (data.startsWith("{") && data.endsWith("}")) {
+                                await options.onStream?.(JSON.parse(data));
+                            }
+                        },
+                    }),
+                },
+            });
+        }
+        finally {
+            clearTimeout(progressTimeout);
+        }
     }
 }
 exports.ResticUtil = ResticUtil;

package/util/datatruck/config-util.d.ts

@@ -31,17 +31,17 @@ export declare const pkgPathParams: {
     [name in "temp" | Exclude<keyof ResolvePackagePathParamsType, "path">]: string;
 };
 export declare const pkgIncludeParams: {
-    snapshotDate: string;
     packageName: string;
     temp: string;
     snapshotId: string;
+    snapshotDate: string;
     action: string;
 };
 export declare const pkgExcludeParams: {
-    snapshotDate: string;
     packageName: string;
     temp: string;
     snapshotId: string;
+    snapshotDate: string;
     action: string;
 };
 export declare const pkgRestorePathParams: {
@@ -52,32 +52,32 @@ export declare const dbNameParams: {
 };
 export declare const params: {
     pkgPath: {
-        snapshotDate: string;
         packageName: string;
         temp: string;
         snapshotId: string;
+        snapshotDate: string;
         action: string;
     };
     pkgRestorePath: {
-        snapshotDate: string;
         packageName: string;
         path: string;
         temp: string;
         snapshotId: string;
+        snapshotDate: string;
         action: string;
     };
     pkgInclude: {
-        snapshotDate: string;
         packageName: string;
         temp: string;
         snapshotId: string;
+        snapshotDate: string;
         action: string;
     };
     pkgExclude: {
-        snapshotDate: string;
         packageName: string;
         temp: string;
         snapshotId: string;
+        snapshotDate: string;
         action: string;
     };
     dbName: {

package/util/fs-util.d.ts

@@ -1,9 +1,20 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-import { Stats } from "fs";
+/// <reference types="node" />
+import { Dirent, Stats } from "fs";
+import { WriteStream } from "fs";
 import { Interface } from "readline";
 export declare const isWSLSystem: boolean;
+export declare function isEmptyDir(path: string): Promise<boolean>;
+declare type EntryObject = {
+    name: string;
+    path: string;
+    dirent: Dirent;
+    stats: Stats;
+};
+export declare function applyPermissions(baseDir: string, permissionsPath: string): Promise<void>;
+export declare function pathIterator(stream: AsyncIterable<string | Buffer>): AsyncIterable<EntryObject>;
 export declare function isLocalDir(path: string): boolean;
 export declare function isDirEmpty(path: string): Promise<boolean>;
 export declare function mkdirIfNotExists(path: string): Promise<string>;
@@ -23,11 +34,12 @@ export declare function existsFile(path: string): Promise<boolean>;
 export declare function parentTmpDir(): string;
 export declare function sessionTmpDir(): string;
 export declare function tmpDir(prefix: string, id?: string): string;
+export declare function fastFolderSizeAsync(path: string): Promise<number>;
 export declare function mkTmpDir(prefix: string, id?: string): Promise<string>;
 export declare function readPartialFile(path: string, positions: [number, number?]): Promise<string>;
 export declare function checkFile(path: string): Promise<boolean>;
 export declare function checkDir(path: string): Promise<boolean>;
-export declare function readDir(path: string): Promise<string[]>;
+export declare function readDir(path: string, optional?: boolean): Promise<string[]>;
 export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
 /**
  * @experimental
@@ -36,26 +48,10 @@ export declare function fastglobToGitIgnore(patterns: string[], baseDir: string)
 export declare function writeGitIgnoreList(options: {
     paths: NodeJS.ReadableStream | string[];
 }): Promise<string>;
-export declare function writePathLists(options: {
-    paths: NodeJS.ReadableStream | string[];
-    packs?: {
-        include: string[];
-        exclude?: string[];
-        multiple?: boolean;
-    }[];
-}): Promise<{
-    path: string;
-    includedPackPaths: string[];
-    excludedPackPaths: string[];
-    total: {
-        all: number;
-        path: number;
-        packsPaths: number[];
-        multipleStats: Record<string, number>;
-    };
-}>;
+export declare function waitForClose(stream: WriteStream): Promise<WriteStream>;
 export declare function copyFileWithStreams(source: string, target: string): Promise<unknown>;
 export declare function updateFileStats(path: string, fileInfo: Stats): Promise<void>;
+export declare function isNotFoundError(error: unknown): boolean;
 export declare function cpy(options: {
     input: {
         type: "glob";
@@ -76,6 +72,7 @@ export declare function cpy(options: {
      * @default 1
      */
     concurrency?: number;
+    skipNotFoundError?: boolean;
     onPath?: (data: {
         isDir: boolean;
         entryPath: string;
@@ -88,3 +85,4 @@ export declare function cpy(options: {
         };
     }) => Promise<boolean | void>;
 }): Promise<void>;
+export {};