@datatruck/cli 0.16.0 → 0.16.2

package/Task/TaskAbstract.js

@@ -1,9 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TaskAbstract = void 0;
+const fs_util_1 = require("../util/fs-util");
 class TaskAbstract {
     constructor(config) {
         this.config = config;
+        this.tmpDirs = [];
+    }
+    async mkTmpDir(prefix, id) {
+        const dir = await (0, fs_util_1.mkTmpDir)(prefix, id);
+        this.tmpDirs.push(dir);
+        return dir;
     }
     async onBeforeBackup(data) {
         return undefined;

package/package.json

@@ -1,23 +1,23 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.16.0",
+  "version": "0.16.2",
   "dependencies": {
-    "ajv": "^8.11.0",
+    "ajv": "^8.11.2",
     "async": "^3.2.4",
     "chalk": "^4.1.2",
-    "cli-table3": "^0.6.2",
-    "commander": "^9.4.0",
-    "dayjs": "^1.11.5",
+    "cli-table3": "^0.6.3",
+    "commander": "^9.4.1",
+    "dayjs": "^1.11.6",
     "fast-folder-size": "^1.7.1",
-    "fast-glob": "^3.2.11",
+    "fast-glob": "^3.2.12",
     "micromatch": "^4.0.5",
     "pretty-bytes": "^5.6.0",
     "sqlite": "^4.1.2",
-    "sqlite3": "^5.0.11"
+    "sqlite3": "^5.1.2"
   },
   "optionalDependencies": {
     "ts-node": "^10.9.1",
-    "yaml": "^2.1.1"
+    "yaml": "^2.1.3"
   },
   "engine": {
     "node": ">=16.0.0"

package/util/DataFormat.d.ts

@@ -1,4 +1,4 @@
-export declare type FormatType = "json" | "pjson" | "table" | "yaml" | "custom" | "tpl";
+export type FormatType = "json" | "pjson" | "table" | "yaml" | "custom" | "tpl";
 export declare class DataFormat<TItem extends Record<string, unknown>> {
     readonly options: {
         items: TItem[];

package/util/ObjectVault.d.ts

@@ -1,4 +1,4 @@
-export declare type KeysType = (string | number)[];
+export type KeysType = (string | number)[];
 export declare class ObjectVault<TObject> {
     protected counter: number;
     protected readonly ids: Record<string, number>;

package/util/ResticUtil.d.ts

@@ -1,6 +1,6 @@
 import { ExecResultType, ExecSettingsInterface } from "./process-util";
 import { UriType } from "./string-util";
-export declare type RepositoryType = {
+export type RepositoryType = {
     name?: string;
     env?: Record<string, string>;
     password?: string | {
@@ -8,7 +8,7 @@ export declare type RepositoryType = {
     };
     backend: "local" | "rest" | "sftp" | "s3" | "azure" | "gs" | "rclone";
 } & Omit<UriType, "password">;
-export declare type BackupStreamType = {
+export type BackupStreamType = {
     message_type: "status";
     seconds_elapsed?: number;
     percent_done: number;
@@ -90,6 +90,7 @@ export declare class ResticUtil {
         parent?: string;
         allowEmptySnapshot?: boolean;
         onStream?: (data: BackupStreamType) => void;
+        createEmptyDir?: () => Promise<string>;
     }): Promise<ExecResultType>;
     copy(options: {
         id: string;

package/util/ResticUtil.js

@@ -148,16 +148,18 @@ class ResticUtil {
         catch (error) {
             if (options.allowEmptySnapshot &&
                 error.message.includes("unable to save snapshot: snapshot is empty")) {
-                const emptyPath = await (0, fs_util_1.mkTmpDir)("empty");
-                await (0, promises_1.writeFile)(`${emptyPath}/.empty`, "");
-                return await this.backup({
-                    ...options,
-                    cwd: emptyPath,
-                    allowEmptySnapshot: false,
-                    paths: ["."],
-                    exclude: [],
-                    excludeFile: [],
-                });
+                if (options.createEmptyDir) {
+                    const emptyPath = await options.createEmptyDir();
+                    await (0, promises_1.writeFile)(`${emptyPath}/.empty`, "");
+                    return await this.backup({
+                        ...options,
+                        cwd: emptyPath,
+                        allowEmptySnapshot: false,
+                        paths: ["."],
+                        exclude: [],
+                        excludeFile: [],
+                    });
+                }
             }
             throw error;
         }

package/util/cli-util.d.ts

@@ -10,7 +10,7 @@ export declare function logVars(data: Record<string, any>): void;
 export declare function logExec(command: string, argv?: string[], env?: NodeJS.ProcessEnv, logToStderr?: boolean): void;
 export declare function resultColumn(error: Error | null | string, state?: "started" | "ended"): "❌" | " ? " | "✅";
 export declare function errorColumn(error: Error | null | string, verbose: number): string;
-export declare type OptionsType<T1, T2 extends {
+export type OptionsType<T1, T2 extends {
     [K in keyof T1]: unknown;
 }> = {
     [K in keyof Required<T1>]: {

package/util/datatruck/config-util.d.ts

@@ -10,17 +10,17 @@ export declare function filterPackages(config: ConfigType, options: {
     repositoryTypes?: string[];
     sourceAction?: RepositoryConfigEnabledActionType;
 }): PackageConfigType[];
-declare type ResolvePackagePathParamsType = ResolvePackageParamsType & {
+type ResolvePackagePathParamsType = ResolvePackageParamsType & {
     packageName: string;
     path: string | undefined;
 };
 export declare function resolvePackagePath(value: string, params: ResolvePackagePathParamsType): string;
-declare type ResolveDatabaseNameParamsType = ResolvePackageParamsType & {
+type ResolveDatabaseNameParamsType = ResolvePackageParamsType & {
     packageName: string;
     database: string | undefined;
 };
 export declare function resolveDatabaseName(value: string, params: ResolveDatabaseNameParamsType): string;
-declare type ResolvePackageParamsType = {
+type ResolvePackageParamsType = {
     snapshotId: string;
     snapshotDate: string;
     action: "backup" | "restore";
@@ -32,15 +32,15 @@ export declare const pkgPathParams: {
 };
 export declare const pkgIncludeParams: {
     packageName: string;
-    temp: string;
     snapshotId: string;
+    temp: string;
     snapshotDate: string;
     action: string;
 };
 export declare const pkgExcludeParams: {
     packageName: string;
-    temp: string;
     snapshotId: string;
+    temp: string;
     snapshotDate: string;
     action: string;
 };
@@ -53,30 +53,30 @@ export declare const dbNameParams: {
 export declare const params: {
     pkgPath: {
         packageName: string;
-        temp: string;
         snapshotId: string;
+        temp: string;
         snapshotDate: string;
         action: string;
     };
     pkgRestorePath: {
         packageName: string;
         path: string;
-        temp: string;
         snapshotId: string;
+        temp: string;
         snapshotDate: string;
         action: string;
     };
     pkgInclude: {
         packageName: string;
-        temp: string;
         snapshotId: string;
+        temp: string;
         snapshotDate: string;
         action: string;
     };
     pkgExclude: {
         packageName: string;
-        temp: string;
         snapshotId: string;
+        temp: string;
         snapshotDate: string;
         action: string;
     };

package/util/date-util.d.ts

@@ -1,4 +1,4 @@
-export declare type FilterByLastOptionsType = {
+export type FilterByLastOptionsType = {
     last?: number;
     lastMinutely?: number;
     lastHourly?: number;

package/util/fs-util.d.ts

@@ -7,7 +7,7 @@ import { WriteStream } from "fs";
 import { Interface } from "readline";
 export declare const isWSLSystem: boolean;
 export declare function isEmptyDir(path: string): Promise<boolean>;
-declare type EntryObject = {
+type EntryObject = {
     name: string;
     path: string;
     dirent: Dirent;
@@ -33,6 +33,8 @@ export declare function findFile(sourcePath: string, baseName: string, extension
 export declare function existsFile(path: string): Promise<boolean>;
 export declare function parentTmpDir(): string;
 export declare function sessionTmpDir(): string;
+export declare function isTmpDir(path: string): boolean;
+export declare function rmTmpDir(input: string | string[]): Promise<void>;
 export declare function tmpDir(prefix: string, id?: string): string;
 export declare function fastFolderSizeAsync(path: string): Promise<number>;
 export declare function mkTmpDir(prefix: string, id?: string): Promise<string>;
@@ -47,6 +49,7 @@ export declare function forEachFile(dirPath: string, cb: (path: string, dir: boo
 export declare function fastglobToGitIgnore(patterns: string[], baseDir: string): string[];
 export declare function writeGitIgnoreList(options: {
     paths: NodeJS.ReadableStream | string[];
+    outDir: string;
 }): Promise<string>;
 export declare function waitForClose(stream: WriteStream): Promise<WriteStream>;
 export declare function copyFileWithStreams(source: string, target: string): Promise<unknown>;

package/util/fs-util.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.pathIterator = exports.applyPermissions = exports.isEmptyDir = exports.isWSLSystem = void 0;
+exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.pathIterator = exports.applyPermissions = exports.isEmptyDir = exports.isWSLSystem = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
 const path_util_1 = require("./path-util");
 const async_1 = require("async");
@@ -160,6 +160,24 @@ function sessionTmpDir() {
     return (0, path_1.join)(parentTmpDir(), process.pid.toString());
 }
 exports.sessionTmpDir = sessionTmpDir;
+function isTmpDir(path) {
+    return path.startsWith(sessionTmpDir()) && path.includes("datatruck-temp");
+}
+exports.isTmpDir = isTmpDir;
+async function rmTmpDir(input) {
+    if (typeof input === "string") {
+        if (!isTmpDir(input))
+            throw new Error(`Path is not a temp dir: ${input}`);
+        await (0, promises_1.rm)(input, {
+            recursive: true,
+        });
+    }
+    else {
+        for (const path of input)
+            await rmTmpDir(path);
+    }
+}
+exports.rmTmpDir = rmTmpDir;
 function tmpDir(prefix, id) {
     if (!id)
         id = (0, crypto_1.randomUUID)().slice(0, 8);
@@ -265,8 +283,8 @@ function fastglobToGitIgnore(patterns, baseDir) {
 }
 exports.fastglobToGitIgnore = fastglobToGitIgnore;
 async function writeGitIgnoreList(options) {
-    const tempDir = await mkTmpDir("gitignore-list");
-    const path = (0, path_1.join)(tempDir, `.gitignore`);
+    const { outDir } = options;
+    const path = (0, path_1.join)(outDir, `.gitignore`);
     const stream = (0, fs_2.createWriteStream)(path);
     const dirs = new Set();
     stream.write("*\n");

package/util/object-util.d.ts

@@ -1,7 +1,7 @@
 export declare function merge<T extends Record<string, unknown>>(target: T, ...sources: Record<string, unknown>[]): T;
 export declare function push<T>(map: Record<string, T[]>, key: string, object: T): void;
 export declare function getErrorProperties(error: Error): Record<string, string>;
-declare type GroupByKeyParamType<TItem> = ((item: TItem) => string[] | string) | (keyof TItem)[] | keyof TItem;
+type GroupByKeyParamType<TItem> = ((item: TItem) => string[] | string) | (keyof TItem)[] | keyof TItem;
 export declare function groupBy<TItem>(items: TItem[], keyOrCb: GroupByKeyParamType<TItem>): Record<string, TItem[]>;
 export declare function groupBy<TItem>(items: TItem[], keyOrCb: GroupByKeyParamType<TItem>, single: true): Record<string, TItem>;
 export {};

package/util/process-util.d.ts

@@ -2,7 +2,7 @@
 /// <reference types="node" />
 import { SpawnOptions, ChildProcess } from "child_process";
 import { ReadStream, WriteStream } from "fs";
-export declare type ExecLogSettingsType = {
+export type ExecLogSettingsType = {
     colorize?: boolean;
     exec?: boolean;
     stdout?: boolean;
@@ -44,12 +44,12 @@ export declare function logExecStdout(input: {
     lineSalt?: boolean;
 }): void;
 export declare function logExecStderr(data: string, colorize?: boolean): void;
-export declare type ExecResultType = {
+export type ExecResultType = {
     stdout: string;
     stderr: string;
     exitCode: number;
 };
 export declare function exec(command: string, argv?: string[], options?: SpawnOptions | null, settings?: ExecSettingsInterface): Promise<ExecResultType>;
-declare type EventNameType = "exit" | "SIGINT" | "SIGUSR1" | "SIGUSR2" | "SIGTERM" | "uncaughtException";
+type EventNameType = "exit" | "SIGINT" | "SIGUSR1" | "SIGUSR2" | "SIGTERM" | "uncaughtException";
 export declare function onExit(cb: (eventName: EventNameType, ...args: any[]) => void): void;
 export {};

package/util/progress.d.ts

@@ -1,4 +1,4 @@
-export declare type ProgressStats = {
+export type ProgressStats = {
     percent?: number;
     total?: number;
     current?: number;
@@ -6,7 +6,7 @@ export declare type ProgressStats = {
     payload?: string;
     format?: "amount" | "size";
 };
-export declare type Progress = {
+export type Progress = {
     absolute?: ProgressStats;
     relative?: ProgressStats;
 };

package/util/string-util.d.ts

@@ -4,7 +4,7 @@ export declare function lcfirst(value: string): string;
 export declare function snakeCase(value: string, char?: string): string;
 export declare function render(subject: string, vars: Record<string, string | undefined>): string;
 export declare function parseStringList(value: string, validValues?: string[]): string[];
-export declare type UriType = {
+export type UriType = {
     protocol?: "http" | "https";
     host?: string;
     username?: string;

package/util/zip-util.d.ts

@@ -37,7 +37,7 @@ export interface UnzipDataType {
 }
 export declare function buildArguments(filters: (ZipDataFilterType | string)[]): string[];
 export declare function checkSSEOption(command?: string): Promise<boolean>;
-declare type ListZipStream = {
+type ListZipStream = {
     Path?: string;
     Folder?: string;
     Size?: string;
@@ -62,7 +62,7 @@ export declare function listZip(data: {
     onStream: (item: ListZipStream) => Promise<void>;
     verbose?: boolean;
 }): Promise<void>;
-export declare type ZipStream = {
+export type ZipStream = {
     type: "progress";
     data: {
         progress: number;
@@ -80,7 +80,7 @@ export declare function zip(data: ZipDataType): Promise<{
     folders: number;
     files: number;
 }>;
-export declare type UnzipStream = {
+export type UnzipStream = {
     type: "progress";
     data: {
         percent: number;

package/util/zip-util.js

@@ -173,9 +173,9 @@ async function zip(data) {
 exports.zip = zip;
 function parseUnzipLine(line) {
     let matches = null;
-    if ((matches = /^\s*(\d+)% (\d+) \-/.exec(line))) {
-        const progress = Number(matches[1]);
-        const files = Number(matches[2]);
+    if ((matches = /^\s*(?<percent>\d+)%(?: (?<files>\d+))? \-/.exec(line))) {
+        const progress = Number(matches.groups["percent"]);
+        const files = matches.groups["files"] ? Number(matches[2]) : 1;
         const path = line.slice(line.indexOf("-") + 1).trim();
         return {
             type: "progress",
@@ -204,20 +204,22 @@ async function unzip(data) {
         stderr: { toExitCode: true },
         stdout: {
             ...((data.onStream || data.onProgress) && {
-                parseLines: true,
-                onData: async (line) => {
-                    const stream = parseUnzipLine(line);
-                    if (stream) {
-                        if (stream.type === "progress") {
-                            const current = Math.max(0, stream.data.files - 1);
-                            summary.files = stream.data.files;
-                            await data.onProgress?.({
-                                current,
-                                percent: stream.data.percent,
-                                path: stream.data.path,
-                            });
+                onData: async (chunk) => {
+                    const lines = chunk.replaceAll("\b", "").split(/\r?\n/);
+                    for (const line of lines) {
+                        const stream = parseUnzipLine(line);
+                        if (stream) {
+                            if (stream.type === "progress") {
+                                const current = Math.max(0, stream.data.files - 1);
+                                summary.files = stream.data.files;
+                                await data.onProgress?.({
+                                    current,
+                                    percent: stream.data.percent,
+                                    path: stream.data.path,
+                                });
+                            }
+                            await data.onStream?.(stream);
                         }
-                        await data.onStream?.(stream);
                     }
                 },
             }),