@datatruck/cli 0.18.0 → 0.20.0

package/Task/MssqlTask.js

@@ -25,6 +25,8 @@ exports.mssqlTaskDefinition = {
     },
 };
 class MssqlTask extends TaskAbstract_1.TaskAbstract {
+    static SUFFIX = ".BAK";
+    verbose;
     get command() {
         return this.config.command ?? "sqlcmd";
     }
@@ -106,4 +108,3 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
     }
 }
 exports.MssqlTask = MssqlTask;
-MssqlTask.SUFFIX = ".BAK";

package/Task/ScriptTask.js

@@ -92,6 +92,7 @@ exports.scriptTaskDefinition = {
     },
 };
 class ScriptTask extends TaskAbstract_1.TaskAbstract {
+    verbose;
     async onBeforeBackup() {
         return {
             targetPath: await this.mkTmpDir(ScriptTask.name),

package/Task/SqlDumpTaskAbstract.js

@@ -85,6 +85,7 @@ function parseSqlFile(fileName) {
     }
 }
 class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
+    verbose;
     async fetchPassword() {
         if (typeof this.config.password === "string")
             return this.config.password;
@@ -114,7 +115,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
             return true;
         });
         (0, assert_1.ok)(typeof outputPath === "string");
-        if (!(await (0, fs_1.checkDir)(outputPath)))
+        if (!(await (0, fs_1.existsDir)(outputPath)))
             await (0, promises_1.mkdir)(outputPath, { recursive: true });
         if (!this.config.oneFileByTable) {
             const outPath = (0, path_1.join)(outputPath, serializeSqlFile({ database: this.config.database }));

package/Task/TaskAbstract.js

@@ -3,9 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.TaskAbstract = void 0;
 const fs_1 = require("../utils/fs");
 class TaskAbstract {
+    config;
+    tmpDirs = [];
     constructor(config) {
         this.config = config;
-        this.tmpDirs = [];
     }
     async mkTmpDir(prefix, id) {
         const dir = await (0, fs_1.mkTmpDir)(prefix, id);

package/config.schema.json

@@ -458,45 +458,31 @@
       "additionalProperties": false,
       "properties": {
         "compress": {
-          "anyOf": [
-            {
-              "type": "boolean"
-            },
-            {
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "packs": {
-                  "type": "array",
-                  "items": {
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                      "include"
-                    ],
-                    "properties": {
-                      "name": {
-                        "type": "string"
-                      },
-                      "include": {
-                        "$ref": "#/definitions/stringlist-util"
-                      },
-                      "exclude": {
-                        "$ref": "#/definitions/stringlist-util"
-                      },
-                      "onePackByResult": {
-                        "type": "boolean"
-                      }
-                    }
-                  }
-                }
+          "type": "boolean"
+        },
+        "packs": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "include"
+            ],
+            "properties": {
+              "name": {
+                "type": "string"
+              },
+              "include": {
+                "$ref": "#/definitions/stringlist-util"
+              },
+              "exclude": {
+                "$ref": "#/definitions/stringlist-util"
+              },
+              "onePackByResult": {
+                "type": "boolean"
               }
             }
-          ]
-        },
-        "fileCopyConcurrency": {
-          "type": "integer",
-          "minimum": 1
+          }
         }
       }
     },

package/index.d.ts

@@ -0,0 +1 @@
+export {};

package/index.js

@@ -1 +1,2 @@
 "use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,23 +1,23 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.18.0",
+  "version": "0.20.0",
   "dependencies": {
     "ajv": "^8.12.0",
     "async": "^3.2.4",
     "chalk": "^4.1.2",
     "cli-table3": "^0.6.3",
-    "commander": "^9.4.1",
-    "dayjs": "^1.11.7",
-    "fast-folder-size": "^1.7.1",
-    "fast-glob": "^3.2.12",
+    "commander": "^11.0.0",
+    "dayjs": "^1.11.10",
+    "fast-folder-size": "^2.2.0",
+    "fast-glob": "^3.3.1",
     "micromatch": "^4.0.5",
     "pretty-bytes": "^5.6.0",
-    "sqlite": "^4.1.2",
-    "sqlite3": "^5.1.4"
+    "sqlite": "^5.0.1",
+    "sqlite3": "^5.1.6"
   },
   "optionalDependencies": {
     "ts-node": "^10.9.1",
-    "yaml": "^2.2.1"
+    "yaml": "^2.3.2"
   },
   "engine": {
     "node": ">=16.0.0"

package/utils/DataFormat.js

@@ -10,6 +10,7 @@ const util_1 = require("util");
 const customPrefix = "custom=";
 const tplPrefix = "tpl=";
 class DataFormat {
+    options;
     constructor(options) {
         this.options = options;
     }

package/utils/Git.js

@@ -4,6 +4,7 @@ exports.Git = void 0;
 const fs_1 = require("./fs");
 const process_1 = require("./process");
 class Git {
+    options;
     constructor(options) {
         this.options = options;
     }
@@ -15,7 +16,7 @@ class Git {
     }
     async canBeInit(repo) {
         return ((0, fs_1.isLocalDir)(repo) &&
-            (!(await (0, fs_1.checkDir)(repo)) || !(await (0, fs_1.readDir)(repo)).length));
+            (!(await (0, fs_1.existsDir)(repo)) || !(await (0, fs_1.readDir)(repo)).length));
     }
     async clone(options) {
         return await this.exec([

package/utils/ObjectVault.js

@@ -2,11 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ObjectVault = void 0;
 class ObjectVault {
-    constructor() {
-        this.counter = 0;
-        this.ids = {};
-        this.objects = {};
-    }
+    counter = 0;
+    ids = {};
+    objects = {};
     static serializeKeys(keys) {
         return JSON.stringify(keys);
     }

package/utils/Restic.js

@@ -7,6 +7,7 @@ const string_1 = require("./string");
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
 class Restic {
+    options;
     constructor(options) {
         this.options = options;
     }

package/utils/datatruck/paths.js

@@ -9,7 +9,11 @@ async function parsePaths(values, options) {
             paths.push(value);
         }
         else if (value.type === "spawn") {
-            const spawnResult = await (0, process_1.exec)(value.command, value.args, { cwd: options.cwd }, { log: options.verbose, stderr: { save: true }, stdout: { save: true } });
+            const spawnResult = await (0, process_1.exec)(value.command, value.args, { cwd: options.cwd }, {
+                log: options.verbose,
+                stderr: { save: true },
+                stdout: { save: true },
+            });
             const spawnFiles = [spawnResult.stderr, spawnResult.stdout].flatMap((text) => text
                 .split(/\r?\n/)
                 .map((v) => v.trim())

package/utils/fs.d.ts

@@ -1,29 +1,20 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-/// <reference types="node" />
 import { Progress } from "./progress";
 import { Entry, Options } from "fast-glob";
-import { Dirent, ReadStream, Stats } from "fs";
+import { ReadStream, Stats } from "fs";
 import { WriteStream } from "fs";
 import { Interface } from "readline";
 export declare const isWSLSystem: boolean;
 export declare function isEmptyDir(path: string): Promise<boolean>;
-type EntryObject = {
-    name: string;
-    path: string;
-    dirent: Dirent;
-    stats: Stats;
-};
-export declare function applyPermissions(baseDir: string, permissionsPath: string): Promise<void>;
-export declare function pathIterator(stream: AsyncIterable<string | Buffer>): AsyncIterable<EntryObject>;
 export declare function isLocalDir(path: string): boolean;
-export declare function isDirEmpty(path: string): Promise<boolean>;
 export declare function mkdirIfNotExists(path: string): Promise<string>;
 export declare function ensureEmptyDir(path: string): Promise<void>;
+export declare function safeStat(path: string): Promise<Stats | undefined>;
 export declare function existsDir(path: string): Promise<boolean>;
+export declare function existsFile(path: string): Promise<boolean>;
 export declare function writeJSONFile<T = any>(path: string, json: T): Promise<void>;
-export declare function readdirIfExists(path: string): Promise<string[]>;
 export declare const parseFileExtensions: string[];
 export declare function parseFile(path: string, jsKey?: string): Promise<any>;
 export declare function parsePackageFile(): {
@@ -32,7 +23,6 @@ export declare function parsePackageFile(): {
     description: string;
 };
 export declare function findFile(sourcePath: string, baseName: string, extensions: string[], errorMessage?: string): Promise<string>;
-export declare function existsFile(path: string): Promise<boolean>;
 export declare function parentTmpDir(): string;
 export declare function sessionTmpDir(): string;
 export declare function isTmpDir(path: string): boolean;
@@ -41,8 +31,6 @@ export declare function tmpDir(prefix: string, id?: string): string;
 export declare function fastFolderSizeAsync(path: string): Promise<number>;
 export declare function mkTmpDir(prefix: string, id?: string): Promise<string>;
 export declare function readPartialFile(path: string, positions: [number, number?]): Promise<string>;
-export declare function checkFile(path: string): Promise<boolean>;
-export declare function checkDir(path: string): Promise<boolean>;
 export declare function readDir(path: string, optional?: boolean): Promise<string[]>;
 export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
 /**
@@ -104,18 +92,29 @@ export declare function createFileScanner(options: {
         include: string[];
     };
     onProgress: (data: Progress) => Promise<void>;
-    disableCounting?: boolean;
-    disableEndProgress?: boolean;
 }): Promise<{
+    disposed: boolean;
     total: number;
     current: number;
-    progress: (description: string, data: {
-        path?: string;
-        current: number;
-        type?: "start" | "end";
-        percent?: number;
-    }) => Promise<void>;
-    updateProgress: (end?: boolean) => Promise<void>;
+    progress: (description: string, path?: string) => Promise<void>;
+    end: () => Promise<void>;
     start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
 }>;
+type StreamItem = {
+    key: string;
+    stream: WriteStream;
+    finished: boolean;
+    error?: Error;
+    written?: boolean;
+};
+export declare function createWriteStreamPool(options: {
+    path: string;
+    onStreamPath?: (key: string) => string;
+}): {
+    pool: Record<string, StreamItem>;
+    path(key: string | number): string | undefined;
+    writeLine(key: string | number, v: string): boolean;
+    end(): Promise<void>;
+};
+export declare function countFileLines(path: string): Promise<number>;
 export {};

package/utils/fs.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = exports.pathIterator = exports.applyPermissions = exports.isEmptyDir = exports.isWSLSystem = void 0;
+exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
 const math_1 = require("./math");
 const path_1 = require("./path");
@@ -35,37 +35,13 @@ async function isEmptyDir(path) {
     }
 }
 exports.isEmptyDir = isEmptyDir;
-async function applyPermissions(baseDir, permissionsPath) {
-    const singleReader = (0, readline_1.createInterface)({
-        input: (0, fs_1.createReadStream)(permissionsPath),
-    });
-    for await (const line of singleReader) {
-        const [rpath, rawUid, rawGui, rawMode] = line.split(":");
-        const path = (0, path_2.join)(baseDir, rpath);
-        if (!path.startsWith(baseDir)) {
-            throw new Error(`Entry path is out of the base dir: (${path}, ${baseDir})`);
-        }
-        const uid = Number(rawUid);
-        const guid = Number(rawGui);
-        await (0, promises_1.chown)(path, uid, guid);
-        const mode = Number(rawMode);
-        await (0, promises_1.chmod)(path, mode);
-    }
-}
-exports.applyPermissions = applyPermissions;
 function pathIterator(stream) {
     return stream;
 }
-exports.pathIterator = pathIterator;
 function isLocalDir(path) {
     return /^[\/\.]|([A-Z]:)/i.test(path);
 }
 exports.isLocalDir = isLocalDir;
-async function isDirEmpty(path) {
-    const files = await readDir(path);
-    return !files.length;
-}
-exports.isDirEmpty = isDirEmpty;
 async function mkdirIfNotExists(path) {
     await (0, promises_1.mkdir)(path, {
         recursive: true,
@@ -74,28 +50,29 @@ async function mkdirIfNotExists(path) {
 }
 exports.mkdirIfNotExists = mkdirIfNotExists;
 async function ensureEmptyDir(path) {
-    if (!(await isDirEmpty(path)))
+    if (!(await isEmptyDir(path)))
         throw new Error(`Dir is not empty: ${path}`);
 }
 exports.ensureEmptyDir = ensureEmptyDir;
-async function existsDir(path) {
+async function safeStat(path) {
     try {
-        const info = await (0, promises_1.stat)(path);
-        return info.isDirectory();
-    }
-    catch (e) {
-        return false;
+        return await (0, promises_1.stat)(path);
     }
+    catch (e) { }
+}
+exports.safeStat = safeStat;
+async function existsDir(path) {
+    return (await safeStat(path))?.isDirectory() ?? false;
 }
 exports.existsDir = existsDir;
+async function existsFile(path) {
+    return (await safeStat(path))?.isFile() ?? false;
+}
+exports.existsFile = existsFile;
 async function writeJSONFile(path, json) {
     await (0, promises_1.writeFile)(path, JSON.stringify(json));
 }
 exports.writeJSONFile = writeJSONFile;
-async function readdirIfExists(path) {
-    return await readDir(path, true);
-}
-exports.readdirIfExists = readdirIfExists;
 exports.parseFileExtensions = ["json", "js", "ts", "yaml", "yml"];
 async function parseFile(path, jsKey) {
     if (!(0, path_3.isAbsolute)(path))
@@ -140,16 +117,6 @@ async function findFile(sourcePath, baseName, extensions, errorMessage = "Path n
     return path;
 }
 exports.findFile = findFile;
-async function existsFile(path) {
-    try {
-        const info = await (0, promises_1.stat)(path);
-        return info.isFile();
-    }
-    catch (e) {
-        return false;
-    }
-}
-exports.existsFile = existsFile;
 function parentTmpDir() {
     return (0, path_2.join)(globalData_1.default.tempDir, "datatruck-temp");
 }
@@ -219,24 +186,6 @@ async function readPartialFile(path, positions) {
     });
 }
 exports.readPartialFile = readPartialFile;
-async function checkFile(path) {
-    try {
-        return (await (0, promises_1.stat)(path)).isFile();
-    }
-    catch (e) {
-        return false;
-    }
-}
-exports.checkFile = checkFile;
-async function checkDir(path) {
-    try {
-        return (await (0, promises_1.stat)(path)).isDirectory();
-    }
-    catch (e) {
-        return false;
-    }
-}
-exports.checkDir = checkDir;
 async function readDir(path, optional) {
     try {
         return await (0, promises_1.readdir)(path);
@@ -458,62 +407,125 @@ async function cpy(options) {
 exports.cpy = cpy;
 async function createFileScanner(options) {
     const object = {
+        disposed: false,
         total: 0,
         current: 0,
-        progress: async (description, data) => {
+        progress: async (description, path) => {
+            if (object.disposed)
+                return;
+            if (path)
+                object.current++;
             await options.onProgress({
                 relative: {
                     description,
-                    payload: data.path,
-                    percent: data.percent,
+                    payload: path,
                 },
                 absolute: {
                     total: object.total,
-                    current: object.current + data.current,
-                    percent: (0, math_1.progressPercent)(object.total, object.current + data.current),
+                    current: object.current,
+                    percent: (0, math_1.progressPercent)(object.total, object.current),
                 },
             });
-            if (data.type === "end") {
-                object.current += data.current;
-            }
         },
-        updateProgress: async (end) => {
-            const currentTime = performance.now();
-            const diff = currentTime - lastTime;
-            if (end || diff > 1000) {
-                await options.onProgress({
-                    relative: {
-                        description: end ? "Scanned files" : "Scanning files",
-                        payload: object.total.toString(),
-                    },
-                });
-                lastTime = currentTime;
-            }
+        end: async () => {
+            if (!object.disposed)
+                await object.progress("Finished");
+            object.disposed = true;
         },
         start: async (cb) => {
+            let lastTime = performance.now();
+            await object.progress("Scanning files");
             for await (const entry of pathIterator(stream)) {
-                if (!options.disableCounting)
+                if (cb) {
+                    if (await cb(entry))
+                        object.total++;
+                }
+                else {
                     object.total++;
-                await object.updateProgress();
-                if (cb)
-                    await cb(entry);
+                }
+                if (lastTime - performance.now() > 500)
+                    await object.progress("Scanning files");
             }
-            if (!options.disableEndProgress)
-                await object.updateProgress(true);
+            await object.progress("Scanned files");
         },
     };
-    await options.onProgress({
-        relative: {
-            description: "Scanning files",
-        },
-    });
     const stream = fast_glob_1.default.stream(options.glob.include, {
         dot: true,
         markDirectories: true,
         stats: true,
         ...options.glob,
     });
-    let lastTime = performance.now();
     return object;
 }
 exports.createFileScanner = createFileScanner;
+function createWriteStreamPool(options) {
+    const pool = {};
+    const create = (key) => {
+        const item = {
+            key,
+            stream: (0, fs_2.createWriteStream)((0, path_2.join)(options.path, options.onStreamPath ? options.onStreamPath(key) : key)),
+            finished: false,
+        };
+        item.stream
+            .once("error", (error) => {
+            item.finished = true;
+            item.error = error;
+        })
+            .once("close", () => (item.finished = true));
+        return (pool[item.key] = item);
+    };
+    return {
+        pool,
+        path(key) {
+            const item = pool[key];
+            if (!item)
+                return;
+            if (typeof item.stream.path !== "string")
+                throw new Error(`Stream path is not defined: ${key}`);
+            return item.stream.path;
+        },
+        writeLine(key, v) {
+            const item = pool[key] || create(key.toString());
+            if (item.finished) {
+                return false;
+            }
+            else if (item.written) {
+                return item.stream.write(`\n${v}`);
+            }
+            else {
+                item.written = true;
+                return item.stream.write(`${v}`);
+            }
+        },
+        async end() {
+            const items = Object.values(pool);
+            for (const item of items)
+                if (!item.finished)
+                    item.stream.end();
+            const itemWithErrors = items.filter((v) => v.error);
+            if (itemWithErrors.length) {
+                const keys = itemWithErrors.map((item) => item.key);
+                throw new AggregateError(itemWithErrors.map((item) => item.error), `Streams faileds: ${keys.join(", ")}`);
+            }
+            await Promise.all(items
+                .filter((item) => !item.finished)
+                .map((item) => waitForClose(item.stream)));
+        },
+    };
+}
+exports.createWriteStreamPool = createWriteStreamPool;
+function countFileLines(path) {
+    let lines = 0;
+    const rl = (0, readline_1.createInterface)({
+        input: (0, fs_1.createReadStream)(path),
+    });
+    return new Promise((resolve, reject) => {
+        rl.on("line", (line) => {
+            if (!line.length)
+                lines++;
+        });
+        rl.on("close", () => resolve(lines));
+        rl.on("error", reject);
+    });
+}
+exports.countFileLines = countFileLines;

package/utils/process.d.ts

@@ -33,7 +33,7 @@ export interface ExecSettingsInterface {
     onSpawn?: (p: ChildProcess) => any;
     stdout?: {
         save?: boolean;
-        parseLines?: boolean;
+        parseLines?: boolean | "skip-empty";
         onData?: (data: string) => void;
     };
     stderr?: {