@datatruck/cli 0.5.0 → 0.7.0

package/util/ResticUtil.js

@@ -4,7 +4,6 @@ exports.ResticUtil = void 0;
 const fs_util_1 = require("./fs-util");
 const process_util_1 = require("./process-util");
 const string_util_1 = require("./string-util");
-const fs_extra_1 = require("fs-extra");
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
 class ResticUtil {
@@ -20,7 +19,7 @@ class ResticUtil {
         if (input.passwordFile)
             input = {
                 ...input,
-                password: (await (0, fs_extra_1.readFile)(input.passwordFile)).toString(),
+                password: (await (0, promises_1.readFile)(input.passwordFile)).toString(),
             };
         return `${input.backend}:${(0, string_util_1.formatUri)(input, hidePassword)}`;
     }
@@ -116,7 +115,7 @@ class ResticUtil {
             },
             stdout: {
                 ...(options.onStream && {
-                    onData: async (data) => {
+                    onData: (data) => {
                         for (const rawLine of data.split("\n")) {
                             const line = rawLine.trim();
                             if (line.startsWith("{") && line.endsWith("}")) {
@@ -126,7 +125,7 @@ class ResticUtil {
                                 }
                                 catch (error) { }
                                 if (parsedLine)
-                                    await options.onStream?.(parsedLine);
+                                    options.onStream?.(parsedLine);
                             }
                         }
                     },

package/util/cli-util.d.ts

@@ -6,6 +6,7 @@ export declare const clearCommand = "\r\u001B[K";
 export declare const hideCursorCommand = "\u001B[?25l";
 export declare function renderSpinner(counter: number): string;
 export declare function renderProgressBar(progress: number, size?: number): string;
+export declare function logVars(data: Record<string, any>): void;
 export declare function logExec(command: string, argv?: string[], env?: NodeJS.ProcessEnv, logToStderr?: boolean): void;
 export declare function resultColumn(error: Error | null | string, state?: "started" | "ended"): "❌" | " ? " | "✅";
 export declare function errorColumn(error: Error | null | string, verbose: number): string;

package/util/cli-util.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.confirm = exports.truncate = exports.parseOptions = exports.errorColumn = exports.resultColumn = exports.logExec = exports.renderProgressBar = exports.renderSpinner = exports.hideCursorCommand = exports.clearCommand = exports.showCursorCommand = exports.spinnerChars = exports.clearLastLine = void 0;
+exports.confirm = exports.truncate = exports.parseOptions = exports.errorColumn = exports.resultColumn = exports.logExec = exports.logVars = exports.renderProgressBar = exports.renderSpinner = exports.hideCursorCommand = exports.clearCommand = exports.showCursorCommand = exports.spinnerChars = exports.clearLastLine = void 0;
 const chalk_1 = __importDefault(require("chalk"));
 const chalk_2 = require("chalk");
 const readline_1 = require("readline");
@@ -28,6 +28,18 @@ function renderProgressBar(progress, size = 10) {
     return completeChar.repeat(completedSize) + incompleteChar.repeat(restSize);
 }
 exports.renderProgressBar = renderProgressBar;
+function logVars(data) {
+    let first = true;
+    for (const key in data) {
+        if (first) {
+            console.log();
+            first = false;
+        }
+        const value = data[key];
+        console.log(`${chalk_1.default.cyan(key)}${chalk_1.default.grey(":")} ${chalk_1.default.white(value ?? "")}`);
+    }
+}
+exports.logVars = logVars;
 function logExec(command, argv = [], env, logToStderr) {
     const envText = env
         ? Object.keys(env)

package/util/datatruck/config-util.d.ts

@@ -5,6 +5,7 @@ export declare function findRepositoryOrFail(config: ConfigType, repositoryName:
 export declare function filterRepository(repository: RepositoryConfigType, action?: RepositoryConfigEnabledActionType): boolean;
 export declare function filterPackages(config: ConfigType, options: {
     packageNames?: string[];
+    packageTaskNames?: string[];
     repositoryNames?: string[];
     repositoryTypes?: string[];
     sourceAction?: RepositoryConfigEnabledActionType;

package/util/datatruck/config-util.js

@@ -22,6 +22,7 @@ function filterRepository(repository, action) {
 exports.filterRepository = filterRepository;
 function filterPackages(config, options) {
     const packagePatterns = (0, string_util_1.makePathPatterns)(options.packageNames);
+    const taskNamePatterns = (0, string_util_1.makePathPatterns)(options.packageTaskNames);
     return config.packages
         .map((pkg) => {
         pkg = Object.assign({}, pkg);
@@ -37,6 +38,8 @@ function filterPackages(config, options) {
         return pkg;
     })
         .filter((pkg) => {
+        if (taskNamePatterns && !(0, micromatch_1.isMatch)(pkg.task?.name ?? "", taskNamePatterns))
+            return false;
         return ((typeof pkg.enabled !== "boolean" || pkg.enabled) &&
             !!pkg.repositoryNames?.length &&
             (!packagePatterns || (0, micromatch_1.isMatch)(pkg.name, packagePatterns)));

package/util/date-util.d.ts

@@ -10,3 +10,7 @@ export declare type FilterByLastOptionsType = {
 export declare function filterByLast<TItem extends {
     date: string;
 }>(items: TItem[], options: FilterByLastOptionsType, reasons?: Record<number, string[]>): TItem[];
+export declare function createChron(): {
+    start: () => number;
+    elapsed: (formatted?: boolean | undefined) => string | number;
+};

package/util/date-util.js

@@ -3,7 +3,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.filterByLast = void 0;
+exports.createChron = exports.filterByLast = void 0;
+const string_util_1 = require("./string-util");
 const dayjs_1 = __importDefault(require("dayjs"));
 const advancedFormat_1 = __importDefault(require("dayjs/plugin/advancedFormat"));
 const isoWeek_1 = __importDefault(require("dayjs/plugin/isoWeek"));
@@ -72,3 +73,18 @@ function filterByLast(items, options, reasons) {
     return items.filter((item) => validItems.includes(item));
 }
 exports.filterByLast = filterByLast;
+function createChron() {
+    let startTime;
+    return {
+        start: () => (startTime = Date.now()),
+        elapsed: (formatted) => {
+            if (!startTime)
+                throw new Error(`Chron was not started`);
+            const seconds = (Date.now() - startTime) / 1000;
+            if (formatted)
+                return (0, string_util_1.formatSeconds)(seconds);
+            return seconds;
+        },
+    };
+}
+exports.createChron = createChron;

package/util/fs-util.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="node" />
+import { Interface } from "readline";
 export declare function isLocalDir(path: string): boolean;
 export declare function isDirEmpty(path: string): Promise<boolean>;
 export declare function mkdirIfNotExists(path: string): Promise<string>;
@@ -44,3 +45,35 @@ export declare function writePathLists(options: {
         multipleStats: Record<string, number>;
     };
 }>;
+export declare function cpy(options: {
+    input: {
+        type: "glob";
+        sourcePath: string;
+        include?: string[];
+        exclude?: string[];
+    } | {
+        type: "stream";
+        basePath: string;
+        value: Interface;
+    } | {
+        type: "pathList";
+        path: string;
+        basePath: string;
+    };
+    targetPath: string;
+    /**
+     * @default 1
+     */
+    concurrency?: number;
+    onPath?: (data: {
+        isDir: boolean;
+        entryPath: string;
+        entrySourcePath: string;
+        entryTargetPath: string;
+        stats: {
+            paths: number;
+            files: number;
+            dirs: number;
+        };
+    }) => Promise<boolean | void>;
+}): Promise<void>;

package/util/fs-util.js

@@ -3,17 +3,19 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.writePathLists = exports.writeGitIgnoreList = exports.forEachFile = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.tmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = void 0;
+exports.cpy = exports.writePathLists = exports.writeGitIgnoreList = exports.forEachFile = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.tmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
 const path_util_1 = require("./path-util");
+const async_1 = require("async");
 const crypto_1 = require("crypto");
+const fast_glob_1 = __importDefault(require("fast-glob"));
 const fs_1 = require("fs");
 const fs_2 = require("fs");
-const fs_extra_1 = require("fs-extra");
 const promises_1 = require("fs/promises");
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
 const path_2 = require("path");
+const readline_1 = require("readline");
 function isLocalDir(path) {
     return /^[\/\.]|([A-Z]:)/i.test(path);
 }
@@ -25,7 +27,7 @@ async function isDirEmpty(path) {
 exports.isDirEmpty = isDirEmpty;
 async function mkdirIfNotExists(path) {
     try {
-        await (0, fs_extra_1.mkdir)(path, {
+        await (0, promises_1.mkdir)(path, {
             recursive: true,
         });
     }
@@ -128,7 +130,7 @@ function tmpDir(prefix, id) {
 exports.tmpDir = tmpDir;
 async function mkTmpDir(prefix, id) {
     const path = tmpDir(prefix, id);
-    await (0, fs_extra_1.mkdir)(path, { recursive: true });
+    await (0, promises_1.mkdir)(path, { recursive: true });
     return path;
 }
 exports.mkTmpDir = mkTmpDir;
@@ -313,3 +315,70 @@ async function writePathLists(options) {
     };
 }
 exports.writePathLists = writePathLists;
+async function cpy(options) {
+    const stats = { paths: 0, files: 0, dirs: 0 };
+    const dirs = new Set();
+    const makeRecursiveDir = async (path) => {
+        if (!dirs.has(path)) {
+            stats.paths++;
+            stats.dirs++;
+            await (0, promises_1.mkdir)(path, {
+                recursive: true,
+            });
+            dirs.add(path);
+        }
+    };
+    const task = async (rawEntryPath, basePath) => {
+        const isDir = rawEntryPath.endsWith("/");
+        const entryPath = (0, path_1.normalize)(rawEntryPath);
+        const entrySourcePath = (0, path_1.resolve)((0, path_1.join)(basePath, rawEntryPath));
+        const entryTargetPath = (0, path_1.resolve)((0, path_1.join)(options.targetPath, rawEntryPath));
+        const onPathResult = await options?.onPath?.({
+            isDir,
+            entryPath,
+            entrySourcePath,
+            entryTargetPath,
+            stats,
+        });
+        if (onPathResult === false) {
+            return;
+        }
+        else if (isDir) {
+            await makeRecursiveDir(entryTargetPath);
+        }
+        else {
+            const dir = (0, path_1.dirname)(entryTargetPath);
+            await makeRecursiveDir(dir);
+            stats.files++;
+            await (0, promises_1.cp)(entrySourcePath, entryTargetPath);
+        }
+    };
+    const { input } = options;
+    if (input.type === "glob") {
+        const stream = await (0, fast_glob_1.default)(input.include || ["**"], {
+            cwd: input.sourcePath,
+            ignore: input.exclude,
+            dot: true,
+            onlyFiles: false,
+            markDirectories: true,
+        });
+        await (0, async_1.eachLimit)(stream, options.concurrency ?? 1, async (entryPath) => await task(entryPath, input.sourcePath));
+    }
+    else if (input.type === "stream") {
+        await (0, async_1.eachLimit)(input.value, options.concurrency ?? 1, async (entryPath) => await task(entryPath, input.basePath));
+    }
+    else if (input.type === "pathList") {
+        const stream = (0, readline_1.createInterface)({
+            input: (0, fs_1.createReadStream)(input.path),
+        });
+        await cpy({
+            ...options,
+            input: {
+                type: "stream",
+                value: stream,
+                basePath: input.basePath,
+            },
+        });
+    }
+}
+exports.cpy = cpy;

package/util/process-util.d.ts

@@ -23,11 +23,11 @@ export interface ExecSettingsInterface {
     onSpawn?: (p: ChildProcess) => void;
     stdout?: {
         save?: boolean;
-        onData?: (data: string) => Promise<void>;
+        onData?: (data: string) => void;
     };
     stderr?: {
         save?: boolean;
-        onData?: (data: string) => Promise<void>;
+        onData?: (data: string) => void;
         toExitCode?: boolean;
     };
     onExitCodeError?: (data: ExecResultType, error: Error) => Error | false;

package/util/process-util.js

@@ -68,10 +68,10 @@ async function exec(command, argv = [], options = null, settings = {}) {
             stderr: "",
             exitCode: 0,
         };
-        let finishListens = pipe ? 2 : 1;
+        let finishListeners = pipe?.stream instanceof fs_1.WriteStream ? 2 : 1;
         let streamError;
         const tryFinish = () => {
-            if (!--finishListens)
+            if (!--finishListeners)
                 finish();
         };
         const finish = () => {
@@ -127,7 +127,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
         if (log.stdout || settings.stdout) {
             if (!p.stdout)
                 throw new Error(`stdout is not defined`);
-            p.stdout.on("data", async (data) => {
+            p.stdout.on("data", (data) => {
                 if (log.stdout)
                     logExecStdout({
                         data: data.toString(),
@@ -137,13 +137,13 @@ async function exec(command, argv = [], options = null, settings = {}) {
                 if (settings.stdout?.save)
                     spawnData.stdout += data.toString();
                 if (settings.stdout?.onData)
-                    await settings.stdout.onData(data.toString());
+                    settings.stdout.onData(data.toString());
             });
         }
         if (log.stderr || settings.stderr) {
             if (!p.stderr)
                 throw new Error(`stderr is not defined`);
-            p.stderr.on("data", async (data) => {
+            p.stderr.on("data", (data) => {
                 if (log.stderr)
                     logExecStdout({
                         data: data.toString(),
@@ -153,7 +153,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
                 if (settings.stderr?.save || settings.stderr?.toExitCode)
                     spawnData.stderr += data.toString();
                 if (settings.stderr?.onData)
-                    await settings.stderr.onData(data.toString());
+                    settings.stderr.onData(data.toString());
             });
         }
         p.on("error", (error) => (spawnError = error)).on("close", (exitCode) => {

package/util/string-util.d.ts

@@ -13,5 +13,6 @@ export declare type UriType = {
     path?: string;
 };
 export declare function formatUri(input: UriType, hidePassword?: boolean): string;
+export declare function formatSeconds(seconds: number): string;
 export declare function makePathPatterns(values: string[] | undefined): string[] | undefined;
 export declare function formatDateTime(datetime: string): string;

package/util/string-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatDateTime = exports.makePathPatterns = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.lcfirst = exports.ucfirst = exports.serialize = void 0;
+exports.formatDateTime = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.lcfirst = exports.ucfirst = exports.serialize = void 0;
 const AppError_1 = require("../Error/AppError");
 function serialize(message, data) {
     if (data)
@@ -67,6 +67,26 @@ function formatUri(input, hidePassword) {
     return uri;
 }
 exports.formatUri = formatUri;
+function formatSeconds(seconds) {
+    let unit;
+    let value;
+    if (seconds > 60 * 60) {
+        value = seconds / 60 / 60;
+        unit = `hour`;
+    }
+    else if (seconds > 60) {
+        value = seconds / 60;
+        unit = `minute`;
+    }
+    else {
+        value = seconds;
+        unit = `second`;
+    }
+    if (value !== 1)
+        unit += `s`;
+    return `${value.toFixed(2)} ${unit}`;
+}
+exports.formatSeconds = formatSeconds;
 function makePathPatterns(values) {
     return values?.flatMap((v) => [v, `${v}/**`]);
 }

package/util/zip-util.d.ts

@@ -26,7 +26,7 @@ export interface ZipDataType {
     includeList?: string;
     excludeList?: string;
     verbose?: boolean;
-    onStream?: (data: ZipStreamDataType) => Promise<void>;
+    onStream?: (data: ZipStreamDataType) => void;
 }
 export interface UnzipDataType {
     command?: string;
@@ -34,7 +34,7 @@ export interface UnzipDataType {
     files?: (ZipDataFilterType | string)[];
     output: string;
     verbose?: boolean;
-    onStream?: (data: UnzipStreamDataType) => Promise<void>;
+    onStream?: (data: UnzipStreamDataType) => void;
 }
 export declare type UnzipStreamDataType = {
     type: "progress";

package/util/zip-util.js

@@ -30,7 +30,7 @@ function buildArguments(filters) {
     return args;
 }
 exports.buildArguments = buildArguments;
-async function parseZipStream(chunk, buffer, cb) {
+function parseZipStream(chunk, buffer, cb) {
     const lines = chunk.trim().split(/\r?\n/g);
     for (const line of lines) {
         let matches = null;
@@ -42,7 +42,7 @@ async function parseZipStream(chunk, buffer, cb) {
             if (path !== buffer.lastPath)
                 buffer.currentPaths++;
             buffer.lastPath = path;
-            await cb({
+            cb({
                 type: "progress",
                 data: { progress, path, files: buffer.currentPaths },
             });
@@ -50,7 +50,7 @@ async function parseZipStream(chunk, buffer, cb) {
         else if (line.startsWith("Add new data to archive:")) {
             const [, folders] = /(\d+) folders?/i.exec(line) || [, 0];
             const [, files] = /(\d+) files?/i.exec(line) || [, 0];
-            await cb({
+            cb({
                 type: "summary",
                 data: {
                     folders: Number(folders),
@@ -83,9 +83,9 @@ async function zip(data) {
             toExitCode: true,
         },
         stdout: {
-            onData: async (chunk) => {
-                parseZipStream(chunk, buffer, async (stream) => {
-                    await data.onStream?.(stream);
+            onData: (chunk) => {
+                parseZipStream(chunk, buffer, (stream) => {
+                    data.onStream?.(stream);
                     if (stream.type === "summary")
                         result = stream.data;
                 });
@@ -95,7 +95,7 @@ async function zip(data) {
     return result;
 }
 exports.zip = zip;
-async function parseUnzipStream(chunk, cb) {
+function parseUnzipStream(chunk, cb) {
     const lines = chunk.trim().split(/\r?\n/g);
     for (const line of lines) {
         let matches = null;
@@ -103,7 +103,7 @@ async function parseUnzipStream(chunk, cb) {
             const progress = Number(matches[1]);
             const files = Number(matches[2]);
             const path = line.slice(line.indexOf("-") + 1).trim();
-            await cb({
+            cb({
                 type: "progress",
                 data: { progress, path, files },
             });
@@ -124,7 +124,7 @@ async function unzip(data) {
         stderr: { toExitCode: true },
         stdout: {
             ...(data.onStream && {
-                onData: async (chunk) => {
+                onData: (chunk) => {
                     if (data.onStream)
                         parseUnzipStream(chunk, data.onStream);
                 },