@datatruck/cli 0.27.0 → 0.28.0

package/Action/BackupAction.js

@@ -1,286 +1,326 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BackupAction = void 0;
-const AppError_1 = require("../Error/AppError");
 const RepositoryFactory_1 = require("../Factory/RepositoryFactory");
 const TaskFactory_1 = require("../Factory/TaskFactory");
+const DataFormat_1 = require("../utils/DataFormat");
+const cli_1 = require("../utils/cli");
 const config_1 = require("../utils/datatruck/config");
+const date_1 = require("../utils/date");
 const fs_1 = require("../utils/fs");
+const list_1 = require("../utils/list");
+const progress_1 = require("../utils/progress");
+const steps_1 = require("../utils/steps");
+const temp_1 = require("../utils/temp");
+const assert_1 = require("assert");
+const chalk_1 = __importDefault(require("chalk"));
 const crypto_1 = require("crypto");
 class BackupAction {
     config;
     options;
-    taskErrors = {};
-    repoErrors = {};
+    pm;
     constructor(config, options = {}) {
         this.config = config;
         this.options = options;
+        this.pm = new progress_1.ProgressManager({
+            verbose: options.verbose,
+            tty: options.tty,
+            enabled: options.progress,
+            interval: options.progressInterval,
+        });
     }
-    async init(session) {
-        const snapshot = {
+    prepareSnapshot() {
+        return {
             id: (0, crypto_1.randomUUID)().replaceAll("-", ""),
             date: this.options.date ?? new Date().toISOString(),
         };
-        await session.initDrivers();
-        let packages = (0, config_1.filterPackages)(this.config, {
+    }
+    getPackages(snapshot) {
+        const packages = (0, config_1.filterPackages)(this.config, {
             packageNames: this.options.packageNames,
             packageTaskNames: this.options.packageTaskNames,
             repositoryNames: this.options.repositoryNames,
             repositoryTypes: this.options.repositoryTypes,
             sourceAction: "backup",
         });
-        packages = (0, config_1.resolvePackages)(packages, {
+        return (0, config_1.resolvePackages)(packages, {
             snapshotId: snapshot.id,
             snapshotDate: snapshot.date,
             action: "backup",
         });
-        for (const pkg of packages) {
-            const sessionId = await session.init({
-                snapshotId: snapshot.id,
-                packageName: pkg.name,
-                tags: this.options.tags?.join(",") ?? "",
-            });
-            if (pkg.task)
-                await session.initTask({
-                    sessionId: sessionId,
-                    taskName: pkg.task.name,
-                });
-            for (const repositoryName of pkg.repositoryNames ?? []) {
-                const repo = (0, config_1.findRepositoryOrFail)(this.config, repositoryName);
-                await session.initRepository({
-                    sessionId: sessionId,
-                    repositoryName: repositoryName,
-                    repositoryType: repo.type,
-                });
-            }
-        }
-        return [snapshot, packages];
     }
-    async task(session, pkg, task, snapshot, targetPath) {
-        const taskId = session.findTaskId({
-            packageName: pkg.name,
-            taskName: pkg.task.name,
-        });
-        await session.startTask({
-            id: taskId,
-        });
-        const key = `${pkg.name}`;
-        let error;
-        if (this.taskErrors[key]?.length) {
-            error = AppError_1.AppError.create("Previous task failed", this.taskErrors[key]);
-        }
-        else {
-            try {
-                await task.onBackup({
-                    package: pkg,
-                    options: this.options,
-                    snapshot,
-                    targetPath,
-                    onProgress: async (progress) => {
-                        await session.progressTask({
-                            id: taskId,
-                            progress,
-                        });
-                    },
-                });
-            }
-            catch (_) {
-                if (!this.taskErrors[key])
-                    this.taskErrors[key] = [];
-                this.taskErrors[key].push((error = _));
-            }
+    getRepositoryNames(repositoryNames) {
+        const items = [];
+        const exclude = new Set();
+        for (const name of repositoryNames) {
+            const repo = (0, config_1.findRepositoryOrFail)(this.config, name);
+            const mirrors = (repo.mirrorRepoNames || []).filter((mirror) => !exclude.has(mirror));
+            for (const mirror of mirrors)
+                exclude.add(mirror);
+            items.push({ name, mirrors });
         }
-        await session.endTask({
-            id: taskId,
-            error: error?.stack,
-        });
-        return {
-            error: error ? false : true,
-            tmpDirs: task.tmpDirs,
-        };
+        return items.filter((item) => !exclude.has(item.name));
     }
-    async backup(session, pkg, repo, snapshot, targetPath) {
-        const repositoryId = session.findRepositoryId({
-            packageName: pkg.name,
-            repositoryName: repo.name,
+    async backup(data) {
+        const repoConfig = (0, config_1.findRepositoryOrFail)(this.config, data.repositoryName);
+        const pkg = { ...data.pkg, path: data.snapshotPath ?? data.pkg.path };
+        (0, assert_1.ok)(pkg.path);
+        await (0, fs_1.ensureExistsDir)(pkg.path);
+        const repo = (0, RepositoryFactory_1.createRepo)(repoConfig);
+        if (this.config.minFreeDiskSpace)
+            await repo.ensureFreeDiskSpace(repoConfig.config, this.config.minFreeDiskSpace);
+        const packageConfig = pkg.repositoryConfigs?.find((config) => config.type === repoConfig.type &&
+            (!config.names || config.names.includes(repoConfig.name)))?.config;
+        await repo.backup({
+            options: this.options,
+            snapshot: data.snapshot,
+            package: pkg,
+            packageConfig,
+            onProgress: data.onProgress,
         });
-        await session.startRepository({
-            id: repositoryId,
-        });
-        let error;
-        let repoInstance;
-        if (this.taskErrors[pkg.name]?.length) {
-            error = AppError_1.AppError.create("Task failed", this.taskErrors[pkg.name]);
-        }
-        else {
-            try {
-                repoInstance = (0, RepositoryFactory_1.RepositoryFactory)(repo);
-                await repoInstance.onBackup({
-                    package: pkg,
-                    targetPath,
-                    packageConfig: pkg.repositoryConfigs?.find((config) => config.type === repo.type &&
-                        (!config.names || config.names.includes(repo.name)))?.config,
-                    options: this.options,
-                    snapshot: snapshot,
-                    onProgress: async (progress) => {
-                        await session.progressRepository({
-                            id: repositoryId,
-                            progress,
-                        });
-                    },
-                });
-            }
-            catch (_) {
-                if (!this.repoErrors[pkg.name])
-                    this.repoErrors[pkg.name] = [];
-                this.repoErrors[pkg.name].push((error = _));
-            }
-        }
-        await session.endRepository({
-            id: repositoryId,
-            error: error?.stack,
+    }
+    async copy(data) {
+        const repoConfig = (0, config_1.findRepositoryOrFail)(this.config, data.repositoryName);
+        const mirrorRepoConfig = (0, config_1.findRepositoryOrFail)(this.config, data.mirrorRepositoryName);
+        const repo = (0, RepositoryFactory_1.createRepo)(repoConfig);
+        const mirrorRepo = (0, RepositoryFactory_1.createRepo)(mirrorRepoConfig);
+        if (this.config.minFreeDiskSpace)
+            await mirrorRepo.ensureFreeDiskSpace(mirrorRepoConfig.config, this.config.minFreeDiskSpace);
+        await repo.copy({
+            options: this.options,
+            package: data.pkg,
+            snapshot: data.snapshot,
+            mirrorRepositoryConfig: mirrorRepoConfig.config,
+            onProgress: data.onProgress,
         });
-        return {
-            error: error ? false : true,
-            tmpDirs: repoInstance?.tmpDirs ?? [],
+    }
+    dataFormat(result, options = {}) {
+        const renderTitle = (item, color) => {
+            let title = item.key.slice(0, 1).toUpperCase() + item.key.slice(1);
+            return item.key === "backup" && color ? chalk_1.default.cyan(title) : title;
         };
+        const renderData = (item, color) => {
+            const g = (v) => (color ? `${chalk_1.default.gray(`(${v})`)}` : `(${v})`);
+            return item.key === "snapshot"
+                ? item.data.id
+                : item.key === "task"
+                    ? `${item.data.packageName} ${g(item.data.taskName)}`
+                    : item.key === "backup"
+                        ? `${item.data.packageName} ${g(item.data.repositoryName)}`
+                        : item.key === "copy"
+                            ? `${item.data.packageName} ${g(item.data.mirrorRepositoryName)}`
+                            : item.key === "summary"
+                                ? `Errors: ${item.data.errors}`
+                                : item.key === "report"
+                                    ? item.data.type
+                                    : "";
+        };
+        return new DataFormat_1.DataFormat({
+            streams: options.streams,
+            json: result,
+            list: () => result
+                .filter((item) => item.key !== "cleanup")
+                .map((item) => {
+                const icon = (0, cli_1.resultColumn)(item.error, false);
+                const title = renderTitle(item);
+                const data = renderData(item, false);
+                return `${icon} ${title}: ${data}`;
+            }),
+            table: {
+                headers: [
+                    { value: "", width: 3 },
+                    { value: "Title", width: 15 },
+                    { value: "Data", align: "left" },
+                    { value: "Duration", width: 10 },
+                    { value: "Error", width: 50 },
+                ],
+                rows: () => result
+                    .filter((item) => item.key !== "cleanup")
+                    .map((item) => [
+                    (0, cli_1.resultColumn)(item.error),
+                    renderTitle(item, true),
+                    renderData(item, true),
+                    (0, date_1.duration)(item.elapsed),
+                    (0, cli_1.errorColumn)(item.error, options.verbose),
+                ]),
+            },
+        });
     }
-    async copyBackup(session, pkg, repo, mirrorRepo, snapshot) {
-        const repositoryId = session.findRepositoryId({
-            packageName: pkg.name,
-            repositoryName: mirrorRepo.name,
+    async exec() {
+        const { options } = this;
+        const pm = new progress_1.ProgressManager({
+            verbose: options.verbose,
+            tty: options.tty,
+            enabled: options.progress,
+            interval: options.progressInterval,
         });
-        await session.startRepository({
-            id: repositoryId,
+        const l = new list_1.Listr3({
+            streams: this.options.streams,
+            progressManager: pm,
         });
-        let error;
-        let repoInstance;
-        if (this.taskErrors[pkg.name]?.length) {
-            error = AppError_1.AppError.create("Task failed", this.taskErrors[pkg.name]);
-        }
-        else {
-            try {
-                repoInstance = (0, RepositoryFactory_1.RepositoryFactory)(repo);
-                await repoInstance.onCopyBackup({
-                    options: this.options,
-                    package: pkg,
-                    snapshot,
-                    mirrorRepositoryConfig: mirrorRepo.config,
-                    onProgress: async (progress) => {
-                        await session.progressRepository({
-                            id: repositoryId,
-                            progress,
+        return l
+            .add(l.$tasks({
+            key: "snapshot",
+            data: { id: "" },
+            exitOnError: false,
+            title: {
+                initial: "Prepare snapshots",
+                started: "Preparing snapshots",
+                failed: "Snapshots prepare failed",
+            },
+            run: async (task, data) => {
+                const { minFreeDiskSpace } = this.config;
+                if (minFreeDiskSpace)
+                    await (0, temp_1.ensureFreeDiskTempSpace)(minFreeDiskSpace);
+                const snapshot = this.prepareSnapshot();
+                const packages = this.getPackages(snapshot);
+                const snapshotId = (data.id = snapshot.id.slice(0, 8));
+                task.title = `Snapshots prepared: ${snapshotId} (${packages.length} packages)`;
+                return packages.flatMap((pkg) => {
+                    let taskResult = {};
+                    const gc = new temp_1.GargabeCollector();
+                    const repositories = this.getRepositoryNames(pkg.repositoryNames ?? []);
+                    const mirrorRepositories = repositories
+                        .filter((r) => r.mirrors.length)
+                        .flatMap(({ name, mirrors }) => mirrors.map((mirror) => ({ name, mirror })));
+                    return l.$tasks(!!pkg.task &&
+                        l.$task({
+                            key: "task",
+                            keyIndex: pkg.name,
+                            data: { taskName: pkg.task.name, packageName: pkg.name },
+                            title: {
+                                initial: `Execute task: ${pkg.task?.name}`,
+                                started: `Executing task: ${pkg.task?.name}`,
+                                failed: `Task execute failed: ${pkg.task?.name}`,
+                                completed: `Task executed: ${pkg.task?.name}`,
+                            },
+                            exitOnError: false,
+                            runWrapper: gc.cleanupIfFail.bind(gc),
+                            run: async (task) => {
+                                taskResult = await (0, TaskFactory_1.createTask)(pkg.task).backup({
+                                    options,
+                                    package: pkg,
+                                    snapshot,
+                                    onProgress: (p) => pm.update(p, (t) => (task.output = t)),
+                                });
+                            },
+                        }), ...repositories.map(({ name: repositoryName }) => l.$task({
+                        key: "backup",
+                        keyIndex: [pkg.name, repositoryName],
+                        data: {
+                            packageName: pkg.name,
+                            repositoryName: repositoryName,
+                        },
+                        title: {
+                            initial: `Create backup: ${repositoryName}`,
+                            started: `Creating backup: ${repositoryName}`,
+                            completed: `Backup created: ${repositoryName}`,
+                            failed: `Backup create failed: ${repositoryName}`,
+                        },
+                        exitOnError: false,
+                        runWrapper: gc.cleanupOnFinish.bind(gc),
+                        run: async (task) => {
+                            const taskSummary = pkg.task
+                                ? l.result("task", pkg.name)
+                                : undefined;
+                            if (taskSummary?.error)
+                                throw new Error(`Task failed`);
+                            await this.backup({
+                                pkg,
+                                repositoryName,
+                                snapshot,
+                                snapshotPath: taskResult?.snapshotPath,
+                                onProgress: (p) => this.pm.update(p, (t) => (task.output = t)),
+                            });
+                        },
+                    })), l.$task({
+                        key: "cleanup",
+                        keyIndex: pkg.name,
+                        data: {},
+                        title: {
+                            initial: "Clean task files",
+                            started: "Cleaning task files",
+                            completed: "Task files cleaned",
+                            failed: "Task files clean failed",
+                        },
+                        exitOnError: false,
+                        enabled: gc.pending,
+                        run: () => gc.cleanup(),
+                    }), ...mirrorRepositories.map(({ name, mirror }) => l.$task({
+                        key: "copy",
+                        keyIndex: [pkg.name, mirror],
+                        data: {
+                            packageName: pkg.name,
+                            repositoryName: name,
+                            mirrorRepositoryName: mirror,
+                        },
+                        title: {
+                            initial: `Copy snapshot: ${mirror}`,
+                            started: `Copying snapshot: ${mirror}`,
+                            completed: `Snapshot copied: ${mirror}`,
+                            failed: `Snapshot copy failed: ${mirror}`,
+                        },
+                        exitOnError: false,
+                        runWrapper: gc.cleanup.bind(gc),
+                        run: async (task) => {
+                            const backupSummary = l.result("backup", [
+                                pkg.name,
+                                name,
+                            ]);
+                            if (backupSummary.error)
+                                throw new Error(`Backup failed`);
+                            await this.copy({
+                                repositoryName: name,
+                                mirrorRepositoryName: mirror,
+                                pkg,
+                                snapshot,
+                                onProgress: (p) => pm.update(p, (t) => (task.output = t)),
+                            });
+                        },
+                    })), ...(this.config.reports || []).map((report, index) => {
+                        const reportIndex = index + 1;
+                        return l.$task({
+                            title: {
+                                initial: `Send report ${reportIndex}`,
+                                started: `Sending report ${reportIndex}`,
+                                completed: `Report sent: ${reportIndex}`,
+                                failed: `Report send failed: ${reportIndex}`,
+                            },
+                            key: "report",
+                            keyIndex: index,
+                            data: { type: report.run.type },
+                            exitOnError: false,
+                            run: async (task) => {
+                                const result = l
+                                    .getResult()
+                                    .filter((r) => r.key !== "report");
+                                const success = result.every((r) => !r.error);
+                                const enabled = !report.when ||
+                                    (report.when === "success" && success) ||
+                                    (report.when === "error" && !success);
+                                if (!enabled)
+                                    return task.skip(`Report send skipped: ${reportIndex}`);
+                                const text = this.dataFormat(result).format(report.format ?? "list");
+                                await (0, steps_1.runSteps)(report.run, {
+                                    verbose: this.options.verbose,
+                                    process: { vars: { DTT_REPORT: text } },
+                                    telegram: { vars: { TEXT: text } },
+                                    node: {
+                                        vars: {
+                                            dtt: { report: text, result },
+                                        },
+                                    },
+                                });
+                            },
                         });
-                    },
-                });
-            }
-            catch (_) {
-                if (!this.repoErrors[pkg.name])
-                    this.repoErrors[pkg.name] = [];
-                this.repoErrors[pkg.name].push((error = _));
-            }
-        }
-        await session.endRepository({
-            id: repositoryId,
-            error: error?.stack,
-        });
-        return {
-            error: error ? false : true,
-            tmpDirs: repoInstance?.tmpDirs ?? [],
-        };
-    }
-    getError(pkg) {
-        const taskErrors = this.taskErrors[pkg.name] || [];
-        const repoErrors = this.repoErrors[pkg.name] || [];
-        const errors = [...taskErrors, ...repoErrors];
-        if (!errors.length)
-            return;
-        return AppError_1.AppError.create(taskErrors.length && repoErrors.length
-            ? "Task and repository failed"
-            : taskErrors.length && !repoErrors.length
-                ? "Task failed"
-                : "Repository failed", errors);
-    }
-    splitRepositories(repositoryNames) {
-        const mirrorRepoMap = {};
-        const allMirrorRepoNames = [];
-        const repoNames = repositoryNames ?? [];
-        for (const repoName of repoNames) {
-            const repo = (0, config_1.findRepositoryOrFail)(this.config, repoName);
-            if (repo.mirrorRepoNames)
-                mirrorRepoMap[repoName] = repo.mirrorRepoNames.filter((mirrorRepoName) => {
-                    allMirrorRepoNames.push(mirrorRepoName);
-                    return repoNames.includes(mirrorRepoName);
+                    }));
                 });
-        }
-        return {
-            repoNames: repoNames.filter((v) => !allMirrorRepoNames.includes(v)),
-            mirrors: repoNames.flatMap((sourceName) => {
-                const mirrorNames = mirrorRepoMap[sourceName] || [];
-                return mirrorNames.map((name) => ({
-                    sourceName,
-                    name,
-                }));
-            }),
-        };
-    }
-    async exec(session) {
-        const [snapshot, packages] = await this.init(session);
-        const errors = [];
-        for (const pkg of packages) {
-            const id = session.findId({
-                packageName: pkg.name,
-            });
-            await session.start({
-                id,
-            });
-            let targetPath;
-            let taskTmpDirs = [];
-            if (pkg.task) {
-                const taskInstance = (0, TaskFactory_1.TaskFactory)(pkg.task);
-                const result = await taskInstance.onBeforeBackup({
-                    options: this.options,
-                    package: pkg,
-                    snapshot,
-                });
-                const taskResult = await this.task(session, pkg, taskInstance, snapshot, (targetPath = result?.targetPath));
-                taskTmpDirs.push(...taskResult.tmpDirs);
-            }
-            const { repoNames, mirrors } = this.splitRepositories(pkg.repositoryNames ?? []);
-            for (const repoName of repoNames) {
-                const repo = (0, config_1.findRepositoryOrFail)(this.config, repoName);
-                const { tmpDirs } = await this.backup(session, pkg, repo, snapshot, targetPath);
-                if (!this.options.verbose)
-                    await (0, fs_1.rmTmpDir)(tmpDirs);
-            }
-            if (!this.options.verbose) {
-                await (0, fs_1.rmTmpDir)(taskTmpDirs);
-                if (pkg.path && (0, fs_1.isTmpDir)(pkg.path)) {
-                    await (0, fs_1.rmTmpDir)(pkg.path);
-                }
-            }
-            for (const mirror of mirrors) {
-                const repo = (0, config_1.findRepositoryOrFail)(this.config, mirror.sourceName);
-                const mirrorRepo = (0, config_1.findRepositoryOrFail)(this.config, mirror.name);
-                const { tmpDirs } = await this.copyBackup(session, pkg, repo, mirrorRepo, snapshot);
-                if (!this.options.verbose)
-                    await (0, fs_1.rmTmpDir)(tmpDirs);
-            }
-            const error = this.getError(pkg);
-            if (error)
-                errors.push(error);
-            await session.end({
-                id: id,
-                error: error?.message,
-            });
-        }
-        await session.endDrivers({
-            snapshotId: snapshot.id.slice(0, 8),
-        });
-        return {
-            total: packages.length,
-            errors,
-        };
+            },
+        }))
+            .exec();
     }
 }
 exports.BackupAction = BackupAction;

package/Action/CleanCacheAction.d.ts

@@ -1,9 +1,13 @@
 import { IfRequireKeys } from "../utils/ts";
-export type CleanCacheActionOptionsType = {
+export type CleanCacheActionOptions = {
     verbose?: boolean;
 };
 export declare class CleanCacheAction<TRequired extends boolean = true> {
-    readonly options: IfRequireKeys<TRequired, CleanCacheActionOptionsType>;
-    constructor(options: IfRequireKeys<TRequired, CleanCacheActionOptionsType>);
-    exec(): Promise<void>;
+    readonly options: IfRequireKeys<TRequired, CleanCacheActionOptions>;
+    constructor(options: IfRequireKeys<TRequired, CleanCacheActionOptions>);
+    exec(): Promise<{
+        errors: never[];
+        path: string;
+        freedSize: number;
+    }>;
 }

package/Action/CleanCacheAction.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CleanCacheAction = void 0;
 const fs_1 = require("../utils/fs");
+const temp_1 = require("../utils/temp");
 const promises_1 = require("fs/promises");
 class CleanCacheAction {
     options;
@@ -9,11 +10,13 @@ class CleanCacheAction {
         this.options = options;
     }
     async exec() {
-        const path = (0, fs_1.parentTmpDir)();
-        if (await (0, fs_1.existsDir)(path))
-            await (0, promises_1.rm)(path, {
-                recursive: true,
-            });
+        const path = (0, temp_1.parentTmpDir)();
+        let freedSize = 0;
+        if (await (0, fs_1.existsDir)(path)) {
+            freedSize = await (0, fs_1.fastFolderSizeAsync)(path);
+            await (0, promises_1.rm)(path, { recursive: true });
+        }
+        return { errors: [], path, freedSize };
     }
 }
 exports.CleanCacheAction = CleanCacheAction;

package/Action/ConfigAction.d.ts

@@ -1,17 +1,24 @@
-import { GlobalOptionsType } from "../Command/CommandAbstract";
+import { GlobalOptions } from "../Command/CommandAbstract";
 import type { ConfigType } from "../Config/Config";
 import { IfRequireKeys } from "../utils/ts";
-export type ConfigActionOptionsType = {
+export type ConfigActionOptions = {
     path: string;
     verbose?: boolean;
 };
 export declare class ConfigAction<TRequired extends boolean = true> {
-    readonly options: IfRequireKeys<TRequired, ConfigActionOptionsType>;
-    constructor(options: IfRequireKeys<TRequired, ConfigActionOptionsType>);
+    readonly options: IfRequireKeys<TRequired, ConfigActionOptions>;
+    constructor(options: IfRequireKeys<TRequired, ConfigActionOptions>);
     static validate(config: ConfigType): void;
     static check(config: ConfigType): void;
     static normalize(config: ConfigType): ConfigType;
-    static fromGlobalOptions(globalOptions: GlobalOptionsType<true>): Promise<ConfigType>;
+    static fromGlobalOptionsWithPath(globalOptions: GlobalOptions<true>): Promise<{
+        path: string;
+        data: ConfigType;
+    } | {
+        path: null;
+        data: ConfigType;
+    }>;
+    static fromGlobalOptions(globalOptions: GlobalOptions<true>): Promise<ConfigType>;
     exec(): Promise<{
         path: string;
         data: ConfigType;