@datatruck/cli 0.27.0 → 0.28.0

package/Action/ConfigAction.js

@@ -10,7 +10,6 @@ const config_1 = require("../utils/datatruck/config");
 const fs_1 = require("../utils/fs");
 const ajv_1 = __importDefault(require("ajv"));
 const assert_1 = require("assert");
-const path_1 = require("path");
 class ConfigAction {
     options;
     constructor(options) {
@@ -55,12 +54,6 @@ class ConfigAction {
         config = Object.assign({}, config);
         config.packages = config.packages.map((pkg) => {
             pkg = Object.assign({}, pkg);
-            if (!pkg.restorePath)
-                pkg.restorePath = pkg.path
-                    ? pkg.path
-                    : (0, path_1.normalize)(`{temp}/{snapshotId}-{action}/{packageName}`);
-            if (!pkg.path)
-                pkg.path = (0, path_1.normalize)(`{temp}/{snapshotId}-{action}/{packageName}`);
             pkg.repositoryNames =
                 pkg.repositoryNames ?? config.repositories.map((repo) => repo.name);
             (0, assert_1.ok)(Array.isArray(pkg.repositoryNames));
@@ -70,18 +63,21 @@ class ConfigAction {
         });
         return config;
     }
+    static async fromGlobalOptionsWithPath(globalOptions) {
+        if (typeof globalOptions.config !== "string")
+            return {
+                path: null,
+                data: globalOptions.config,
+            };
+        const configAction = new ConfigAction({
+            path: globalOptions.config,
+            verbose: !!globalOptions.verbose && globalOptions.verbose > 0,
+        });
+        return await configAction.exec();
+    }
     static async fromGlobalOptions(globalOptions) {
-        if (typeof globalOptions.config === "string") {
-            const configAction = new ConfigAction({
-                path: globalOptions.config,
-                verbose: !!globalOptions.verbose && globalOptions.verbose > 0,
-            });
-            const result = await configAction.exec();
-            return result.data;
-        }
-        else {
-            return globalOptions.config;
-        }
+        const config = await ConfigAction.fromGlobalOptionsWithPath(globalOptions);
+        return config.data;
     }
     async exec() {
         const path = await (0, fs_1.findFile)(this.options.path, "datatruck.config", fs_1.parseFileExtensions, "Config path not found");

package/Action/CopyAction.d.ts

@@ -0,0 +1,49 @@
+import type { ConfigType } from "../Config/Config";
+import { Snapshot } from "../Repository/RepositoryAbstract";
+import { DataFormat } from "../utils/DataFormat";
+import { Listr3TaskResultEnd } from "../utils/list";
+import { Streams } from "../utils/stream";
+import { IfRequireKeys } from "../utils/ts";
+export type CopyActionOptionsType = {
+    ids: string[];
+    repositoryName: string;
+    packageNames?: string[];
+    packageTaskNames?: string[];
+    repositoryNames2?: string[];
+    verbose?: boolean;
+    tty?: "auto" | boolean;
+    progress?: "auto" | "interval" | boolean;
+    progressInterval?: number;
+};
+export type CopyActionResult = {
+    errors: Error[];
+};
+export type Context = {
+    snapshots: {
+        snapshots: Snapshot[];
+    };
+    copy: {
+        snapshotId: string;
+        packageName: string;
+        repositoryName: string;
+        mirrorRepositoryName: string;
+    };
+};
+export declare class CopyAction<TRequired extends boolean = true> {
+    readonly config: ConfigType;
+    readonly options: IfRequireKeys<TRequired, CopyActionOptionsType>;
+    constructor(config: ConfigType, options: IfRequireKeys<TRequired, CopyActionOptionsType>);
+    dataFormat(result: Listr3TaskResultEnd<Context>[], options?: {
+        streams?: Streams;
+        verbose?: number;
+    }): DataFormat;
+    exec(): Promise<({
+        key: "summary";
+        keyIndex?: string | undefined;
+        data: {
+            errors: number;
+        };
+        elapsed: number;
+        error?: Error | undefined;
+    } | import("../utils/list").Listr3TaskResult<Context>)[]>;
+}

package/Action/CopyAction.js

@@ -0,0 +1,144 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CopyAction = void 0;
+const RepositoryFactory_1 = require("../Factory/RepositoryFactory");
+const DataFormat_1 = require("../utils/DataFormat");
+const cli_1 = require("../utils/cli");
+const config_1 = require("../utils/datatruck/config");
+const date_1 = require("../utils/date");
+const list_1 = require("../utils/list");
+const progress_1 = require("../utils/progress");
+const temp_1 = require("../utils/temp");
+const chalk_1 = __importDefault(require("chalk"));
+class CopyAction {
+    config;
+    options;
+    constructor(config, options) {
+        this.config = config;
+        this.options = options;
+    }
+    dataFormat(result, options = {}) {
+        const renderTitle = (item, color) => {
+            let title = item.key.slice(0, 1).toUpperCase() + item.key.slice(1);
+            return item.key === "copy" && color ? chalk_1.default.cyan(title) : title;
+        };
+        const renderData = (item, color) => {
+            const g = (v) => (color ? `${chalk_1.default.gray(`(${v})`)}` : `(${v})`);
+            return item.key === "snapshots"
+                ? item.data.snapshots.length
+                : item.key === "copy"
+                    ? `${item.data.packageName} ${g(item.data.mirrorRepositoryName)}`
+                    : "";
+        };
+        return new DataFormat_1.DataFormat({
+            streams: options.streams,
+            json: result,
+            table: {
+                headers: [
+                    { value: "", width: 3 },
+                    { value: "Title", width: 15 },
+                    { value: "Data" },
+                    { value: "Duration", width: 10 },
+                    { value: "Error", width: 50 },
+                ],
+                rows: () => result.map((item) => [
+                    (0, cli_1.resultColumn)(item.error),
+                    renderTitle(item, true),
+                    renderData(item, true),
+                    (0, date_1.duration)(item.elapsed),
+                    (0, cli_1.errorColumn)(item.error, options.verbose),
+                ]),
+            },
+        });
+    }
+    async exec() {
+        const { options } = this;
+        const pm = new progress_1.ProgressManager({
+            verbose: options.verbose,
+            tty: options.tty,
+            enabled: options.progress,
+            interval: options.progressInterval,
+        });
+        const l = new list_1.Listr3({ progressManager: pm });
+        return l
+            .add(l.$task({
+            key: "snapshots",
+            data: { snapshots: [] },
+            title: {
+                initial: "Fetch snapshots",
+                started: "Fetching snapshots",
+                completed: "Snapshots fetched",
+                failed: "Snapshot fetch failed",
+            },
+            run: async (task, data) => {
+                if (this.config.minFreeDiskSpace)
+                    await (0, temp_1.ensureFreeDiskTempSpace)(this.config.minFreeDiskSpace);
+                const sourceRepoConfig = (0, config_1.findRepositoryOrFail)(this.config, this.options.repositoryName);
+                const repo = (0, RepositoryFactory_1.createRepo)(sourceRepoConfig);
+                const snapshots = await repo.fetchSnapshots({
+                    options: {
+                        ids: this.options.ids,
+                        packageNames: this.options.packageNames,
+                        packageTaskNames: this.options.packageTaskNames,
+                    },
+                });
+                data.snapshots = snapshots;
+                task.title = `Snapshots fetched: ${snapshots.length}`;
+                if (!snapshots.length)
+                    throw new Error("No snapshots found");
+                const repositoryNames2 = this.options.repositoryNames2 ||
+                    this.config.repositories
+                        .filter((r) => r.name !== sourceRepoConfig.name &&
+                        r.type === sourceRepoConfig.type &&
+                        (0, config_1.filterRepository)(r, "backup"))
+                        .map((r) => r.name);
+                if (!repositoryNames2.length)
+                    throw new Error("No mirror snapshots found");
+                return snapshots.flatMap((snapshot) => repositoryNames2.map((repo2) => l.$task({
+                    key: "copy",
+                    keyIndex: [snapshot.packageName, repo2],
+                    data: {
+                        snapshotId: snapshot.id,
+                        packageName: snapshot.packageName,
+                        repositoryName: sourceRepoConfig.name,
+                        mirrorRepositoryName: repo2,
+                    },
+                    title: {
+                        initial: `Copy snapshot: ${repo2}`,
+                        started: `Copying snapshot: ${repo2}`,
+                        completed: `Snapshot copied: ${repo2}`,
+                        failed: `Snapshot copy failed: ${repo2}`,
+                    },
+                    exitOnError: false,
+                    run: async (task) => {
+                        const mirrorConfig = (0, config_1.findRepositoryOrFail)(this.config, repo2);
+                        const mirrorRepo = (0, RepositoryFactory_1.createRepo)(mirrorConfig);
+                        (0, config_1.ensureSameRepositoryType)(sourceRepoConfig, mirrorConfig);
+                        const currentCopies = await mirrorRepo.fetchSnapshots({
+                            options: {
+                                ids: [snapshot.id],
+                                packageNames: [snapshot.packageName],
+                            },
+                        });
+                        if (currentCopies.length)
+                            return task.skip(`Already exists at ${mirrorConfig.name}`);
+                        if (this.config.minFreeDiskSpace)
+                            await mirrorRepo.ensureFreeDiskSpace(mirrorConfig.config, this.config.minFreeDiskSpace);
+                        await repo.copy({
+                            mirrorRepositoryConfig: mirrorConfig.config,
+                            options: { verbose: this.options.verbose },
+                            package: { name: snapshot.packageName },
+                            snapshot,
+                            onProgress: (p) => pm.update(p, (d) => (task.output = d)),
+                        });
+                    },
+                })));
+            },
+        }))
+            .exec();
+    }
+}
+exports.CopyAction = CopyAction;

package/Action/InitAction.d.ts

@@ -1,14 +1,14 @@
 import type { ConfigType } from "../Config/Config";
 import { IfRequireKeys } from "../utils/ts";
-export type InitActionOptionsType = {
+export type InitActionOptions = {
     repositoryNames?: string[];
     repositoryTypes?: string[];
     verbose?: boolean;
 };
 export declare class InitAction<TRequired extends boolean = true> {
     readonly config: ConfigType;
-    readonly options: IfRequireKeys<TRequired, InitActionOptionsType>;
-    constructor(config: ConfigType, options: IfRequireKeys<TRequired, InitActionOptionsType>);
+    readonly options: IfRequireKeys<TRequired, InitActionOptions>;
+    constructor(config: ConfigType, options: IfRequireKeys<TRequired, InitActionOptions>);
     exec(): Promise<{
         repositoryName: string;
         repositoryType: string;

package/Action/InitAction.js

@@ -12,19 +12,19 @@ class InitAction {
     }
     async exec() {
         const result = [];
-        for (const repo of this.config.repositories) {
-            if (!(0, config_1.filterRepository)(repo, "init"))
+        for (const repoConfig of this.config.repositories) {
+            if (!(0, config_1.filterRepository)(repoConfig, "init"))
                 continue;
             if (this.options.repositoryNames &&
-                !this.options.repositoryNames.includes(repo.name))
+                !this.options.repositoryNames.includes(repoConfig.name))
                 continue;
             if (this.options.repositoryTypes &&
-                !this.options.repositoryTypes.includes(repo.type))
+                !this.options.repositoryTypes.includes(repoConfig.type))
                 continue;
-            const repoInstance = (0, RepositoryFactory_1.RepositoryFactory)(repo);
+            const repo = (0, RepositoryFactory_1.createRepo)(repoConfig);
             let initError = null;
             try {
-                await repoInstance.onInit({
+                await repo.init({
                     options: this.options,
                 });
             }
@@ -32,9 +32,9 @@ class InitAction {
                 initError = error;
             }
             result.push({
-                repositoryName: repo.name,
-                repositoryType: repo.type,
-                repositorySource: repoInstance.onGetSource(),
+                repositoryName: repoConfig.name,
+                repositoryType: repoConfig.type,
+                repositorySource: repo.getSource(),
                 error: initError,
             });
         }

package/Action/PruneAction.d.ts

@@ -1,8 +1,8 @@
 import type { ConfigType } from "../Config/Config";
 import { RepositoryConfigType } from "../Config/RepositoryConfig";
 import { IfRequireKeys } from "../utils/ts";
-import { SnapshotExtendedType, SnapshotsActionOptionsType } from "./SnapshotsAction";
-export type PruneActionsOptionsType = {
+import { ExtendedSnapshot, SnapshotsActionOptions } from "./SnapshotsAction";
+export type PruneActionsOptions = {
     ids?: string[];
     packageNames?: string[];
     repositoryNames?: string[];
@@ -16,22 +16,22 @@ export type PruneActionsOptionsType = {
     keepMonthly?: number;
     keepYearly?: number;
     verbose?: boolean;
-    groupBy?: SnapshotsActionOptionsType["groupBy"];
+    groupBy?: SnapshotsActionOptions["groupBy"];
     dryRun?: boolean;
     longId?: boolean;
     returnsAll?: boolean;
 };
-export type PruneResultType = {
+export type PruneResult = {
     total: number;
     prune: number;
-    snapshots: (SnapshotExtendedType & {
+    snapshots: (ExtendedSnapshot & {
         exclusionReasons: string[];
     })[];
 };
 export declare class PruneAction<TRequired extends boolean = true> {
     readonly config: ConfigType;
-    readonly options: IfRequireKeys<TRequired, PruneActionsOptionsType>;
-    constructor(config: ConfigType, options: IfRequireKeys<TRequired, PruneActionsOptionsType>);
-    confirm(snapshots: PruneResultType["snapshots"]): Promise<void>;
-    exec(): Promise<PruneResultType>;
+    readonly options: IfRequireKeys<TRequired, PruneActionsOptions>;
+    constructor(config: ConfigType, options: IfRequireKeys<TRequired, PruneActionsOptions>);
+    confirm(snapshots: PruneResult["snapshots"]): Promise<void>;
+    exec(): Promise<PruneResult>;
 }

package/Action/PruneAction.js

@@ -16,8 +16,8 @@ class PruneAction {
         const repository = (0, object_1.groupBy)(this.config.repositories, "name", true);
         for (const snapshot of snapshots) {
             if (!snapshot.exclusionReasons?.length) {
-                const repoInstance = (0, RepositoryFactory_1.RepositoryFactory)(repository[snapshot.repositoryName]);
-                await repoInstance.onPrune({
+                const repo = (0, RepositoryFactory_1.createRepo)(repository[snapshot.repositoryName]);
+                await repo.prune({
                     snapshot: snapshot,
                     options: { verbose: this.options.verbose },
                 });
@@ -25,11 +25,22 @@ class PruneAction {
         }
     }
     async exec() {
-        const snapshotsAction = new SnapshotsAction_1.SnapshotsAction(this.config, this.options);
+        const snapshotsAction = new SnapshotsAction_1.SnapshotsAction(this.config, {
+            groupBy: this.options.groupBy,
+            ids: this.options.ids,
+            packageNames: this.options.packageNames,
+            repositoryNames: this.options.repositoryNames,
+            repositoryTypes: this.options.repositoryTypes,
+            tags: this.options.tags,
+            verbose: this.options.verbose,
+        });
         const snapshots = await snapshotsAction.exec("prune");
         const snapshotsDeleted = [];
         const reasons = {};
-        const inputFilter = {
+        const idFilter = {
+            id: this.options.ids,
+        };
+        const keepFilter = {
             last: this.options.keepLast,
             lastMinutely: this.options.keepMinutely,
             lastHourly: this.options.keepHourly,
@@ -38,25 +49,30 @@ class PruneAction {
             lastWeekly: this.options.keepWeekly,
             lastYearly: this.options.keepYearly,
         };
-        const hasInputFilter = Object.values(inputFilter).some(Number);
-        const usePrunePolicyConfig = !hasInputFilter && this.options.groupBy?.includes("packageName");
-        const keepSnapshots = (0, snapshot_1.groupAndFilter)(snapshots, this.options.groupBy, usePrunePolicyConfig
-            ? (groupedSnapshots) => {
-                const [firstSnapshot] = groupedSnapshots;
-                const packageName = firstSnapshot.packageName;
-                const config = this.config.packages.find((pkg) => pkg.name === packageName);
-                const prunePolicy = config?.prunePolicy ?? {};
-                return {
-                    last: prunePolicy.keepLast,
-                    lastMinutely: prunePolicy.keepMinutely,
-                    lastHourly: prunePolicy.keepHourly,
-                    lastDaily: prunePolicy.keepDaily,
-                    lastMonthly: prunePolicy.keepMonthly,
-                    lastWeekly: prunePolicy.keepWeekly,
-                    lastYearly: prunePolicy.keepYearly,
-                };
-            }
-            : inputFilter, reasons);
+        const hasIdFilter = Object.values(idFilter).some((v) => typeof v !== "undefined");
+        const hasKeepFilter = Object.values(keepFilter).some((v) => typeof v !== "undefined");
+        if (hasIdFilter && hasKeepFilter)
+            throw new Error(`Snapshot id filter can not be used with 'keep' filters`);
+        const usePrunePolicyConfig = !hasKeepFilter && this.options.groupBy?.includes("packageName");
+        const keepSnapshots = hasKeepFilter
+            ? (0, snapshot_1.groupAndFilter)(snapshots, this.options.groupBy, usePrunePolicyConfig
+                ? (groupedSnapshots) => {
+                    const [firstSnapshot] = groupedSnapshots;
+                    const packageName = firstSnapshot.packageName;
+                    const config = this.config.packages.find((pkg) => pkg.name === packageName);
+                    const prunePolicy = config?.prunePolicy ?? {};
+                    return {
+                        last: prunePolicy.keepLast,
+                        lastMinutely: prunePolicy.keepMinutely,
+                        lastHourly: prunePolicy.keepHourly,
+                        lastDaily: prunePolicy.keepDaily,
+                        lastMonthly: prunePolicy.keepMonthly,
+                        lastWeekly: prunePolicy.keepWeekly,
+                        lastYearly: prunePolicy.keepYearly,
+                    };
+                }
+                : keepFilter, reasons)
+            : [];
         const result = {
             total: snapshots.length,
             prune: 0,

package/Action/RestoreAction.d.ts

@@ -1,11 +1,14 @@
 import type { ConfigType } from "../Config/Config";
 import { PackageConfigType } from "../Config/PackageConfig";
-import { RepositoryConfigType } from "../Config/RepositoryConfig";
-import { SnapshotResultType } from "../Repository/RepositoryAbstract";
-import { RestoreSessionManager } from "../SessionManager/RestoreSessionManager";
+import { Snapshot } from "../Repository/RepositoryAbstract";
 import { TaskAbstract } from "../Task/TaskAbstract";
+import { DataFormat } from "../utils/DataFormat";
+import { Listr3TaskResultEnd } from "../utils/list";
+import { Progress } from "../utils/progress";
+import { Streams } from "../utils/stream";
+import { GargabeCollector } from "../utils/temp";
 import { IfRequireKeys } from "../utils/ts";
-export type RestoreActionOptionsType = {
+export type RestoreActionOptions = {
     snapshotId: string;
     tags?: string[];
     packageNames?: string[];
@@ -15,32 +18,54 @@ export type RestoreActionOptionsType = {
     repositoryTypes?: string[];
     verbose?: boolean;
     restorePath?: boolean;
+    tty?: "auto" | boolean;
+    progress?: "auto" | "interval" | boolean;
+    progressInterval?: number;
+    streams?: Streams;
 };
-type SnapshotType = SnapshotResultType & {
+type RestoreSnapshot = Snapshot & {
     repositoryName: string;
 };
-type SnapshotAndConfigType = [SnapshotType, PackageConfigType | null];
+type Context = {
+    snapshots: {
+        id: string;
+        packages: number;
+    };
+    task: {
+        taskName: string;
+        packageName: string;
+    };
+    restore: RestoreSnapshot;
+};
 export declare class RestoreAction<TRequired extends boolean = true> {
     readonly config: ConfigType;
-    readonly options: IfRequireKeys<TRequired, RestoreActionOptionsType>;
+    readonly options: IfRequireKeys<TRequired, RestoreActionOptions>;
     protected taskErrors: Record<string, Error[]>;
     protected repoErrors: Record<string, Error[]>;
-    constructor(config: ConfigType, options: IfRequireKeys<TRequired, RestoreActionOptionsType>);
-    protected assocConfigs(packages: PackageConfigType[], snapshots: SnapshotType[]): [SnapshotType, PackageConfigType | null][];
-    protected init(session: RestoreSessionManager, snapshotId: string, snapshots: SnapshotAndConfigType[]): Promise<void>;
-    protected findSnapshots(): Promise<SnapshotType[]>;
-    protected groupSnapshots(snapshots: SnapshotType[]): SnapshotType[];
-    protected task(session: RestoreSessionManager, pkg: PackageConfigType, task: TaskAbstract<any>, snapshot: SnapshotType, targetPath: string | undefined): Promise<{
-        error: boolean;
-        tmpDirs: string[];
-    }>;
-    protected restore(session: RestoreSessionManager, pkg: PackageConfigType, repo: RepositoryConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<{
-        error: boolean;
-        tmpDirs: string[];
-    }>;
-    protected getError(pkg: PackageConfigType): Error | undefined;
-    exec(session: RestoreSessionManager): Promise<{
-        errors: Error[];
+    constructor(config: ConfigType, options: IfRequireKeys<TRequired, RestoreActionOptions>);
+    protected findSnapshots(): Promise<RestoreSnapshot[]>;
+    protected groupSnapshots(snapshots: RestoreSnapshot[]): RestoreSnapshot[];
+    protected restore(data: {
+        pkg: PackageConfigType;
+        task: TaskAbstract | undefined;
+        snapshot: RestoreSnapshot;
+        gc: GargabeCollector;
+        onProgress: (progress: Progress) => void;
+    }): Promise<{
+        snapshotPath: string | undefined;
     }>;
+    dataFormat(result: Listr3TaskResultEnd<Context>[], options?: {
+        streams?: Streams;
+        verbose?: number;
+    }): DataFormat;
+    exec(): Promise<({
+        key: "summary";
+        keyIndex?: string | undefined;
+        data: {
+            errors: number;
+        };
+        elapsed: number;
+        error?: Error | undefined;
+    } | import("../utils/list").Listr3TaskResult<Context>)[]>;
 }
 export {};