@datatruck/cli 0.0.1

package/lib/Action/BackupAction.d.ts

@@ -0,0 +1,32 @@
+import type { ConfigType } from "../Config/Config";
+import { PackageConfigType } from "../Config/PackageConfig";
+import { RepositoryConfigType } from "../Config/RepositoryConfig";
+import { TaskConfigType } from "../Config/TaskConfig";
+import { AppError } from "../Error/AppError";
+import { SnapshotType } from "../Repository/RepositoryAbstract";
+import { BackupSessionManager } from "../SessionManager/BackupSessionManager";
+import { IfRequireKeys } from "../util/ts-util";
+export declare type BackupActionOptionsType = {
+    repositoryNames?: string[];
+    repositoryTypes?: string[];
+    packageNames?: string[];
+    tags?: string[];
+    dryRun?: boolean;
+    verbose?: boolean;
+    date?: string;
+};
+export declare class BackupAction<TRequired extends boolean = true> {
+    readonly config: ConfigType;
+    readonly options: IfRequireKeys<TRequired, BackupActionOptionsType>;
+    protected taskErrors: Record<string, Error[]>;
+    protected repoErrors: Record<string, Error[]>;
+    constructor(config: ConfigType, options?: IfRequireKeys<TRequired, BackupActionOptionsType>);
+    protected init(session: BackupSessionManager): Promise<[SnapshotType, PackageConfigType[]]>;
+    protected execTask(session: BackupSessionManager, pkg: PackageConfigType, task: TaskConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
+    protected execRepository(session: BackupSessionManager, pkg: PackageConfigType, repo: RepositoryConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
+    protected getError(pkg: PackageConfigType): AppError | null;
+    exec(session: BackupSessionManager): Promise<{
+        total: number;
+        errors: number;
+    }>;
+}

package/lib/Action/BackupAction.js

@@ -0,0 +1,201 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BackupAction = void 0;
+const AppError_1 = require("../Error/AppError");
+const RepositoryFactory_1 = require("../Factory/RepositoryFactory");
+const TaskFactory_1 = require("../Factory/TaskFactory");
+const config_util_1 = require("../util/datatruck/config-util");
+const crypto_1 = require("crypto");
+class BackupAction {
+    constructor(config, options = {}) {
+        this.config = config;
+        this.options = options;
+        this.taskErrors = {};
+        this.repoErrors = {};
+    }
+    async init(session) {
+        const snapshot = {
+            id: (0, crypto_1.randomBytes)(20).toString("hex"),
+            date: this.options.date ?? new Date().toISOString(),
+        };
+        await session.initDrivers();
+        let packages = (0, config_util_1.filterPackages)(this.config, {
+            packageNames: this.options.packageNames,
+            repositoryNames: this.options.repositoryNames,
+            repositoryTypes: this.options.repositoryTypes,
+        });
+        packages = (0, config_util_1.resolvePackages)(packages, {
+            snapshotId: snapshot.id,
+            action: "backup",
+        });
+        for (const pkg of packages) {
+            const sessionId = await session.init({
+                snapshotId: snapshot.id,
+                packageName: pkg.name,
+                tags: this.options.tags?.join(",") ?? "",
+            });
+            if (pkg.task)
+                await session.initTask({
+                    sessionId: sessionId,
+                    taskName: pkg.task.name,
+                });
+            for (const repositoryName of pkg.repositoryNames ?? []) {
+                const repo = (0, config_util_1.findRepositoryOrFail)(this.config, repositoryName);
+                await session.initRepository({
+                    sessionId: sessionId,
+                    repositoryName: repositoryName,
+                    repositoryType: repo.type,
+                });
+            }
+        }
+        return [snapshot, packages];
+    }
+    async execTask(session, pkg, task, snapshot, targetPath) {
+        const taskId = session.findTaskId({
+            packageName: pkg.name,
+            taskName: task.name,
+        });
+        await session.startTask({
+            id: taskId,
+        });
+        const key = `${pkg.name}`;
+        let error;
+        if (this.taskErrors[key]?.length) {
+            error = new AppError_1.AppError("Previous task failed");
+        }
+        else {
+            try {
+                const taskInstance = (0, TaskFactory_1.TaskFactory)(task);
+                await taskInstance.onBackup({
+                    package: pkg,
+                    options: this.options,
+                    snapshot,
+                    targetPath,
+                    onProgress: async (data) => {
+                        await session.progressTask({
+                            id: taskId,
+                            progressCurrent: data.current,
+                            progressPercent: data.percent,
+                            progressStep: data.step,
+                            progressStepPercent: data.stepPercent,
+                            progressTotal: data.total,
+                        });
+                    },
+                });
+            }
+            catch (_) {
+                if (!this.taskErrors[key])
+                    this.taskErrors[key] = [];
+                this.taskErrors[key].push((error = _));
+            }
+        }
+        await session.endTask({
+            id: taskId,
+            error: error?.stack,
+        });
+        return error ? false : true;
+    }
+    async execRepository(session, pkg, repo, snapshot, targetPath) {
+        const repositoryId = session.findRepositoryId({
+            packageName: pkg.name,
+            repositoryName: repo.name,
+        });
+        await session.startRepository({
+            id: repositoryId,
+        });
+        let error;
+        if (this.taskErrors[pkg.name]?.length) {
+            error = new AppError_1.AppError("Task failed");
+        }
+        else {
+            try {
+                const repoInstance = (0, RepositoryFactory_1.RepositoryFactory)(repo);
+                await repoInstance.onBackup({
+                    package: pkg,
+                    targetPath,
+                    packageConfig: pkg.repositoryConfigs?.find((config) => config.type === repo.type &&
+                        (!config.names || config.names.includes(repo.name)))?.config,
+                    options: this.options,
+                    snapshot: snapshot,
+                    onProgress: async (data) => {
+                        await session.progressRepository({
+                            id: repositoryId,
+                            progressCurrent: data.current,
+                            progressPercent: data.percent,
+                            progressStep: data.step,
+                            progressStepPercent: data.stepPercent,
+                            progressTotal: data.total,
+                        });
+                    },
+                });
+            }
+            catch (_) {
+                if (!this.repoErrors[pkg.name])
+                    this.repoErrors[pkg.name] = [];
+                this.repoErrors[pkg.name].push((error = _));
+            }
+        }
+        await session.endRepository({
+            id: repositoryId,
+            error: error?.stack,
+        });
+        return error ? false : true;
+    }
+    getError(pkg) {
+        const taskErrors = this.taskErrors[pkg.name]?.length;
+        const repoErrors = this.repoErrors[pkg.name]?.length;
+        if (taskErrors && repoErrors) {
+            return new AppError_1.AppError("Task and repository failed");
+        }
+        else if (taskErrors && !repoErrors) {
+            return new AppError_1.AppError("Task failed");
+        }
+        else if (!taskErrors && repoErrors) {
+            return new AppError_1.AppError("Repository failed");
+        }
+        else {
+            return null;
+        }
+    }
+    async exec(session) {
+        const [snapshot, packages] = await this.init(session);
+        let total = 0;
+        let errors = 0;
+        for (const pkg of packages) {
+            total++;
+            const id = session.findId({
+                packageName: pkg.name,
+            });
+            await session.start({
+                id,
+            });
+            let targetPath;
+            if (pkg.task) {
+                const taskInstance = (0, TaskFactory_1.TaskFactory)(pkg.task);
+                const result = await taskInstance.onBeforeBackup({
+                    options: this.options,
+                    package: pkg,
+                    snapshot,
+                });
+                await this.execTask(session, pkg, pkg.task, snapshot, (targetPath = result?.targetPath));
+            }
+            for (const repoName of pkg.repositoryNames ?? []) {
+                const repo = (0, config_util_1.findRepositoryOrFail)(this.config, repoName);
+                await this.execRepository(session, pkg, repo, snapshot, targetPath);
+            }
+            const error = this.getError(pkg);
+            if (error)
+                errors++;
+            await session.end({
+                id: id,
+                error: error?.message,
+            });
+        }
+        await session.endDrivers();
+        return {
+            total: total,
+            errors: errors,
+        };
+    }
+}
+exports.BackupAction = BackupAction;

package/lib/Action/BackupSessionsAction.d.ts

@@ -0,0 +1,13 @@
+import type { ConfigType } from "../Config/Config";
+import { ReadDataType } from "../SessionDriver/SessionDriverAbstract";
+import { BackupSessionManager } from "../SessionManager/BackupSessionManager";
+import { IfRequireKeys } from "../util/ts-util";
+export declare type BackupSessionsActionOptionsType = ReadDataType & {
+    verbose?: boolean;
+};
+export declare class BackupSessionsAction<TRequired extends boolean = true> {
+    readonly config: ConfigType;
+    readonly options: IfRequireKeys<TRequired, BackupSessionsActionOptionsType>;
+    constructor(config: ConfigType, options: IfRequireKeys<TRequired, BackupSessionsActionOptionsType>);
+    exec(session: BackupSessionManager): Promise<import("../SessionDriver/SessionDriverAbstract").ReadResultType[]>;
+}

package/lib/Action/BackupSessionsAction.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BackupSessionsAction = void 0;
+class BackupSessionsAction {
+    constructor(config, options) {
+        this.config = config;
+        this.options = options;
+    }
+    async exec(session) {
+        await session.initDrivers();
+        const result = await session.readAll(this.options);
+        await session.endDrivers();
+        return result;
+    }
+}
+exports.BackupSessionsAction = BackupSessionsAction;

package/lib/Action/CleanCacheAction.d.ts

@@ -0,0 +1,9 @@
+import { IfRequireKeys } from "../util/ts-util";
+export declare type CleanCacheActionOptionsType = {
+    verbose?: boolean;
+};
+export declare class CleanCacheAction<TRequired extends boolean = true> {
+    readonly options: IfRequireKeys<TRequired, CleanCacheActionOptionsType>;
+    constructor(options: IfRequireKeys<TRequired, CleanCacheActionOptionsType>);
+    exec(): Promise<void>;
+}

package/lib/Action/CleanCacheAction.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CleanCacheAction = void 0;
+const fs_util_1 = require("../util/fs-util");
+const promises_1 = require("fs/promises");
+class CleanCacheAction {
+    constructor(options) {
+        this.options = options;
+    }
+    async exec() {
+        const path = (0, fs_util_1.parentTmpDir)();
+        if (await (0, fs_util_1.existsDir)(path))
+            await (0, promises_1.rm)(path, {
+                recursive: true,
+            });
+    }
+}
+exports.CleanCacheAction = CleanCacheAction;

package/lib/Action/ConfigAction.d.ts

@@ -0,0 +1,14 @@
+import type { ConfigType } from "../Config/Config";
+import { IfRequireKeys } from "../util/ts-util";
+export declare type ConfigActionOptionsType = {
+    path: string;
+    verbose?: boolean;
+};
+export declare class ConfigAction<TRequired extends boolean = true> {
+    readonly options: IfRequireKeys<TRequired, ConfigActionOptionsType>;
+    constructor(options: IfRequireKeys<TRequired, ConfigActionOptionsType>);
+    static validate(config: ConfigType): void;
+    static check(config: ConfigType): void;
+    static normalize(config: ConfigType): ConfigType;
+    exec(): Promise<ConfigType>;
+}

package/lib/Action/ConfigAction.js

@@ -0,0 +1,64 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConfigAction = void 0;
+const AppError_1 = require("../Error/AppError");
+const JsonSchema_1 = require("../JsonSchema/JsonSchema");
+const config_util_1 = require("../util/datatruck/config-util");
+const fs_util_1 = require("../util/fs-util");
+const ajv_1 = __importDefault(require("ajv"));
+const assert_1 = require("assert");
+const path_1 = require("path");
+class ConfigAction {
+    constructor(options) {
+        this.options = options;
+    }
+    static validate(config) {
+        const validate = new ajv_1.default().compile(JsonSchema_1.schema);
+        if (!validate(config))
+            throw new AppError_1.AppError("Json schema error: " + JSON.stringify(validate.errors, null, 2));
+    }
+    static check(config) {
+        const repositoryNames = [];
+        for (const repo of config.repositories) {
+            if (repositoryNames.includes(repo.name))
+                throw new AppError_1.AppError(`Duplicated repository name: ${repo.name}`);
+            repositoryNames.push(repo.name);
+        }
+        const packageNames = [];
+        for (const pkg of config.packages) {
+            if (packageNames.includes(pkg.name))
+                throw new AppError_1.AppError(`Duplicated package name: ${pkg.name}`);
+            repositoryNames.push(pkg.name);
+        }
+    }
+    static normalize(config) {
+        config = Object.assign({}, config);
+        config.packages = config.packages.map((pkg) => {
+            pkg = Object.assign({}, pkg);
+            if (!pkg.restorePath)
+                pkg.restorePath = pkg.path
+                    ? pkg.path
+                    : (0, path_1.normalize)(`{temp}/{snapshotId}-{action}/{packageName}`);
+            if (!pkg.path)
+                pkg.path = (0, path_1.normalize)(`{temp}/{snapshotId}-{action}/{packageName}`);
+            pkg.repositoryNames =
+                pkg.repositoryNames ?? config.repositories.map((repo) => repo.name);
+            (0, assert_1.ok)(Array.isArray(pkg.repositoryNames));
+            for (const repoName of pkg.repositoryNames)
+                (0, config_util_1.findRepositoryOrFail)(config, repoName);
+            return pkg;
+        });
+        return config;
+    }
+    async exec() {
+        const path = await (0, fs_util_1.findFile)(this.options.path, "datatruck.config", fs_util_1.parseFileExtensions, "Config path not found");
+        const config = await (0, fs_util_1.parseFile)(path, "config");
+        ConfigAction.validate(config);
+        ConfigAction.check(config);
+        return ConfigAction.normalize(config);
+    }
+}
+exports.ConfigAction = ConfigAction;

package/lib/Action/InitAction.d.ts

@@ -0,0 +1,18 @@
+import type { ConfigType } from "../Config/Config";
+import { IfRequireKeys } from "../util/ts-util";
+export declare type InitActionOptionsType = {
+    repositoryNames?: string[];
+    repositoryTypes?: string[];
+    verbose?: boolean;
+};
+export declare class InitAction<TRequired extends boolean = true> {
+    readonly config: ConfigType;
+    readonly options: IfRequireKeys<TRequired, InitActionOptionsType>;
+    constructor(config: ConfigType, options: IfRequireKeys<TRequired, InitActionOptionsType>);
+    exec(): Promise<{
+        repositoryName: string;
+        repositoryType: string;
+        repositorySource: string;
+        error: Error | null;
+    }[]>;
+}

package/lib/Action/InitAction.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InitAction = void 0;
+const RepositoryFactory_1 = require("../Factory/RepositoryFactory");
+class InitAction {
+    constructor(config, options) {
+        this.config = config;
+        this.options = options;
+    }
+    async exec() {
+        const result = [];
+        for (const repo of this.config.repositories) {
+            if (this.options.repositoryNames &&
+                !this.options.repositoryNames.includes(repo.name))
+                continue;
+            if (this.options.repositoryTypes &&
+                !this.options.repositoryTypes.includes(repo.type))
+                continue;
+            const repoInstance = (0, RepositoryFactory_1.RepositoryFactory)(repo);
+            let initError = null;
+            try {
+                await repoInstance.onInit({
+                    options: this.options,
+                });
+            }
+            catch (error) {
+                initError = error;
+            }
+            result.push({
+                repositoryName: repo.name,
+                repositoryType: repo.type,
+                repositorySource: repoInstance.onGetSource(),
+                error: initError,
+            });
+        }
+        return result;
+    }
+}
+exports.InitAction = InitAction;

package/lib/Action/PruneAction.d.ts

@@ -0,0 +1,37 @@
+import type { ConfigType } from "../Config/Config";
+import { RepositoryConfigType } from "../Config/RepositoryConfig";
+import { IfRequireKeys } from "../util/ts-util";
+import { SnapshotExtendedType, SnapshotsActionOptionsType } from "./SnapshotsAction";
+export declare type PruneActionsOptionsType = {
+    ids?: string[];
+    packageNames?: string[];
+    repositoryNames?: string[];
+    repositoryTypes?: RepositoryConfigType["type"][];
+    tags?: string[];
+    keepLast?: number;
+    keepMinutely?: number;
+    keepHourly?: number;
+    keepDaily?: number;
+    keepWeekly?: number;
+    keepMonthly?: number;
+    keepYearly?: number;
+    verbose?: boolean;
+    groupBy?: SnapshotsActionOptionsType["groupBy"];
+    dryRun?: boolean;
+    longId?: boolean;
+    returnsAll?: boolean;
+};
+export declare type PruneResultType = {
+    total: number;
+    prune: number;
+    snapshots: (SnapshotExtendedType & {
+        exclusionReasons: string[];
+    })[];
+};
+export declare class PruneAction<TRequired extends boolean = true> {
+    readonly config: ConfigType;
+    readonly options: IfRequireKeys<TRequired, PruneActionsOptionsType>;
+    constructor(config: ConfigType, options: IfRequireKeys<TRequired, PruneActionsOptionsType>);
+    confirm(snapshots: PruneResultType["snapshots"]): Promise<void>;
+    exec(): Promise<PruneResultType>;
+}

package/lib/Action/PruneAction.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PruneAction = void 0;
+const RepositoryFactory_1 = require("../Factory/RepositoryFactory");
+const snapshot_util_1 = require("../util/datatruck/snapshot-util");
+const object_util_1 = require("../util/object-util");
+const SnapshotsAction_1 = require("./SnapshotsAction");
+class PruneAction {
+    constructor(config, options) {
+        this.config = config;
+        this.options = options;
+    }
+    async confirm(snapshots) {
+        const repository = (0, object_util_1.groupBy)(this.config.repositories, "name", true);
+        for (const snapshot of snapshots) {
+            if (!snapshot.exclusionReasons?.length) {
+                const repoInstance = (0, RepositoryFactory_1.RepositoryFactory)(repository[snapshot.repositoryName]);
+                await repoInstance.onPrune({
+                    snapshot: snapshot,
+                    options: { verbose: this.options.verbose },
+                });
+            }
+        }
+    }
+    async exec() {
+        const snapshotsAction = new SnapshotsAction_1.SnapshotsAction(this.config, this.options);
+        const snapshots = await snapshotsAction.exec();
+        const snapshotsDeleted = [];
+        const reasons = {};
+        const inputFilter = {
+            last: this.options.keepLast,
+            lastMinutely: this.options.keepMinutely,
+            lastHourly: this.options.keepHourly,
+            lastDaily: this.options.keepDaily,
+            lastMonthly: this.options.keepMonthly,
+            lastWeekly: this.options.keepWeekly,
+            lastYearly: this.options.keepYearly,
+        };
+        const hasInputFilter = Object.values(inputFilter).some(Number);
+        const usePrunePolicyConfig = !hasInputFilter && this.options.groupBy?.includes("packageName");
+        const keepSnapshots = (0, snapshot_util_1.groupAndFilter)(snapshots, this.options.groupBy, usePrunePolicyConfig
+            ? (groupedSnapshots) => {
+                const [firstSnapshot] = groupedSnapshots;
+                const packageName = firstSnapshot.packageName;
+                const config = this.config.packages.find((pkg) => pkg.name === packageName);
+                const prunePolicy = config?.prunePolicy ?? {};
+                return {
+                    last: prunePolicy.keepLast,
+                    lastMinutely: prunePolicy.keepMinutely,
+                    lastHourly: prunePolicy.keepHourly,
+                    lastDaily: prunePolicy.keepDaily,
+                    lastMonthly: prunePolicy.keepMonthly,
+                    lastWeekly: prunePolicy.keepWeekly,
+                    lastYearly: prunePolicy.keepYearly,
+                };
+            }
+            : inputFilter, reasons);
+        const result = {
+            total: snapshots.length,
+            prune: 0,
+            snapshots: snapshotsDeleted,
+        };
+        let snapshotIndex = 0;
+        for (const snapshot of snapshots) {
+            const prune = !keepSnapshots.includes(snapshot);
+            if (prune)
+                result.prune++;
+            if (prune || this.options.returnsAll)
+                snapshotsDeleted.push({
+                    ...snapshot,
+                    exclusionReasons: reasons[snapshotIndex],
+                });
+            snapshotIndex++;
+        }
+        if (!this.options.dryRun)
+            await this.confirm(snapshotsDeleted);
+        return result;
+    }
+}
+exports.PruneAction = PruneAction;

package/lib/Action/RestoreAction.d.ts

@@ -0,0 +1,36 @@
+import type { ConfigType } from "../Config/Config";
+import { PackageConfigType } from "../Config/PackageConfig";
+import { RepositoryConfigType } from "../Config/RepositoryConfig";
+import { TaskConfigType } from "../Config/TaskConfig";
+import { AppError } from "../Error/AppError";
+import { SnapshotResultType } from "../Repository/RepositoryAbstract";
+import { RestoreSessionManager } from "../SessionManager/RestoreSessionManager";
+import { IfRequireKeys } from "../util/ts-util";
+export declare type RestoreActionOptionsType = {
+    snapshotId: string;
+    tags?: string[];
+    packageNames?: string[];
+    repositoryNames?: string[];
+    repositoryTypes?: string[];
+    verbose?: boolean;
+};
+declare type SnapshotType = SnapshotResultType & {
+    repositoryName: string;
+};
+declare type SnapshotAndConfigType = [SnapshotType, PackageConfigType | null];
+export declare class RestoreAction<TRequired extends boolean = true> {
+    readonly config: ConfigType;
+    readonly options: IfRequireKeys<TRequired, RestoreActionOptionsType>;
+    protected taskErrors: Record<string, Error[]>;
+    protected repoErrors: Record<string, Error[]>;
+    constructor(config: ConfigType, options: IfRequireKeys<TRequired, RestoreActionOptionsType>);
+    protected assocConfigs(packages: PackageConfigType[], snapshots: SnapshotType[]): [SnapshotType, PackageConfigType | null][];
+    protected init(session: RestoreSessionManager, snapshotId: string, snapshots: SnapshotAndConfigType[]): Promise<void>;
+    protected findSnapshots(): Promise<SnapshotType[]>;
+    protected groupSnapshots(snapshots: SnapshotType[]): SnapshotType[];
+    protected execTask(session: RestoreSessionManager, pkg: PackageConfigType, task: TaskConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
+    protected execRepository(session: RestoreSessionManager, pkg: PackageConfigType, repo: RepositoryConfigType, snapshot: SnapshotType, targetPath: string | undefined): Promise<boolean>;
+    protected getError(pkg: PackageConfigType): AppError | null;
+    exec(session: RestoreSessionManager): Promise<boolean>;
+}
+export {};