@datatruck/cli 0.0.1

package/lib/Repository/ResticRepository.js

@@ -0,0 +1,230 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ResticRepository = exports.resticPackageRepositoryDefinition = exports.resticRepositoryDefinition = exports.resticRepositoryName = void 0;
+const AppError_1 = require("../Error/AppError");
+const ResticUtil_1 = require("../util/ResticUtil");
+const paths_util_1 = require("../util/datatruck/paths-util");
+const fs_util_1 = require("../util/fs-util");
+const string_util_1 = require("../util/string-util");
+const RepositoryAbstract_1 = require("./RepositoryAbstract");
+const assert_1 = require("assert");
+const micromatch_1 = require("micromatch");
+const path_1 = require("path");
+exports.resticRepositoryName = "restic";
+exports.resticRepositoryDefinition = {
+    type: "object",
+    required: ["passwordFile", "repository"],
+    additionalProperties: false,
+    properties: {
+        passwordFile: { type: "string" },
+        repository: {
+            type: "object",
+            additionalProperties: false,
+            required: ["backend"],
+            properties: {
+                name: { type: "string" },
+                env: {
+                    type: "object",
+                    patternProperties: {
+                        ".+": { type: "string" },
+                    },
+                },
+                backend: {
+                    enum: ["local", "rest", "sftp", "s3", "azure", "gs", "rclone"],
+                },
+                protocol: {
+                    enum: ["http", "https"],
+                },
+                host: { type: "string" },
+                username: { type: "string" },
+                passwordFile: { type: "string" },
+                port: { type: "integer" },
+                path: { type: "string" },
+            },
+        },
+    },
+};
+exports.resticPackageRepositoryDefinition = {
+    type: "object",
+    additionalProperties: false,
+    properties: {},
+};
+class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
+    async buildEnv() {
+        if (this.env)
+            return this.env;
+        return (this.env = {
+            RESTIC_PASSWORD_FILE: this.config.passwordFile,
+            RESTIC_REPOSITORY: await ResticUtil_1.ResticUtil.formatRepository(this.config.repository),
+        });
+    }
+    static buildSnapshotTag(name, value) {
+        return `${ResticRepository.refPrefix}${name}:${value}`;
+    }
+    static parseSnapshotTag(tag) {
+        for (const metaName in RepositoryAbstract_1.SnapshotTagEnum) {
+            const name = RepositoryAbstract_1.SnapshotTagEnum[metaName];
+            const prefix = `${ResticRepository.refPrefix}${name}:`;
+            if (tag.startsWith(prefix))
+                return {
+                    name: name,
+                    value: tag.slice(prefix.length),
+                };
+        }
+        return null;
+    }
+    static parseSnapshotTags(tags) {
+        const result = {
+            tags: [],
+        };
+        for (const tag of tags) {
+            const tagItem = ResticRepository.parseSnapshotTag(tag);
+            if (tagItem && tagItem.name !== "tags") {
+                result[tagItem.name] = tagItem.value;
+            }
+            else {
+                result.tags.push(tag);
+            }
+        }
+        return result;
+    }
+    onGetSource() {
+        return (0, string_util_1.formatUri)({ ...this.config.repository, password: undefined });
+    }
+    async onInit(data) {
+        const restic = new ResticUtil_1.ResticUtil({
+            env: await this.buildEnv(),
+            log: data.options.verbose,
+        });
+        if (this.config.repository.backend === "local")
+            await (0, fs_util_1.mkdirIfNotExists)(this.env.RESTIC_REPOSITORY);
+        if (!(await restic.checkRepository()))
+            await restic.exec(["init"]);
+    }
+    async onSnapshots(data) {
+        const restic = new ResticUtil_1.ResticUtil({
+            env: await this.buildEnv(),
+            log: data.options.verbose,
+        });
+        const packagePatterns = (0, string_util_1.makePathPatterns)(data.options.packageNames);
+        const result = await restic.snapshots({
+            json: true,
+            tags: [
+                ...(data.options.ids?.map((id) => ResticRepository.buildSnapshotTag(id.length === 8 ? RepositoryAbstract_1.SnapshotTagEnum.SHORT_ID : RepositoryAbstract_1.SnapshotTagEnum.ID, id)) ?? []),
+            ],
+        });
+        return result.reduce((items, item) => {
+            const tag = ResticRepository.parseSnapshotTags(item.tags ?? []);
+            if (!tag.id)
+                return items;
+            if (packagePatterns && !(0, micromatch_1.isMatch)(tag.package, packagePatterns))
+                return items;
+            const itemTags = tag.tags ?? [];
+            if (data.options.tags && !itemTags.some((t) => itemTags.includes(t)))
+                return items;
+            items.push({
+                originalId: item.id,
+                packageName: tag.package,
+                date: tag.date,
+                id: tag.id,
+                tags: itemTags,
+            });
+            return items;
+        }, []);
+    }
+    async onPrune(data) {
+        const restic = new ResticUtil_1.ResticUtil({
+            env: await this.buildEnv(),
+            log: data.options.verbose,
+        });
+        await restic.forget({
+            snapshotId: data.snapshot.originalId,
+            prune: true,
+        });
+    }
+    async onBackup(data) {
+        const restic = new ResticUtil_1.ResticUtil({
+            env: await this.buildEnv(),
+            log: data.options.verbose,
+        });
+        const pkg = data.package;
+        const sourcePath = data.targetPath ?? data.package.path;
+        (0, assert_1.ok)(sourcePath);
+        const include = (await (0, paths_util_1.parsePaths)(pkg.include ?? [], {
+            cwd: sourcePath,
+            verbose: data.options.verbose,
+        })).map(path_1.normalize);
+        // https://github.com/restic/restic/issues/233
+        // https://github.com/restic/restic/pull/2311
+        if (include.length)
+            throw new AppError_1.AppError(`Include is not supported`);
+        if (data.options.tags?.some((tag) => tag.startsWith(ResticRepository.refPrefix)))
+            throw new AppError_1.AppError(`Tag prefix is not allowed`);
+        const packageTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.PACKAGE, data.package.name);
+        const [lastSnapshot] = await restic.snapshots({
+            json: true,
+            tags: [packageTag],
+            latest: 1,
+        });
+        const nodePkg = (0, fs_util_1.parsePackageFile)();
+        await restic.backup({
+            cwd: sourcePath,
+            paths: ["."],
+            parent: lastSnapshot?.id,
+            // https://github.com/restic/restic/pull/3200
+            ...((await restic.checkBackupSetPathSupport()) && {
+                setPaths: [`/datatruck/${data.package.name}`],
+            }),
+            exclude: (await (0, paths_util_1.parsePaths)(pkg.exclude ?? [], {
+                cwd: sourcePath,
+                verbose: data.options.verbose,
+            })).map(path_1.normalize),
+            tags: [
+                ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.ID, data.snapshot.id),
+                ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.SHORT_ID, data.snapshot.id.slice(0, 8)),
+                ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.DATE, data.snapshot.date),
+                ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.VERSION, nodePkg.version),
+                packageTag,
+                ...(data.options.tags ?? []),
+            ],
+            onStream: async (streamData) => {
+                if (streamData.message_type === "status") {
+                    await data.onProgress({
+                        total: streamData.total_bytes,
+                        current: streamData.bytes_done ?? 0,
+                        percent: Number((streamData.percent_done * 100).toFixed(2)),
+                        step: streamData.current_files?.join(", ") ?? "-",
+                    });
+                }
+            },
+        });
+        await data.onProgress({
+            current: 100,
+            percent: 100,
+            step: "",
+            total: 100,
+        });
+    }
+    async onRestore(data) {
+        const restorePath = data.targetPath ?? data.package.restorePath;
+        (0, assert_1.ok)(restorePath);
+        const restic = new ResticUtil_1.ResticUtil({
+            env: await this.buildEnv(),
+            log: data.options.verbose,
+        });
+        const [snapshot] = await this.onSnapshots({
+            options: {
+                ids: [data.snapshot.id],
+                packageNames: [data.package.name],
+            },
+        });
+        if (!snapshot)
+            throw new AppError_1.AppError(`Snapshot not found`);
+        await restic.restore({
+            id: snapshot.originalId,
+            target: restorePath,
+        });
+    }
+}
+exports.ResticRepository = ResticRepository;
+ResticRepository.refPrefix = "dt-";

package/lib/SessionDriver/ConsoleSessionDriver.d.ts

@@ -0,0 +1,37 @@
+/// <reference types="node" />
+import { WriteDataType, ReadResultType, SessionDriverAbstract } from "./SessionDriverAbstract";
+declare type BadgeType = {
+    name: string;
+    value: string;
+    color: (input: string) => string;
+};
+declare type MessageType = {
+    sessionId: number;
+    level?: number;
+    textPrefix?: string;
+    text?: string;
+    badges: BadgeType[];
+    errorBadge?: BadgeType;
+    progressCurrent?: number | null;
+    progressTotal?: number | null;
+    progressPercent?: number | null;
+    progressStep?: string | null;
+    progressStepPercent?: number | null;
+};
+export declare class ConsoleSessionDriver extends SessionDriverAbstract {
+    protected lastMessage: MessageType | undefined;
+    protected lastMessageText: string | undefined;
+    protected prints: number;
+    protected renderInterval: NodeJS.Timeout;
+    protected rendering?: boolean;
+    protected lastColumns?: number;
+    protected startTime: number;
+    onInit(): Promise<void>;
+    onEnd(): Promise<void>;
+    onRead(): Promise<ReadResultType[]>;
+    protected printMessage(message: MessageType): void;
+    protected renderSpinner(text: string): string;
+    protected renderMessage(message: MessageType): string;
+    onWrite(data: WriteDataType): Promise<void>;
+}
+export {};

package/lib/SessionDriver/ConsoleSessionDriver.js

@@ -0,0 +1,178 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConsoleSessionDriver = void 0;
+const AppError_1 = require("../Error/AppError");
+const cli_util_1 = require("../util/cli-util");
+const SessionDriverAbstract_1 = require("./SessionDriverAbstract");
+const chalk_1 = require("chalk");
+const sep = (0, chalk_1.grey)(`|`);
+const renderBadge = (badge) => `${badge.color(badge.name)}${(0, chalk_1.grey)(`:`)} ${(0, chalk_1.white)(badge.value)}`;
+const renderBadges = (badges) => badges.map(renderBadge).join(` ${sep} `);
+class ConsoleSessionDriver extends SessionDriverAbstract_1.SessionDriverAbstract {
+    constructor() {
+        super(...arguments);
+        this.prints = 0;
+    }
+    async onInit() {
+        this.startTime = Date.now();
+        this.renderInterval = setInterval(() => {
+            if (this.lastMessage)
+                this.printMessage(this.lastMessage);
+        }, 100);
+    }
+    async onEnd() {
+        clearInterval(this.renderInterval);
+        if (!this.options.verbose)
+            process.stdout.write(cli_util_1.showCursorCommand);
+        const ellapsed = (Date.now() - this.startTime) / 1000;
+        let ellapsedUnit;
+        let ellapsedValue;
+        if (ellapsed > 60 * 60) {
+            ellapsedValue = ellapsed / 60 / 60;
+            ellapsedUnit = `hour`;
+        }
+        else if (ellapsed > 60) {
+            ellapsedValue = ellapsed / 60;
+            ellapsedUnit = `minute`;
+        }
+        else {
+            ellapsedValue = ellapsed;
+            ellapsedUnit = `second`;
+        }
+        if (ellapsedValue !== 1)
+            ellapsedUnit += `s`;
+        const message = `Completed in ${ellapsedValue.toFixed(2)} ${ellapsedUnit}`;
+        console.info(`\n${(0, chalk_1.grey)(message)}`);
+    }
+    async onRead() {
+        throw new AppError_1.AppError("Method not implemented");
+    }
+    printMessage(message) {
+        const text = this.renderMessage(message);
+        if (this.options.verbose && this.lastMessageText === text) {
+            return;
+        }
+        if (this.options.verbose) {
+            process.stdout.write(`${this.renderSpinner(text)}\n`);
+        }
+        else {
+            const columns = process.stdout.columns;
+            const line = this.renderSpinner(text);
+            const [truncatedLine, truncted] = (0, cli_util_1.truncate)(line, columns);
+            if (this.lastColumns && columns !== this.lastColumns && truncted)
+                process.stdout.write(`${cli_util_1.clearCommand}\n`);
+            process.stdout.write(`${cli_util_1.clearCommand}${truncatedLine}${cli_util_1.hideCursorCommand}`);
+            this.lastColumns = columns;
+        }
+        this.prints++;
+        this.lastMessage = message;
+        this.lastMessageText = text;
+    }
+    renderSpinner(text) {
+        return text.replace("{spinner}", (0, chalk_1.grey)(this.options.verbose ? "?" : (0, cli_util_1.renderSpinner)(this.prints)));
+    }
+    renderMessage(message) {
+        const badges = renderBadges([
+            ...message.badges,
+            ...(message.errorBadge ? [message.errorBadge] : []),
+        ]);
+        const padding = "   ".repeat(message.level ?? 0);
+        const haveProgressBar = typeof message.progressPercent === "number";
+        const sessionId = message.sessionId.toString().padStart(2, "0");
+        const parts = [
+            `${padding}${message.textPrefix} [${(0, chalk_1.grey)(sessionId)}] ${message.text}`,
+            badges,
+            ...(haveProgressBar
+                ? [
+                    (0, chalk_1.cyan)((0, cli_util_1.renderProgressBar)(message.progressPercent ?? 0, 10)),
+                    `${message.progressPercent?.toFixed(2)}%`,
+                    `${message.progressCurrent}/${message.progressTotal}`,
+                    message.progressStep,
+                    `${message.progressStepPercent
+                        ? (0, chalk_1.cyan)((0, cli_util_1.renderProgressBar)(message.progressStepPercent ?? 0, 10))
+                        : ""}`,
+                ].filter((v) => !!v?.length)
+                : []),
+        ];
+        return parts.join(` ${sep} `);
+    }
+    async onWrite(data) {
+        if (data.action === SessionDriverAbstract_1.ActionEnum.Init)
+            return;
+        const message = {
+            sessionId: "sessionId" in data.data ? data.data.sessionId : data.data.id,
+            badges: [],
+        };
+        const isHeader = data.entity === SessionDriverAbstract_1.EntityEnum.BackupSession ||
+            data.entity === SessionDriverAbstract_1.EntityEnum.RestoreSession;
+        const hasProgress = isHeader ||
+            data.entity === SessionDriverAbstract_1.EntityEnum.BackupSessionTask ||
+            data.entity === SessionDriverAbstract_1.EntityEnum.BackupSessionRepository ||
+            data.entity === SessionDriverAbstract_1.EntityEnum.RestoreSessionTask ||
+            data.entity === SessionDriverAbstract_1.EntityEnum.RestoreSessionRepository;
+        if (data.action === SessionDriverAbstract_1.ActionEnum.Start) {
+            if (isHeader) {
+                message.textPrefix = data.data.error ? (0, chalk_1.red)("⨉") : (0, chalk_1.green)("✓");
+            }
+            else {
+                message.textPrefix = hasProgress ? "{spinner}" : (0, chalk_1.grey)("?");
+            }
+        }
+        else if (data.action === SessionDriverAbstract_1.ActionEnum.End) {
+            message.textPrefix = data.data.error ? (0, chalk_1.red)("⨉") : (0, chalk_1.green)("✓");
+            if (data.data.error)
+                message.errorBadge = {
+                    name: "error",
+                    value: this.options.verbose
+                        ? data.data.error
+                        : data.data.error.split("\n")[0],
+                    color: chalk_1.red,
+                };
+        }
+        else if (data.action === SessionDriverAbstract_1.ActionEnum.Progress) {
+            message.textPrefix = "{spinner}";
+        }
+        if (hasProgress) {
+            message.progressPercent = data.data.progressPercent;
+            message.progressCurrent = data.data.progressCurrent;
+            message.progressTotal = data.data.progressTotal;
+            message.progressStep = data.data.progressStep;
+            message.progressStepPercent = data.data.progressStepPercent;
+        }
+        if (data.entity === SessionDriverAbstract_1.EntityEnum.BackupSession ||
+            data.entity === SessionDriverAbstract_1.EntityEnum.RestoreSession) {
+            message.text = data.data.packageName;
+            message.badges.push({
+                name: "snap",
+                value: data.data.snapshotId.slice(0, 8),
+                color: chalk_1.cyan,
+            });
+        }
+        else if (data.entity === SessionDriverAbstract_1.EntityEnum.BackupSessionTask ||
+            data.entity === SessionDriverAbstract_1.EntityEnum.RestoreSessionTask) {
+            message.text = data.sessionData.packageName;
+            message.badges.push({
+                name: "task",
+                value: data.data.taskName,
+                color: chalk_1.cyan,
+            });
+        }
+        else if (data.entity === SessionDriverAbstract_1.EntityEnum.BackupSessionRepository ||
+            data.entity === SessionDriverAbstract_1.EntityEnum.RestoreSessionRepository) {
+            message.text = data.sessionData.packageName;
+            message.badges.push({
+                name: "repo",
+                value: data.data.repositoryName,
+                color: chalk_1.cyan,
+            });
+        }
+        if (isHeader && data.action === SessionDriverAbstract_1.ActionEnum.End) {
+            return;
+        }
+        this.printMessage(message);
+        if (!this.options.verbose)
+            if (!hasProgress || data.action === SessionDriverAbstract_1.ActionEnum.End || isHeader)
+                process.stdout.write("\n");
+    }
+}
+exports.ConsoleSessionDriver = ConsoleSessionDriver;

package/lib/SessionDriver/SessionDriverAbstract.d.ts

@@ -0,0 +1,78 @@
+import { BackupSessionEntity } from "../Entity/BackupSessionEntity";
+import { BackupSessionRepositoryEntity } from "../Entity/BackupSessionRepositoryEntity";
+import { BackupSessionTaskEntity } from "../Entity/BackupSessionTaskEntity";
+import { RestoreSessionEntity } from "../Entity/RestoreSessionEntity";
+import { RestoreSessionRepositoryEntity } from "../Entity/RestoreSessionRepositoryEntity";
+import { RestoreSessionTaskEntity } from "../Entity/RestoreSessionTaskEntity";
+export declare enum ActionEnum {
+    Init = 0,
+    Start = 1,
+    Progress = 2,
+    End = 3
+}
+export declare enum EntityEnum {
+    BackupSession = 0,
+    BackupSessionTask = 1,
+    BackupSessionRepository = 2,
+    RestoreSession = 3,
+    RestoreSessionTask = 4,
+    RestoreSessionRepository = 5
+}
+export declare type WriteDataType = {
+    action: ActionEnum;
+    entity: EntityEnum.BackupSession;
+    data: BackupSessionEntity;
+} | {
+    action: ActionEnum;
+    entity: EntityEnum.BackupSessionRepository;
+    data: BackupSessionRepositoryEntity;
+    sessionData: BackupSessionEntity;
+} | {
+    action: ActionEnum;
+    entity: EntityEnum.BackupSessionTask;
+    data: BackupSessionTaskEntity;
+    sessionData: BackupSessionEntity;
+} | {
+    action: ActionEnum;
+    entity: EntityEnum.RestoreSession;
+    data: RestoreSessionEntity;
+} | {
+    action: ActionEnum;
+    entity: EntityEnum.RestoreSessionTask;
+    data: RestoreSessionTaskEntity;
+    sessionData: RestoreSessionEntity;
+} | {
+    action: ActionEnum;
+    entity: EntityEnum.RestoreSessionRepository;
+    data: RestoreSessionRepositoryEntity;
+    sessionData: RestoreSessionEntity;
+};
+export declare type ReadDataType = {
+    repositoryNames?: string[];
+    packageNames?: string[];
+    tags?: string[];
+    limit?: number | null;
+    verbose?: boolean;
+};
+export declare type ReadResultType = {
+    id: number;
+    snapshotId: string;
+    creationDate: string;
+    state: "started" | "ended";
+    packageName: string;
+    repositoryName: string;
+    repositoryType: string;
+    error: string | null;
+};
+export declare abstract class SessionDriverAbstract {
+    readonly options: {
+        verbose?: boolean;
+    };
+    constructor(options: {
+        verbose?: boolean;
+    });
+    onInit(): Promise<void>;
+    abstract onWrite(data: WriteDataType): Promise<void>;
+    onEnd(): Promise<void>;
+    abstract onRead(data: ReadDataType, entity: EntityEnum): Promise<ReadResultType[]>;
+}

package/lib/SessionDriver/SessionDriverAbstract.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SessionDriverAbstract = exports.EntityEnum = exports.ActionEnum = void 0;
+var ActionEnum;
+(function (ActionEnum) {
+    ActionEnum[ActionEnum["Init"] = 0] = "Init";
+    ActionEnum[ActionEnum["Start"] = 1] = "Start";
+    ActionEnum[ActionEnum["Progress"] = 2] = "Progress";
+    ActionEnum[ActionEnum["End"] = 3] = "End";
+})(ActionEnum = exports.ActionEnum || (exports.ActionEnum = {}));
+var EntityEnum;
+(function (EntityEnum) {
+    EntityEnum[EntityEnum["BackupSession"] = 0] = "BackupSession";
+    EntityEnum[EntityEnum["BackupSessionTask"] = 1] = "BackupSessionTask";
+    EntityEnum[EntityEnum["BackupSessionRepository"] = 2] = "BackupSessionRepository";
+    EntityEnum[EntityEnum["RestoreSession"] = 3] = "RestoreSession";
+    EntityEnum[EntityEnum["RestoreSessionTask"] = 4] = "RestoreSessionTask";
+    EntityEnum[EntityEnum["RestoreSessionRepository"] = 5] = "RestoreSessionRepository";
+})(EntityEnum = exports.EntityEnum || (exports.EntityEnum = {}));
+class SessionDriverAbstract {
+    constructor(options) {
+        this.options = options;
+    }
+    async onInit() { }
+    async onEnd() { }
+}
+exports.SessionDriverAbstract = SessionDriverAbstract;

package/lib/SessionDriver/SqliteSessionDriver.d.ts

@@ -0,0 +1,20 @@
+/// <reference types="./vendor-typings/sqlite3" />
+import { WriteDataType, EntityEnum, ReadDataType, SessionDriverAbstract } from "./SessionDriverAbstract";
+import { Database } from "sqlite";
+import sqlite3 from "sqlite3";
+export declare class SqliteSessionDriver extends SessionDriverAbstract {
+    protected idMap: {
+        [entity in EntityEnum]?: Record<number, number>;
+    };
+    protected db: Database<sqlite3.Database, sqlite3.Statement>;
+    onInit(): Promise<void>;
+    private buildInsertStm;
+    private buildUpdateStm;
+    private exec;
+    onRead(data: ReadDataType, type: EntityEnum): Promise<any[]>;
+    private setMapId;
+    private getMapId;
+    private static getEntityTable;
+    private static getParentEntity;
+    onWrite(data: WriteDataType): Promise<void>;
+}