@datatruck/cli 0.0.1

package/lib/Repository/LocalRepository.d.ts

@@ -0,0 +1,51 @@
+import { RepositoryAbstract, BackupDataType, InitDataType, RestoreDataType, SnapshotsDataType, SnapshotResultType, PruneDataType } from "./RepositoryAbstract";
+import type { JSONSchema7 } from "json-schema";
+export declare type MetaDataType = {
+    id: string;
+    date: string;
+    package: string;
+    tags: string[];
+    version: string;
+};
+export declare type LocalRepositoryConfigType = {
+    outPath: string;
+    compress?: boolean;
+};
+declare type CompressObjectType = {
+    packs?: {
+        include: string[];
+        exclude?: string[];
+        onePackByResult?: boolean;
+    }[];
+};
+export declare type LocalPackageRepositoryConfigType = {
+    compress?: CompressObjectType | boolean;
+};
+export declare const localRepositoryName = "local";
+export declare const localRepositoryDefinition: JSONSchema7;
+export declare const localPackageRepositoryDefinition: JSONSchema7;
+export declare class LocalRepository extends RepositoryAbstract<LocalRepositoryConfigType> {
+    static zipBasenameTpl: string;
+    static buildSnapshotName(data: {
+        snapshotId: string;
+        snapshotDate: string;
+        packageName: string;
+    }): string;
+    static parseSnapshotName(name: string): {
+        snapshotDate: string;
+        packageName: string;
+        snapshotShortId: string;
+        sourcePath: string;
+    } | null;
+    protected buildMetaPath(snapshotName: string, packageName: string): string;
+    static parseMetaData(path: string): Promise<MetaDataType>;
+    static stringifyMetaData(data: MetaDataType): string;
+    onGetSource(): string;
+    onInit(data: InitDataType): Promise<void>;
+    onPrune(data: PruneDataType): Promise<void>;
+    onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
+    private normalizeCompressConfig;
+    onBackup(data: BackupDataType<LocalPackageRepositoryConfigType>): Promise<void>;
+    onRestore(data: RestoreDataType<LocalPackageRepositoryConfigType>): Promise<void>;
+}
+export {};

package/lib/Repository/LocalRepository.js

@@ -0,0 +1,358 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalRepository = exports.localPackageRepositoryDefinition = exports.localRepositoryDefinition = exports.localRepositoryName = void 0;
+const AppError_1 = require("../Error/AppError");
+const DefinitionEnum_1 = require("../JsonSchema/DefinitionEnum");
+const cli_util_1 = require("../util/cli-util");
+const paths_util_1 = require("../util/datatruck/paths-util");
+const fs_util_1 = require("../util/fs-util");
+const math_util_1 = require("../util/math-util");
+const string_util_1 = require("../util/string-util");
+const zip_util_1 = require("../util/zip-util");
+const RepositoryAbstract_1 = require("./RepositoryAbstract");
+const assert_1 = require("assert");
+const fast_glob_1 = __importDefault(require("fast-glob"));
+const fs_1 = require("fs");
+const fs_extra_1 = require("fs-extra");
+const promises_1 = require("fs/promises");
+const micromatch_1 = require("micromatch");
+const path_1 = require("path");
+const path_2 = require("path");
+const readline_1 = require("readline");
+exports.localRepositoryName = "local";
+exports.localRepositoryDefinition = {
+    type: "object",
+    required: ["outPath"],
+    additionalProperties: false,
+    properties: {
+        outPath: { type: "string" },
+        compress: { type: "boolean" },
+    },
+};
+exports.localPackageRepositoryDefinition = {
+    type: "object",
+    additionalProperties: false,
+    properties: {
+        compress: {
+            anyOf: [
+                {
+                    type: "boolean",
+                },
+                {
+                    type: "object",
+                    additionalProperties: false,
+                    properties: {
+                        packs: {
+                            type: "array",
+                            items: {
+                                type: "object",
+                                additionalProperties: false,
+                                required: ["include"],
+                                properties: {
+                                    include: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+                                    exclude: (0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.stringListUtil),
+                                    onePackByResult: { type: "boolean" },
+                                },
+                            },
+                        },
+                    },
+                },
+            ],
+        },
+    },
+};
+class LocalRepository extends RepositoryAbstract_1.RepositoryAbstract {
+    static buildSnapshotName(data) {
+        const date = data.snapshotDate.replace(/:/g, "-");
+        const pkgName = encodeURIComponent(data.packageName).replace(/%40/g, "@");
+        const snapshotShortId = data.snapshotId.slice(0, 8);
+        return `${date}_${pkgName}_${snapshotShortId}`;
+    }
+    static parseSnapshotName(name) {
+        if (!name.endsWith(".json"))
+            return null;
+        name = name.replace(/\.json$/, "");
+        const nameParts = name.split("_");
+        if (nameParts.length !== 3)
+            return null;
+        let [snapshotDate, packageName, snapshotShortId] = nameParts;
+        const [date, time] = snapshotDate.split("T");
+        snapshotDate = `${date}T${time.replace(/-/g, ":")}`;
+        packageName = decodeURIComponent(packageName);
+        return { snapshotDate, packageName, snapshotShortId, sourcePath: name };
+    }
+    buildMetaPath(snapshotName, packageName) {
+        return (0, path_1.join)(this.config.outPath, snapshotName, packageName) + ".meta.json";
+    }
+    static async parseMetaData(path) {
+        const contents = await (0, promises_1.readFile)(path);
+        return JSON.parse(contents.toString());
+    }
+    static stringifyMetaData(data) {
+        return JSON.stringify(data);
+    }
+    onGetSource() {
+        return this.config.outPath;
+    }
+    async onInit(data) {
+        await (0, fs_util_1.mkdirIfNotExists)(this.config.outPath);
+    }
+    async onPrune(data) {
+        const snapshotName = LocalRepository.buildSnapshotName({
+            snapshotId: data.snapshot.id,
+            snapshotDate: data.snapshot.date,
+            packageName: data.snapshot.packageName,
+        });
+        const snapshotPath = (0, path_1.join)(this.config.outPath, snapshotName);
+        const metaPath = `${snapshotPath}.json`;
+        if (data.options.verbose)
+            (0, cli_util_1.logExec)(`Deleting ${snapshotPath}`);
+        if (await (0, fs_util_1.checkDir)(snapshotPath))
+            await (0, fs_extra_1.rm)(snapshotPath, {
+                recursive: true,
+            });
+        if (await (0, fs_util_1.checkFile)(metaPath))
+            await (0, fs_extra_1.rm)(metaPath);
+    }
+    async onSnapshots(data) {
+        if (!(await (0, fs_util_1.checkDir)(this.config.outPath)))
+            throw new Error(`Repository (${this.repository.name}) out path does not exist: ${this.config.outPath}`);
+        const snapshotNames = await (0, promises_1.readdir)(this.config.outPath);
+        const snapshots = [];
+        const packagePatterns = (0, string_util_1.makePathPatterns)(data.options.packageNames);
+        for (const snapshotName of snapshotNames) {
+            const snapshotNameData = LocalRepository.parseSnapshotName(snapshotName);
+            if (!snapshotNameData)
+                continue;
+            if (packagePatterns &&
+                !(0, micromatch_1.isMatch)(snapshotNameData.packageName, packagePatterns))
+                continue;
+            if (data.options.ids &&
+                !data.options.ids.some((id) => snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8))))
+                continue;
+            const metaPath = (0, path_1.join)(this.config.outPath, snapshotName);
+            const meta = await LocalRepository.parseMetaData(metaPath);
+            if (data.options.ids &&
+                !data.options.ids.some((id) => meta.id.startsWith(id)))
+                continue;
+            if (data.options.tags &&
+                !data.options.tags.some((value) => data.options.tags?.includes(value)))
+                continue;
+            snapshots.push({
+                originalId: snapshotName,
+                id: meta.id,
+                date: meta.date,
+                packageName: meta.package,
+                tags: meta.tags,
+            });
+        }
+        return snapshots;
+    }
+    normalizeCompressConfig(packageConfig) {
+        let compress = packageConfig?.compress ?? this.config.compress;
+        if (compress === true) {
+            return {
+                packs: [
+                    {
+                        include: ["**"],
+                    },
+                ],
+            };
+        }
+        else if (!compress) {
+            return undefined;
+        }
+        return compress;
+    }
+    async onBackup(data) {
+        const snapshotName = LocalRepository.buildSnapshotName({
+            snapshotId: data.snapshot.id,
+            snapshotDate: data.snapshot.date,
+            packageName: data.package.name,
+        });
+        const outPath = (0, path_1.join)(this.config.outPath, snapshotName);
+        const createdPaths = [];
+        const pkg = data.package;
+        await (0, promises_1.mkdir)(outPath, {
+            recursive: true,
+        });
+        const sourcePath = data.targetPath ?? pkg.path;
+        (0, assert_1.ok)(sourcePath);
+        const compress = this.normalizeCompressConfig(data.packageConfig);
+        const include = await (0, paths_util_1.parsePaths)(pkg.include ?? ["**"], {
+            cwd: sourcePath,
+            verbose: data.options.verbose,
+        });
+        const exclude = pkg.exclude
+            ? await (0, paths_util_1.parsePaths)(pkg.exclude, {
+                cwd: sourcePath,
+                verbose: data.options.verbose,
+            })
+            : undefined;
+        const stream = fast_glob_1.default.stream(include, {
+            cwd: sourcePath,
+            ignore: exclude,
+            dot: true,
+            onlyFiles: compress ? false : true,
+            markDirectories: true,
+        });
+        if (data.options.verbose)
+            (0, cli_util_1.logExec)(`Writing paths lists`);
+        const pathLists = await (0, fs_util_1.writePathLists)({
+            paths: stream,
+            packs: compress?.packs,
+        });
+        let currentFiles = 0;
+        if (compress?.packs) {
+            let packIndex = 0;
+            for (const packsPath of pathLists.includedPackPaths) {
+                const pack = compress.packs[packIndex];
+                if (pack.onePackByResult) {
+                    const reader = (0, readline_1.createInterface)({
+                        input: (0, fs_1.createReadStream)(packsPath),
+                    });
+                    let multipleIndex = 0;
+                    for await (let packPath of reader) {
+                        if (packPath.endsWith("/"))
+                            packPath = packPath.slice(0, -1);
+                        const target = (0, path_1.join)(outPath, `.${packIndex}-${multipleIndex++}-${encodeURIComponent(packPath.replace(/[\\/]/g, "-")).slice(0, 255)}.dd.zip`);
+                        const stats = await (0, zip_util_1.zip)({
+                            path: pkg.path,
+                            output: target,
+                            filter: [{ patterns: [packPath] }],
+                            excludeList: pathLists.excludedPackPaths[packIndex],
+                            verbose: data.options.verbose,
+                            onStream: async (stream) => await data.onProgress({
+                                total: pathLists.total.all,
+                                current: currentFiles + stream.data.files,
+                                percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
+                                step: stream.type === "progress" ? stream.data.path : "",
+                                stepPercent: stream.type === "progress" ? stream.data.progress : null,
+                            }),
+                        });
+                        currentFiles += stats.files;
+                    }
+                }
+                else {
+                    const target = (0, path_1.join)(outPath, `.${packIndex}.dd.zip`);
+                    const stats = await (0, zip_util_1.zip)({
+                        path: sourcePath,
+                        output: target,
+                        includeList: packsPath,
+                        excludeList: pathLists.excludedPackPaths[packIndex],
+                        verbose: data.options.verbose,
+                        onStream: async (stream) => await data.onProgress({
+                            total: pathLists.total.all,
+                            current: currentFiles + stream.data.files,
+                            percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles + stream.data.files),
+                            step: stream.type === "progress" ? stream.data.path : "",
+                        }),
+                    });
+                    currentFiles += stats.files;
+                }
+                packIndex++;
+            }
+        }
+        if (data.options.verbose)
+            (0, cli_util_1.logExec)(`Copying files to ${outPath}`);
+        const reader = (0, readline_1.createInterface)({
+            input: (0, fs_1.createReadStream)(pathLists.path),
+        });
+        for await (const entry of reader) {
+            const source = (0, path_1.join)(sourcePath, entry);
+            const target = (0, path_1.join)(outPath, entry);
+            if (entry.endsWith("/")) {
+                await (0, promises_1.mkdir)(target, {
+                    recursive: true,
+                });
+            }
+            else {
+                currentFiles++;
+                await data.onProgress({
+                    total: pathLists.total.all,
+                    current: currentFiles,
+                    percent: (0, math_util_1.progressPercent)(pathLists.total.all, currentFiles),
+                    step: entry,
+                });
+                const dir = (0, path_2.dirname)(target);
+                if (!createdPaths.includes(dir)) {
+                    await (0, promises_1.mkdir)(dir, {
+                        recursive: true,
+                    });
+                    createdPaths.push(dir);
+                }
+                await (0, promises_1.copyFile)(source, target);
+            }
+        }
+        const metaPath = `${outPath}.json`;
+        const nodePkg = (0, fs_util_1.parsePackageFile)();
+        const meta = {
+            id: data.snapshot.id,
+            date: data.snapshot.date,
+            tags: data.options.tags ?? [],
+            package: data.package.name,
+            version: nodePkg.version,
+        };
+        if (data.options.verbose)
+            (0, cli_util_1.logExec)(`Writing metadata into ${metaPath}`);
+        await (0, promises_1.writeFile)(metaPath, LocalRepository.stringifyMetaData(meta));
+    }
+    async onRestore(data) {
+        const restorePath = data.targetPath ?? data.package.restorePath;
+        (0, assert_1.ok)(restorePath);
+        const [snapshot] = await this.onSnapshots({
+            options: {
+                ids: [data.options.snapshotId],
+            },
+        });
+        if (!snapshot)
+            throw new AppError_1.AppError("Snapshot not found");
+        const snapshotName = LocalRepository.buildSnapshotName({
+            snapshotId: data.snapshot.id,
+            snapshotDate: data.snapshot.date,
+            packageName: data.package.name,
+        });
+        const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
+        let totalFiles = 0;
+        let currentFiles = -1;
+        await (0, fs_util_1.forEachFile)(sourcePath, () => {
+            totalFiles++;
+        }, true);
+        if (data.options.verbose)
+            (0, cli_util_1.logExec)(`Copying files to ${restorePath}`);
+        await (0, fs_extra_1.copy)(sourcePath, restorePath, {
+            filter: async (path) => {
+                const [firstFolder] = (0, path_1.relative)(sourcePath, path).split(path_1.sep);
+                const isZipFile = firstFolder.startsWith(".") && firstFolder.endsWith(".dd.zip");
+                await data.onProgress({
+                    total: totalFiles,
+                    current: Math.max(currentFiles, 0),
+                    percent: (0, math_util_1.progressPercent)(totalFiles, Math.max(currentFiles, 0)),
+                    step: (0, path_1.relative)(sourcePath, path),
+                });
+                if (isZipFile) {
+                    await (0, zip_util_1.unzip)({
+                        input: path,
+                        output: restorePath,
+                        verbose: data.options.verbose,
+                        onStream: async (stream) => await data.onProgress({
+                            total: totalFiles,
+                            current: currentFiles + 1,
+                            percent: (0, math_util_1.progressPercent)(totalFiles, currentFiles + 1),
+                            step: stream.type === "progress"
+                                ? `Extracting ${stream.data.path}`
+                                : "",
+                        }),
+                    });
+                }
+                currentFiles++;
+                return isZipFile ? false : true;
+            },
+        });
+    }
+}
+exports.LocalRepository = LocalRepository;
+LocalRepository.zipBasenameTpl = `.*.dd.zip`;

package/lib/Repository/RepositoryAbstract.d.ts

@@ -0,0 +1,77 @@
+import type { BackupActionOptionsType } from "../Action/BackupAction";
+import type { InitActionOptionsType } from "../Action/InitAction";
+import type { RestoreActionOptionsType } from "../Action/RestoreAction";
+import type { SnapshotExtendedType, SnapshotsActionOptionsType } from "../Action/SnapshotsAction";
+import type { PackageConfigType } from "../Config/PackageConfig";
+import type { RepositoryConfigType } from "../Config/RepositoryConfig";
+export declare type SnapshotType = {
+    id: string;
+    date: string;
+};
+export declare type SnapshotResultType = SnapshotType & {
+    originalId: string;
+    packageName: string;
+    tags: string[];
+};
+export declare type ProgressDataType = {
+    total: number;
+    current: number;
+    percent: number;
+    step: string;
+    stepPercent?: number | null;
+};
+export declare type InitDataType = {
+    options: InitActionOptionsType;
+};
+export declare type SnapshotsDataType = {
+    options: Pick<SnapshotsActionOptionsType, "ids" | "packageNames" | "verbose" | "tags">;
+};
+export declare type BackupDataType<TPackageConfig> = {
+    options: BackupActionOptionsType;
+    snapshot: SnapshotType;
+    package: PackageConfigType;
+    targetPath: string | undefined;
+    packageConfig: TPackageConfig | undefined;
+    onProgress: (data: ProgressDataType) => Promise<void>;
+};
+export declare type RestoreDataType<TPackageConfig> = {
+    options: RestoreActionOptionsType;
+    snapshot: SnapshotType;
+    package: PackageConfigType;
+    targetPath: string | undefined;
+    packageConfig: TPackageConfig;
+    onProgress: (data: ProgressDataType) => Promise<void>;
+};
+export declare type PruneDataType = {
+    snapshot: SnapshotExtendedType;
+    options: {
+        verbose?: boolean;
+    };
+};
+export declare enum SnapshotTagEnum {
+    ID = "id",
+    SHORT_ID = "shortId",
+    DATE = "date",
+    PACKAGE = "package",
+    TAGS = "tags",
+    VERSION = "version"
+}
+export declare type SnapshotTagObjectType = {
+    [SnapshotTagEnum.ID]: string;
+    [SnapshotTagEnum.SHORT_ID]: string;
+    [SnapshotTagEnum.DATE]: string;
+    [SnapshotTagEnum.PACKAGE]: string;
+    [SnapshotTagEnum.TAGS]: string[];
+    [SnapshotTagEnum.VERSION]: string;
+};
+export declare abstract class RepositoryAbstract<TConfig> {
+    readonly repository: RepositoryConfigType;
+    readonly config: TConfig;
+    constructor(repository: RepositoryConfigType);
+    abstract onGetSource(): string;
+    abstract onInit(data: InitDataType): Promise<void>;
+    abstract onPrune(data: PruneDataType): Promise<void>;
+    abstract onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
+    abstract onBackup(data: BackupDataType<unknown>): Promise<void>;
+    abstract onRestore(data: RestoreDataType<unknown>): Promise<void>;
+}

package/lib/Repository/RepositoryAbstract.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RepositoryAbstract = exports.SnapshotTagEnum = void 0;
+var SnapshotTagEnum;
+(function (SnapshotTagEnum) {
+    SnapshotTagEnum["ID"] = "id";
+    SnapshotTagEnum["SHORT_ID"] = "shortId";
+    SnapshotTagEnum["DATE"] = "date";
+    SnapshotTagEnum["PACKAGE"] = "package";
+    SnapshotTagEnum["TAGS"] = "tags";
+    SnapshotTagEnum["VERSION"] = "version";
+})(SnapshotTagEnum = exports.SnapshotTagEnum || (exports.SnapshotTagEnum = {}));
+class RepositoryAbstract {
+    constructor(repository) {
+        this.repository = repository;
+        this.config = repository.config;
+    }
+}
+exports.RepositoryAbstract = RepositoryAbstract;

package/lib/Repository/ResticRepository.d.ts

@@ -0,0 +1,36 @@
+import { RepositoryType } from "../util/ResticUtil";
+import { RepositoryAbstract, BackupDataType, InitDataType, RestoreDataType, SnapshotsDataType, SnapshotResultType, SnapshotTagObjectType, SnapshotTagEnum, PruneDataType } from "./RepositoryAbstract";
+import { JSONSchema7 } from "json-schema";
+export declare type ResticRepositoryConfigType = {
+    passwordFile: string;
+    repository: RepositoryType;
+};
+export declare type ResticPackageRepositoryConfigType = {};
+export declare const resticRepositoryName = "restic";
+export declare const resticRepositoryDefinition: JSONSchema7;
+export declare const resticPackageRepositoryDefinition: JSONSchema7;
+export declare class ResticRepository extends RepositoryAbstract<ResticRepositoryConfigType> {
+    static refPrefix: string;
+    protected env: {
+        RESTIC_PASSWORD_FILE: string;
+        RESTIC_REPOSITORY: string;
+    };
+    buildEnv(): Promise<{
+        RESTIC_PASSWORD_FILE: string;
+        RESTIC_REPOSITORY: string;
+    }>;
+    static buildSnapshotTag(name: SnapshotTagEnum, value: string): string;
+    static parseSnapshotTag(tag: string): {
+        name: SnapshotTagEnum;
+        value: string;
+    } | null;
+    static parseSnapshotTags(tags: string[]): Required<Partial<SnapshotTagObjectType> & {
+        tags: string[];
+    }>;
+    onGetSource(): string;
+    onInit(data: InitDataType): Promise<void>;
+    onSnapshots(data: SnapshotsDataType): Promise<SnapshotResultType[]>;
+    onPrune(data: PruneDataType): Promise<void>;
+    onBackup(data: BackupDataType<ResticPackageRepositoryConfigType>): Promise<void>;
+    onRestore(data: RestoreDataType<ResticPackageRepositoryConfigType>): Promise<void>;
+}