@datatruck/restic 0.0.1 → 0.0.3

package/lib/actions/copy.js

@@ -1,129 +1,119 @@
+import { createRunner } from "../utils/async.js";
 import { checkDiskSpace } from "../utils/fs.js";
-import { Ntfy } from "../utils/ntfy.js";
+import { Action } from "./base.js";
+import { Prune } from "./prune.js";
 import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
 import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
 import { formatBytes } from "@datatruck/cli/utils/bytes.js";
-import { duration } from "@datatruck/cli/utils/date.js";
 import { isLocalDir } from "@datatruck/cli/utils/fs.js";
 import { Restic } from "@datatruck/cli/utils/restic.js";
-export class Copy {
-    config;
-    global;
-    ntfy;
-    verbose;
-    constructor(config, global) {
-        this.config = config;
-        this.global = global;
-        this.verbose = this.global?.verbose ?? this.config.verbose;
-        this.ntfy = new Ntfy({
-            token: this.config.ntfyToken,
-            titlePrefix: `[${this.config.hostname}] `,
+export class Copy extends Action {
+    initializedRepos = new Set();
+    async findSnapshots(name, packages) {
+        const repo = this.cm.findRepository(name);
+        const restic = new Restic({
+            log: this.verbose,
+            env: {
+                RESTIC_REPOSITORY: repo.uri,
+                RESTIC_PASSWORD: repo.password,
+            },
         });
+        const snapshots = (await restic.snapshots({
+            latest: 1,
+            group: ["path"],
+            tags: packages
+                ? packages.map((name) => ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, name))
+                : undefined,
+        }));
+        return snapshots.flatMap((s) => s.snapshots);
     }
-    findRepo(name) {
-        const repo = this.config.repositories.find((repo) => repo.name === name);
-        if (!repo)
-            throw new Error(`Repository '${name}' not found`);
-        return repo;
+    findPackageTag(inTags) {
+        const tags = inTags
+            .map((t) => ResticRepository.parseSnapshotTag(t))
+            .filter((t) => !!t);
+        return tags.find((t) => t.name === SnapshotTagEnum.PACKAGE);
+    }
+    async runSingle(options) {
+        const { snapshot } = options;
+        const targetRepo = this.cm.findRepository(options.target);
+        const sourceRepo = this.cm.findRepository(options.source);
+        const pkgTag = this.findPackageTag(snapshot.tags);
+        const packageName = pkgTag?.value;
+        const targetPath = isLocalDir(targetRepo.uri) ? targetRepo.uri : undefined;
+        const target = new Restic({
+            log: this.verbose,
+            env: {
+                RESTIC_REPOSITORY: targetRepo.uri,
+                RESTIC_PASSWORD: targetRepo.password,
+                ["GODEBUG"]: "http2client=0",
+            },
+        });
+        let space;
+        await createRunner(async () => {
+            if (!this.initializedRepos.has(targetRepo.name)) {
+                await target.tryInit();
+                this.initializedRepos.add(targetRepo.name);
+            }
+            space = await checkDiskSpace({
+                minFreeSpace: this.config.minFreeSpace,
+                targetPath,
+                rutine: () => target.copy({
+                    ids: [snapshot.id],
+                    fromRepo: sourceRepo.uri,
+                    fromRepoPassword: sourceRepo.password,
+                }),
+            });
+        }).start(async (data) => {
+            await this.ntfy.send(`Copy`, {
+                Id: snapshot.short_id,
+                Source: sourceRepo.name,
+                Target: targetRepo.name,
+                Package: packageName,
+                ...(space !== undefined && {
+                    Size: `${formatBytes(space.size)} (${formatBytes(space.diff, true)})`,
+                }),
+                Duration: data.duration,
+                Error: data.error?.message,
+            }, data.error);
+        });
+        return space?.diff ?? 0;
     }
     async run(options) {
-        const now = Date.now();
         let globalDiffSize;
-        let error;
-        try {
+        await createRunner(async () => {
+            const [source] = this.cm.filterRepositories([options.source]);
+            const targets = this.cm.filterRepositories(options.targets);
             await this.ntfy.send(`Copy start`, {
-                "- Source": options.source,
-                "- Targets": options.targets.join(", "),
-            });
-            const sourceRepo = this.findRepo(options.source);
-            const source = new Restic({
-                log: this.verbose,
-                env: {
-                    RESTIC_REPOSITORY: sourceRepo.uri,
-                    RESTIC_PASSWORD: sourceRepo.password,
-                },
+                Source: source.name,
+                Targets: targets.map((t) => t.name).join(", "),
             });
-            const targetRepos = options.targets.map((target) => this.findRepo(target));
-            if (!targetRepos.length)
-                throw new Error(`No target repositories specified`);
-            const inSnapshots = (await source.snapshots({
-                latest: 1,
-                group: ["path"],
-                tags: options.packages
-                    ? options.packages.map((name) => ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, name))
-                    : undefined,
-            }));
-            const snapshots = inSnapshots.flatMap((s) => s.snapshots);
-            for (const targetRepo of targetRepos) {
-                const target = new Restic({
-                    log: this.verbose,
-                    env: {
-                        RESTIC_REPOSITORY: targetRepo.uri,
-                        RESTIC_PASSWORD: targetRepo.password,
-                        ["GODEBUG"]: "http2client=0",
-                    },
-                });
-                const exists = await target.checkRepository();
-                if (!exists && isLocalDir(targetRepo.uri))
-                    await target.exec(["init"]);
+            const snapshots = await this.findSnapshots(source.name, options.packages);
+            for (const target of targets) {
                 for (const snapshot of snapshots) {
-                    const tags = snapshot.tags
-                        .map((t) => ResticRepository.parseSnapshotTag(t))
-                        .filter((t) => !!t);
-                    const pkgTag = tags.find((t) => t.name === SnapshotTagEnum.PACKAGE);
-                    const packageName = pkgTag?.value;
-                    const now = Date.now();
-                    let copyError;
-                    let diffSize;
-                    const targetPath = isLocalDir(targetRepo.uri)
-                        ? targetRepo.uri
-                        : undefined;
-                    try {
-                        diffSize = await checkDiskSpace({
-                            minFreeSpace: this.config.minFreeSpace,
-                            targetPath,
-                            rutine: () => target.copy({
-                                ids: [snapshot.id],
-                                fromRepo: sourceRepo.uri,
-                                fromRepoPassword: sourceRepo.password,
-                            }),
-                        });
-                        if (diffSize !== undefined) {
-                            globalDiffSize = (globalDiffSize ?? 0) + (diffSize ?? 0);
-                        }
-                    }
-                    catch (inError) {
-                        copyError = inError;
-                    }
-                    await this.ntfy.send(`Copy`, {
-                        "- Id": snapshot.short_id,
-                        "- Source": sourceRepo.name,
-                        "- Target": targetRepo.name,
-                        "- Package": packageName,
-                        ...(diffSize !== undefined && {
-                            "- Diff size": (diffSize > 0 ? "+" : "") + formatBytes(diffSize),
-                        }),
-                        "- Duration": duration(Date.now() - now),
-                        "- Error": copyError?.message,
-                    }, {
-                        priority: copyError ? "high" : "default",
-                        tags: [copyError ? "red_circle" : "green_circle"],
+                    const diffSize = await this.runSingle({
+                        snapshot,
+                        source: source.name,
+                        target: target.name,
                     });
+                    globalDiffSize = (globalDiffSize ?? 0) + diffSize;
                 }
             }
-        }
-        catch (inError) {
-            error = inError;
-        }
-        await this.ntfy.send(`Copy end`, {
-            ...(globalDiffSize !== undefined && {
-                "- Diff size": (globalDiffSize > 0 ? "+" : "") + formatBytes(globalDiffSize),
-            }),
-            "- Duration": duration(Date.now() - now),
-            "- Error": error?.message,
-        }, {
-            priority: error ? "high" : "default",
-            tags: [error ? "red_circle" : "green_circle"],
+            if (!options.targets.length)
+                throw new Error(`No target repositories specified`);
+        }).start(async (data) => {
+            await this.ntfy.send(`Copy end`, {
+                ...(globalDiffSize !== undefined && {
+                    "Diff size": formatBytes(globalDiffSize, true),
+                }),
+                Duration: data.duration,
+                Error: data.error?.message,
+            }, data.error);
+            if (options.prune) {
+                await new Prune(this.config, this.global).run({
+                    packages: options.packages,
+                    repositories: options.targets,
+                });
+            }
         });
     }
 }

package/lib/actions/create.d.ts

@@ -0,0 +1,10 @@
+export type CreateOptions = {
+    cwd: string;
+    config: string;
+    force?: boolean;
+};
+export declare class Create {
+    constructor();
+    protected random(): string;
+    run(options: CreateOptions): Promise<void>;
+}

package/lib/actions/create.js

@@ -0,0 +1,49 @@
+import { formatBytes } from "@datatruck/cli/utils/bytes.js";
+import { fetchDiskStats } from "@datatruck/cli/utils/fs.js";
+import { existsSync } from "fs";
+import { writeFile } from "fs/promises";
+import { hostname } from "os";
+import { join, relative } from "path";
+export class Create {
+    constructor() { }
+    random() {
+        return crypto.randomUUID().replaceAll("-", "");
+    }
+    async run(options) {
+        const stats = await fetchDiskStats(options.cwd);
+        const minFreeSpaceBytes = (stats.total * 25) / 100;
+        const minFreeSpace = formatBytes(minFreeSpaceBytes);
+        const cwd = process.cwd();
+        const config = {
+            $schema: "https://unpkg.com/@datatruck/restic/config.schema.json",
+            hostname: hostname(),
+            minFreeSpace,
+            ntfyToken: this.random(),
+            packages: [
+                {
+                    name: "default",
+                    path: join(cwd, "data").replaceAll("\\", "/"),
+                },
+            ],
+            prunePolicy: {
+                keepDaily: 7,
+                keepMonthly: 12,
+                keepYearly: 5,
+            },
+            repositories: [
+                {
+                    name: "default",
+                    uri: join(cwd, "repo").replaceAll("\\", "/"),
+                    password: this.random(),
+                },
+            ],
+        };
+        const configPath = join(cwd, options.config);
+        if (existsSync(configPath) && !options.force)
+            throw new Error(`Config file already exists at path: ${configPath}`);
+        await writeFile(configPath, JSON.stringify(config, null, 2) + "\n");
+        const relativeConfigPath = relative(cwd, configPath);
+        console.log(`- Created config file at path: ${relativeConfigPath}`);
+        console.info(`- Show log remotly via ntfy: https://ntfy.sh/${config.ntfyToken}`);
+    }
+}

package/lib/actions/init.d.ts

@@ -1,13 +1,8 @@
-import { Config, GlobalConfig } from "../config.js";
-import { Ntfy } from "../utils/ntfy.js";
+import { Action } from "./base.js";
 export type InitOptions = {
     repositories?: string[];
 };
-export declare class Init {
-    readonly config: Config;
-    readonly global?: GlobalConfig | undefined;
-    readonly ntfy: Ntfy;
-    protected verbose: boolean | undefined;
-    constructor(config: Config, global?: GlobalConfig | undefined);
+export declare class Init extends Action {
+    protected runSingle(name: string): Promise<void>;
     run(options: InitOptions): Promise<void>;
 }

package/lib/actions/init.js

@@ -1,56 +1,41 @@
-import { Ntfy } from "../utils/ntfy.js";
-import { duration } from "@datatruck/cli/utils/date.js";
-import { isLocalDir } from "@datatruck/cli/utils/fs.js";
+import { createRunner } from "../utils/async.js";
+import { Action } from "./base.js";
 import { Restic } from "@datatruck/cli/utils/restic.js";
-export class Init {
-    config;
-    global;
-    ntfy;
-    verbose;
-    constructor(config, global) {
-        this.config = config;
-        this.global = global;
-        this.verbose = this.global?.verbose ?? this.config.verbose;
-        this.ntfy = new Ntfy({
-            token: this.config.ntfyToken,
-            titlePrefix: `[${this.config.hostname}] `,
+export class Init extends Action {
+    async runSingle(name) {
+        let exists;
+        await createRunner(async () => {
+            const repo = this.cm.findRepository(name);
+            const restic = new Restic({
+                log: this.verbose,
+                env: {
+                    RESTIC_REPOSITORY: repo.uri,
+                    RESTIC_PASSWORD: repo.password,
+                },
+            });
+            exists = await restic.tryInit();
+        }).start(async (data) => {
+            await this.ntfy.send(`Init`, {
+                Repository: name,
+                Exists: exists ? "yes" : "no",
+                Duration: data.duration,
+                Error: data.error?.message,
+            }, data.error);
         });
     }
     async run(options) {
-        const now = Date.now();
-        await this.ntfy.send(`Init start`, {});
-        const repositories = options.repositories ?? this.config.repositories.map((r) => r.name);
-        for (const name of repositories) {
-            let error;
-            let exists;
-            try {
-                const repo = this.config.repositories.find((repo) => repo.name === name);
-                if (!repo)
-                    throw new Error(`Repository not found`);
-                const source = new Restic({
-                    log: this.verbose,
-                    env: {
-                        RESTIC_REPOSITORY: repo.uri,
-                        RESTIC_PASSWORD: repo.password,
-                    },
-                });
-                exists = await source.checkRepository();
-                if (!exists && isLocalDir(repo.uri))
-                    await source.exec(["init"]);
-            }
-            catch (inError) {
-                error = inError;
-            }
-            await this.ntfy.send(`Init`, {
-                "- Repository": name,
-                "- Exists": exists ? "yes" : "no",
-                "- Duration": duration(Date.now() - now),
-                "- Error": error?.message,
-            }, {
-                priority: error ? "high" : "default",
-                tags: [error ? "red_circle" : "green_circle"],
+        await createRunner(async () => {
+            const repositories = this.cm.filterRepositories(options.repositories);
+            await this.ntfy.send(`Init start`, {
+                Repositories: repositories.length,
             });
-        }
-        await this.ntfy.send(`Init end`, {});
+            for (const repo of repositories)
+                await this.runSingle(repo.name);
+        }).start(async (data) => {
+            await this.ntfy.send(`Init end`, {
+                Duration: data.duration,
+                Error: data.error?.message,
+            });
+        });
     }
 }

package/lib/actions/prune.d.ts

@@ -0,0 +1,14 @@
+import { PrunePolicy } from "../config.js";
+import { Action } from "./base.js";
+export type PruneOptions = {
+    packages?: string[];
+    repositories?: string[];
+};
+export declare class Prune extends Action {
+    protected runSingle(repoName: string, pkgName: string, policy: PrunePolicy): Promise<{
+        diffSize: number;
+        removed: number | undefined;
+    }>;
+    protected fetchPackages(repoName: string): Promise<string[]>;
+    run(options: PruneOptions): Promise<void>;
+}

package/lib/actions/prune.js

@@ -0,0 +1,123 @@
+import { createRunner, safeRun } from "../utils/async.js";
+import { checkDiskSpace, fetchMultipleDiskStats } from "../utils/fs.js";
+import { Action } from "./base.js";
+import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
+import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
+import { formatBytes } from "@datatruck/cli/utils/bytes.js";
+import { isLocalDir } from "@datatruck/cli/utils/fs.js";
+import { progressPercent } from "@datatruck/cli/utils/math.js";
+import { match } from "@datatruck/cli/utils/string.js";
+export class Prune extends Action {
+    async runSingle(repoName, pkgName, policy) {
+        let stats;
+        let space;
+        let removed;
+        await createRunner(async () => {
+            const [restic, repo] = this.cm.createRestic(repoName, this.verbose);
+            const targetPath = isLocalDir(repo.uri) ? repo.uri : undefined;
+            space = await checkDiskSpace({
+                minFreeSpace: this.config.minFreeSpace,
+                targetPath,
+                rutine: async () => {
+                    const results = await restic.forget({
+                        ...policy,
+                        json: true,
+                        prune: true,
+                        args: ["--group-by", ""],
+                        tag: [
+                            ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, pkgName),
+                        ],
+                    });
+                    stats = {
+                        keep: results.at(0)?.keep?.length ?? 0,
+                        remove: results.at(0)?.remove?.length ?? 0,
+                    };
+                },
+            });
+        }).start(async (data) => {
+            if (stats)
+                removed = stats.remove;
+            await this.ntfy.send(`Prune`, {
+                Repository: repoName,
+                Package: pkgName,
+                ...(stats && {
+                    Keeped: `${stats.keep}`,
+                    Removed: `${stats.remove}`,
+                }),
+                ...(space !== undefined && {
+                    Size: `${formatBytes(space.size)} (${formatBytes(space.diff, true)})`,
+                }),
+                Duration: data.duration,
+                Error: data.error?.message,
+            }, data.error);
+        });
+        return {
+            diffSize: space?.diff ?? 0,
+            removed,
+        };
+    }
+    async fetchPackages(repoName) {
+        const [restic] = this.cm.createRestic(repoName, this.verbose);
+        if (!(await restic.checkRepository()))
+            return [];
+        const snapshots = await restic.snapshots({ json: true });
+        const packages = new Set(snapshots.map((s) => {
+            const tags = ResticRepository.parseSnapshotTags(s.tags ?? []);
+            return tags[SnapshotTagEnum.PACKAGE];
+        }));
+        return [...packages].filter((v) => typeof v === "string");
+    }
+    async run(options) {
+        let globalDiffSize;
+        let globalRemoved = 0;
+        let localRepositoryPaths = [];
+        await createRunner(async () => {
+            const repositories = this.cm.filterRepositories(options.repositories);
+            await this.ntfy.send(`Prune start`, {
+                Repositories: repositories.length,
+            });
+            localRepositoryPaths = repositories
+                .filter((repo) => isLocalDir(repo.uri))
+                .map((repo) => repo.uri);
+            let some = false;
+            for (const repo of repositories) {
+                const inPackageNames = await this.fetchPackages(repo.name);
+                const packageNames = inPackageNames.filter((pkgName) => options.packages ? match(pkgName, options.packages) : true);
+                for (const pkgName of packageNames) {
+                    const pkg = this.config.packages.find((pkg) => pkg.name === pkgName);
+                    const policy = pkg?.prunePolicy ?? repo.prunePolicy ?? this.config.prunePolicy;
+                    if (policy) {
+                        const { diffSize, removed } = await this.runSingle(repo.name, pkgName, policy);
+                        some = true;
+                        if (diffSize !== undefined)
+                            globalDiffSize = (globalDiffSize ?? 0) + diffSize;
+                        if (removed !== undefined)
+                            globalRemoved += removed;
+                    }
+                }
+            }
+            if (options.packages && !some)
+                throw new Error(`No packages found for filter: ${options.packages.join(", ")}`);
+        }).start(async (data) => {
+            const diskStats = await safeRun(() => fetchMultipleDiskStats(localRepositoryPaths));
+            if (diskStats.error)
+                console.error(diskStats.error);
+            await this.ntfy.send(`Prune end`, {
+                Duration: data.duration,
+                Removed: globalRemoved,
+                ...(globalDiffSize !== undefined && {
+                    "Diff size": (globalDiffSize > 0 ? "+" : "") + formatBytes(globalDiffSize),
+                }),
+                Error: data.error?.message,
+                "": [
+                    !!diskStats.result?.length && { key: "Disk stats", value: "" },
+                    ...(diskStats.result?.map((p) => ({
+                        key: p.name,
+                        value: `${formatBytes(p.free)}/${formatBytes(p.total)} (${progressPercent(p.total, p.free)}%)`,
+                        level: 1,
+                    })) || []),
+                ],
+            });
+        });
+    }
+}

package/lib/bin.js

@@ -1,57 +1,2 @@
-import { Backup } from "./actions/backup.js";
-import { Copy } from "./actions/copy.js";
-import { Init } from "./actions/init.js";
-import { parseConfigFile } from "./config.js";
-import { parseStringList } from "@datatruck/cli/utils/string.js";
-import { program } from "commander";
-import { resolve } from "path";
-function getGlobalOptions() {
-    const options = program.opts();
-    return {
-        ...options,
-        config: resolve(options.config),
-        verbose: process.env.DEBUG ? true : options.verbose,
-    };
-}
-async function load() {
-    const globalOptions = getGlobalOptions();
-    const config = await parseConfigFile(globalOptions.config);
-    return { globalOptions, config };
-}
-program
-    .option("-v, --verbose")
-    .option("-c, --config <path>", "Path to config file", "datatruck.restic.json");
-program
-    .command("init")
-    .alias("i")
-    .description("Run init action")
-    .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
-    .action(async (options) => {
-    const { config, globalOptions } = await load();
-    const init = new Init(config, globalOptions);
-    await init.run(options);
-});
-program
-    .command("backup")
-    .alias("b")
-    .description("Run backup action")
-    .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
-    .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
-    .action(async (options) => {
-    const { config, globalOptions } = await load();
-    const backup = new Backup(config, globalOptions);
-    await backup.run(options);
-});
-program
-    .command("copy")
-    .alias("c")
-    .description("Run copy action")
-    .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
-    .requiredOption("-s, --source <name>", "Source repository name")
-    .requiredOption("-t, --targets <names>", "Target repository names", (v) => parseStringList(v))
-    .action(async (options) => {
-    const { config, globalOptions } = await load();
-    const copy = new Copy(config, globalOptions);
-    await copy.run(options);
-});
-program.parse();
+import { createBin } from "./create-bin.js";
+createBin().parse();