@datatruck/restic 0.0.1 → 0.0.2

package/lib/actions/copy.js

@@ -1,129 +1,119 @@
+import { createRunner } from "../utils/async.js";
 import { checkDiskSpace } from "../utils/fs.js";
-import { Ntfy } from "../utils/ntfy.js";
+import { Action } from "./base.js";
+import { Prune } from "./prune.js";
 import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
 import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
 import { formatBytes } from "@datatruck/cli/utils/bytes.js";
-import { duration } from "@datatruck/cli/utils/date.js";
 import { isLocalDir } from "@datatruck/cli/utils/fs.js";
 import { Restic } from "@datatruck/cli/utils/restic.js";
-export class Copy {
-    config;
-    global;
-    ntfy;
-    verbose;
-    constructor(config, global) {
-        this.config = config;
-        this.global = global;
-        this.verbose = this.global?.verbose ?? this.config.verbose;
-        this.ntfy = new Ntfy({
-            token: this.config.ntfyToken,
-            titlePrefix: `[${this.config.hostname}] `,
+export class Copy extends Action {
+    initializedRepos = new Set();
+    async findSnapshots(name, packages) {
+        const repo = this.cm.findRepository(name);
+        const restic = new Restic({
+            log: this.verbose,
+            env: {
+                RESTIC_REPOSITORY: repo.uri,
+                RESTIC_PASSWORD: repo.password,
+            },
         });
+        const snapshots = (await restic.snapshots({
+            latest: 1,
+            group: ["path"],
+            tags: packages
+                ? packages.map((name) => ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, name))
+                : undefined,
+        }));
+        return snapshots.flatMap((s) => s.snapshots);
     }
-    findRepo(name) {
-        const repo = this.config.repositories.find((repo) => repo.name === name);
-        if (!repo)
-            throw new Error(`Repository '${name}' not found`);
-        return repo;
+    findPackageTag(inTags) {
+        const tags = inTags
+            .map((t) => ResticRepository.parseSnapshotTag(t))
+            .filter((t) => !!t);
+        return tags.find((t) => t.name === SnapshotTagEnum.PACKAGE);
+    }
+    async runSingle(options) {
+        const { snapshot } = options;
+        const targetRepo = this.cm.findRepository(options.target);
+        const sourceRepo = this.cm.findRepository(options.source);
+        const pkgTag = this.findPackageTag(snapshot.tags);
+        const packageName = pkgTag?.value;
+        const targetPath = isLocalDir(targetRepo.uri) ? targetRepo.uri : undefined;
+        const target = new Restic({
+            log: this.verbose,
+            env: {
+                RESTIC_REPOSITORY: targetRepo.uri,
+                RESTIC_PASSWORD: targetRepo.password,
+                ["GODEBUG"]: "http2client=0",
+            },
+        });
+        let space;
+        await createRunner(async () => {
+            if (!this.initializedRepos.has(targetRepo.name)) {
+                await target.tryInit();
+                this.initializedRepos.add(targetRepo.name);
+            }
+            space = await checkDiskSpace({
+                minFreeSpace: this.config.minFreeSpace,
+                targetPath,
+                rutine: () => target.copy({
+                    ids: [snapshot.id],
+                    fromRepo: sourceRepo.uri,
+                    fromRepoPassword: sourceRepo.password,
+                }),
+            });
+        }).start(async (data) => {
+            await this.ntfy.send(`Copy`, {
+                Id: snapshot.short_id,
+                Source: sourceRepo.name,
+                Target: targetRepo.name,
+                Package: packageName,
+                ...(space !== undefined && {
+                    Size: `${formatBytes(space.size)} (${formatBytes(space.diff, true)})`,
+                }),
+                Duration: data.duration,
+                Error: data.error?.message,
+            }, data.error);
+        });
+        return space?.diff ?? 0;
     }
     async run(options) {
-        const now = Date.now();
         let globalDiffSize;
-        let error;
-        try {
+        await createRunner(async () => {
+            const [source] = this.cm.filterRepositories([options.source]);
+            const targets = this.cm.filterRepositories(options.targets);
             await this.ntfy.send(`Copy start`, {
-                "- Source": options.source,
-                "- Targets": options.targets.join(", "),
-            });
-            const sourceRepo = this.findRepo(options.source);
-            const source = new Restic({
-                log: this.verbose,
-                env: {
-                    RESTIC_REPOSITORY: sourceRepo.uri,
-                    RESTIC_PASSWORD: sourceRepo.password,
-                },
+                Source: source.name,
+                Targets: targets.map((t) => t.name).join(", "),
             });
-            const targetRepos = options.targets.map((target) => this.findRepo(target));
-            if (!targetRepos.length)
-                throw new Error(`No target repositories specified`);
-            const inSnapshots = (await source.snapshots({
-                latest: 1,
-                group: ["path"],
-                tags: options.packages
-                    ? options.packages.map((name) => ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, name))
-                    : undefined,
-            }));
-            const snapshots = inSnapshots.flatMap((s) => s.snapshots);
-            for (const targetRepo of targetRepos) {
-                const target = new Restic({
-                    log: this.verbose,
-                    env: {
-                        RESTIC_REPOSITORY: targetRepo.uri,
-                        RESTIC_PASSWORD: targetRepo.password,
-                        ["GODEBUG"]: "http2client=0",
-                    },
-                });
-                const exists = await target.checkRepository();
-                if (!exists && isLocalDir(targetRepo.uri))
-                    await target.exec(["init"]);
+            const snapshots = await this.findSnapshots(source.name, options.packages);
+            for (const target of targets) {
                 for (const snapshot of snapshots) {
-                    const tags = snapshot.tags
-                        .map((t) => ResticRepository.parseSnapshotTag(t))
-                        .filter((t) => !!t);
-                    const pkgTag = tags.find((t) => t.name === SnapshotTagEnum.PACKAGE);
-                    const packageName = pkgTag?.value;
-                    const now = Date.now();
-                    let copyError;
-                    let diffSize;
-                    const targetPath = isLocalDir(targetRepo.uri)
-                        ? targetRepo.uri
-                        : undefined;
-                    try {
-                        diffSize = await checkDiskSpace({
-                            minFreeSpace: this.config.minFreeSpace,
-                            targetPath,
-                            rutine: () => target.copy({
-                                ids: [snapshot.id],
-                                fromRepo: sourceRepo.uri,
-                                fromRepoPassword: sourceRepo.password,
-                            }),
-                        });
-                        if (diffSize !== undefined) {
-                            globalDiffSize = (globalDiffSize ?? 0) + (diffSize ?? 0);
-                        }
-                    }
-                    catch (inError) {
-                        copyError = inError;
-                    }
-                    await this.ntfy.send(`Copy`, {
-                        "- Id": snapshot.short_id,
-                        "- Source": sourceRepo.name,
-                        "- Target": targetRepo.name,
-                        "- Package": packageName,
-                        ...(diffSize !== undefined && {
-                            "- Diff size": (diffSize > 0 ? "+" : "") + formatBytes(diffSize),
-                        }),
-                        "- Duration": duration(Date.now() - now),
-                        "- Error": copyError?.message,
-                    }, {
-                        priority: copyError ? "high" : "default",
-                        tags: [copyError ? "red_circle" : "green_circle"],
+                    const diffSize = await this.runSingle({
+                        snapshot,
+                        source: source.name,
+                        target: target.name,
                     });
+                    globalDiffSize = (globalDiffSize ?? 0) + diffSize;
                 }
             }
-        }
-        catch (inError) {
-            error = inError;
-        }
-        await this.ntfy.send(`Copy end`, {
-            ...(globalDiffSize !== undefined && {
-                "- Diff size": (globalDiffSize > 0 ? "+" : "") + formatBytes(globalDiffSize),
-            }),
-            "- Duration": duration(Date.now() - now),
-            "- Error": error?.message,
-        }, {
-            priority: error ? "high" : "default",
-            tags: [error ? "red_circle" : "green_circle"],
+            if (!options.targets.length)
+                throw new Error(`No target repositories specified`);
+        }).start(async (data) => {
+            await this.ntfy.send(`Copy end`, {
+                ...(globalDiffSize !== undefined && {
+                    "Diff size": formatBytes(globalDiffSize, true),
+                }),
+                Duration: data.duration,
+                Error: data.error?.message,
+            }, data.error);
+            if (options.prune) {
+                await new Prune(this.config, this.global).run({
+                    packages: options.packages,
+                    repositories: options.targets,
+                });
+            }
         });
     }
 }

package/lib/actions/init.d.ts

@@ -1,13 +1,8 @@
-import { Config, GlobalConfig } from "../config.js";
-import { Ntfy } from "../utils/ntfy.js";
+import { Action } from "./base.js";
 export type InitOptions = {
     repositories?: string[];
 };
-export declare class Init {
-    readonly config: Config;
-    readonly global?: GlobalConfig | undefined;
-    readonly ntfy: Ntfy;
-    protected verbose: boolean | undefined;
-    constructor(config: Config, global?: GlobalConfig | undefined);
+export declare class Init extends Action {
+    protected runSingle(name: string): Promise<void>;
     run(options: InitOptions): Promise<void>;
 }

package/lib/actions/init.js

@@ -1,56 +1,41 @@
-import { Ntfy } from "../utils/ntfy.js";
-import { duration } from "@datatruck/cli/utils/date.js";
-import { isLocalDir } from "@datatruck/cli/utils/fs.js";
+import { createRunner } from "../utils/async.js";
+import { Action } from "./base.js";
 import { Restic } from "@datatruck/cli/utils/restic.js";
-export class Init {
-    config;
-    global;
-    ntfy;
-    verbose;
-    constructor(config, global) {
-        this.config = config;
-        this.global = global;
-        this.verbose = this.global?.verbose ?? this.config.verbose;
-        this.ntfy = new Ntfy({
-            token: this.config.ntfyToken,
-            titlePrefix: `[${this.config.hostname}] `,
+export class Init extends Action {
+    async runSingle(name) {
+        let exists;
+        await createRunner(async () => {
+            const repo = this.cm.findRepository(name);
+            const restic = new Restic({
+                log: this.verbose,
+                env: {
+                    RESTIC_REPOSITORY: repo.uri,
+                    RESTIC_PASSWORD: repo.password,
+                },
+            });
+            exists = await restic.tryInit();
+        }).start(async (data) => {
+            await this.ntfy.send(`Init`, {
+                Repository: name,
+                Exists: exists ? "yes" : "no",
+                Duration: data.duration,
+                Error: data.error?.message,
+            }, data.error);
         });
     }
     async run(options) {
-        const now = Date.now();
-        await this.ntfy.send(`Init start`, {});
-        const repositories = options.repositories ?? this.config.repositories.map((r) => r.name);
-        for (const name of repositories) {
-            let error;
-            let exists;
-            try {
-                const repo = this.config.repositories.find((repo) => repo.name === name);
-                if (!repo)
-                    throw new Error(`Repository not found`);
-                const source = new Restic({
-                    log: this.verbose,
-                    env: {
-                        RESTIC_REPOSITORY: repo.uri,
-                        RESTIC_PASSWORD: repo.password,
-                    },
-                });
-                exists = await source.checkRepository();
-                if (!exists && isLocalDir(repo.uri))
-                    await source.exec(["init"]);
-            }
-            catch (inError) {
-                error = inError;
-            }
-            await this.ntfy.send(`Init`, {
-                "- Repository": name,
-                "- Exists": exists ? "yes" : "no",
-                "- Duration": duration(Date.now() - now),
-                "- Error": error?.message,
-            }, {
-                priority: error ? "high" : "default",
-                tags: [error ? "red_circle" : "green_circle"],
+        await createRunner(async () => {
+            const repositories = this.cm.filterRepositories(options.repositories);
+            await this.ntfy.send(`Init start`, {
+                Repositories: repositories.length,
             });
-        }
-        await this.ntfy.send(`Init end`, {});
+            for (const repo of repositories)
+                await this.runSingle(repo.name);
+        }).start(async (data) => {
+            await this.ntfy.send(`Init end`, {
+                Duration: data.duration,
+                Error: data.error?.message,
+            });
+        });
     }
 }

package/lib/actions/prune.d.ts

@@ -0,0 +1,10 @@
+import { PrunePolicy } from "../config.js";
+import { Action } from "./base.js";
+export type PruneOptions = {
+    packages?: string[];
+    repositories?: string[];
+};
+export declare class Prune extends Action {
+    protected runSingle(repoName: string, pkgName: string, policy: PrunePolicy): Promise<number>;
+    run(options: PruneOptions): Promise<void>;
+}

package/lib/actions/prune.js

@@ -0,0 +1,105 @@
+import { createRunner, safeRun } from "../utils/async.js";
+import { checkDiskSpace, fetchMultipleDiskStats } from "../utils/fs.js";
+import { Action } from "./base.js";
+import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
+import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
+import { formatBytes } from "@datatruck/cli/utils/bytes.js";
+import { isLocalDir } from "@datatruck/cli/utils/fs.js";
+import { progressPercent } from "@datatruck/cli/utils/math.js";
+import { Restic } from "@datatruck/cli/utils/restic.js";
+export class Prune extends Action {
+    async runSingle(repoName, pkgName, policy) {
+        let stats;
+        let space;
+        await createRunner(async () => {
+            const repo = this.cm.findRepository(repoName);
+            const pkg = this.cm.findPackage(pkgName);
+            const restic = new Restic({
+                log: this.verbose,
+                env: {
+                    RESTIC_REPOSITORY: repo.uri,
+                    RESTIC_PASSWORD: repo.password,
+                },
+            });
+            const targetPath = isLocalDir(repo.uri) ? repo.uri : undefined;
+            space = await checkDiskSpace({
+                minFreeSpace: this.config.minFreeSpace,
+                targetPath,
+                rutine: async () => {
+                    const results = await restic.forget({
+                        ...policy,
+                        json: true,
+                        prune: true,
+                        args: ["--group-by", ""],
+                        tag: [
+                            ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, pkg.name),
+                        ],
+                    });
+                    stats = {
+                        keep: results.at(0)?.keep?.length ?? 0,
+                        remove: results.at(0)?.remove?.length ?? 0,
+                    };
+                },
+            });
+        }).start(async (data) => {
+            await this.ntfy.send(`Prune`, {
+                Repository: repoName,
+                Package: pkgName,
+                ...(stats && {
+                    Snapshots: `${stats.keep}`,
+                    Removed: `${stats.remove}`,
+                }),
+                ...(space !== undefined && {
+                    Size: `${formatBytes(space.size)} (${formatBytes(space.diff, true)})`,
+                }),
+                Duration: data.duration,
+                Error: data.error?.message,
+            }, data.error);
+        });
+        return space?.diff ?? 0;
+    }
+    async run(options) {
+        let globalDiffSize;
+        let localRepositoryPaths = [];
+        await createRunner(async () => {
+            const repositories = this.cm.filterRepositories(options.repositories);
+            const packages = this.cm.filterPackages(options.packages);
+            await this.ntfy.send(`Prune start`, {
+                Repositories: repositories.length,
+                Packages: packages.length,
+            });
+            localRepositoryPaths = repositories
+                .filter((repo) => isLocalDir(repo.uri))
+                .map((repo) => repo.uri);
+            for (const repo of repositories) {
+                for (const pkg of packages) {
+                    const policy = pkg.prunePolicy ?? repo.prunePolicy ?? this.config.prunePolicy;
+                    if (policy) {
+                        const diffSize = await this.runSingle(repo.name, pkg.name, policy);
+                        if (diffSize !== undefined)
+                            globalDiffSize = (globalDiffSize ?? 0) + diffSize;
+                    }
+                }
+            }
+        }).start(async (data) => {
+            const diskStats = await safeRun(() => fetchMultipleDiskStats(localRepositoryPaths));
+            if (diskStats.error)
+                console.error(diskStats.error);
+            await this.ntfy.send(`Prune end`, {
+                Duration: data.duration,
+                ...(globalDiffSize !== undefined && {
+                    "Diff size": (globalDiffSize > 0 ? "+" : "") + formatBytes(globalDiffSize),
+                }),
+                Error: data.error?.message,
+                "": [
+                    !!diskStats.result?.length && { key: "Disk stats", value: "" },
+                    ...(diskStats.result?.map((p) => ({
+                        key: p.name,
+                        value: `${formatBytes(p.free)}/${formatBytes(p.total)} (${progressPercent(p.total, p.free)}%)`,
+                        level: 1,
+                    })) || []),
+                ],
+            });
+        });
+    }
+}

package/lib/bin.js

@@ -1,6 +1,7 @@
 import { Backup } from "./actions/backup.js";
 import { Copy } from "./actions/copy.js";
 import { Init } from "./actions/init.js";
+import { Prune } from "./actions/prune.js";
 import { parseConfigFile } from "./config.js";
 import { parseStringList } from "@datatruck/cli/utils/string.js";
 import { program } from "commander";
@@ -37,6 +38,7 @@ program
     .description("Run backup action")
     .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
     .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
+    .option("--prune", "Prune after backup")
     .action(async (options) => {
     const { config, globalOptions } = await load();
     const backup = new Backup(config, globalOptions);
@@ -49,9 +51,22 @@ program
     .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
     .requiredOption("-s, --source <name>", "Source repository name")
     .requiredOption("-t, --targets <names>", "Target repository names", (v) => parseStringList(v))
+    .option("--prune", "Prune after copy")
     .action(async (options) => {
     const { config, globalOptions } = await load();
     const copy = new Copy(config, globalOptions);
     await copy.run(options);
 });
+program
+    .command("prune")
+    .alias("p")
+    .description("Run prune action")
+    .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
+    .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
+    .option("--prune", "Prune after copy")
+    .action(async (options) => {
+    const { config, globalOptions } = await load();
+    const prune = new Prune(config, globalOptions);
+    await prune.run(options);
+});
 program.parse();

package/lib/config.d.ts

@@ -1,3 +1,4 @@
+import { MySQLDumpOptions } from "./utils/mysql.js";
 export type GlobalConfig = {
     config?: string;
     verbose?: boolean;
@@ -8,6 +9,7 @@ export type Config = {
     ntfyToken?: string;
     minFreeSpace?: string;
     verbose?: boolean;
+    prunePolicy?: PrunePolicy;
     tasks?: {
         type: "mysql-dump";
         packages: string[];
@@ -20,26 +22,60 @@ export type Config = {
                 path: string | false;
             }[] | string;
             concurrency?: number;
-            connection: {
-                hostname: string;
-                username: string;
-                password: string;
-                database?: string;
-            };
+            connection: MySQLDumpOptions["connection"];
         };
     }[];
     packages: {
         name: string;
         path: string;
         exclude?: string[];
+        prunePolicy?: PrunePolicy;
     }[];
     repositories: {
         name: string;
         password: string;
         uri: string;
+        prunePolicy?: PrunePolicy;
     }[];
 };
+export type PrunePolicy = {
+    keepMinutely?: number;
+    keepDaily?: number;
+    keepHourly?: number;
+    keepLast?: number;
+    keepMonthly?: number;
+    keepWeekly?: number;
+    keepYearly?: number;
+};
 export declare function defineConfig(config: Config): Config;
 export declare function validateConfig(config: unknown): Promise<void>;
 export declare function readConfigSchemaFile(): Promise<unknown>;
 export declare function parseConfigFile(path?: string): Promise<Config>;
+export declare class ConfigManager {
+    readonly config: Config;
+    constructor(config: Config);
+    filterRepositories(filter: string[] | undefined): {
+        name: string;
+        password: string;
+        uri: string;
+        prunePolicy?: PrunePolicy;
+    }[];
+    filterPackages(filter: string[] | undefined): {
+        name: string;
+        path: string;
+        exclude?: string[];
+        prunePolicy?: PrunePolicy;
+    }[];
+    findRepository(name: string): {
+        name: string;
+        password: string;
+        uri: string;
+        prunePolicy?: PrunePolicy;
+    };
+    findPackage(name: string): {
+        name: string;
+        path: string;
+        exclude?: string[];
+        prunePolicy?: PrunePolicy;
+    };
+}

package/lib/config.js

@@ -1,4 +1,5 @@
 import { parseJSONFile } from "./utils/fs.js";
+import { match } from "@datatruck/cli/utils/string.js";
 import { Ajv } from "ajv";
 export function defineConfig(config) {
     return config;
@@ -20,3 +21,33 @@ export async function parseConfigFile(path = "datatruck.restic.json") {
     await validateConfig(config);
     return config;
 }
+export class ConfigManager {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    filterRepositories(filter) {
+        const repositories = this.config.repositories.filter((v) => filter ? match(v.name, filter) : true);
+        if (!repositories.length)
+            throw new Error(`No repositories found for filter: ${filter?.join(", ")}`);
+        return repositories;
+    }
+    filterPackages(filter) {
+        const packages = this.config.packages.filter((v) => filter ? match(v.name, filter) : true);
+        if (!packages.length)
+            throw new Error(`No packages found for filter: ${filter?.join(", ")}`);
+        return packages;
+    }
+    findRepository(name) {
+        const repo = this.config.repositories.find((repo) => repo.name === name);
+        if (!repo)
+            throw new Error(`Repository '${name}' not found`);
+        return repo;
+    }
+    findPackage(name) {
+        const pkg = this.config.packages.find((pkg) => pkg.name === name);
+        if (!pkg)
+            throw new Error(`Package '${name}' not found`);
+        return pkg;
+    }
+}

package/lib/index.d.ts

@@ -1,4 +1,5 @@
-export { Backup, type BackupRunOptions } from "./actions/backup.js";
-export { Copy, type CopyRunOptions } from "./actions/copy.js";
+export { Backup, type BackupOptions } from "./actions/backup.js";
+export { Copy, type CopyOptions } from "./actions/copy.js";
 export { Init, type InitOptions } from "./actions/init.js";
+export { Prune, type PruneOptions } from "./actions/prune.js";
 export { type Config, type GlobalConfig, parseConfigFile, defineConfig, validateConfig, } from "./config.js";

package/lib/index.js

@@ -1,4 +1,5 @@
 export { Backup } from "./actions/backup.js";
 export { Copy } from "./actions/copy.js";
 export { Init } from "./actions/init.js";
+export { Prune } from "./actions/prune.js";
 export { parseConfigFile, defineConfig, validateConfig, } from "./config.js";

package/lib/utils/async.d.ts

@@ -0,0 +1,10 @@
+export declare function createRunner(rutine: () => any): {
+    start: <R = any>(cb: (data: {
+        error: Error | undefined;
+        duration: string;
+    }) => Promise<R>) => Promise<R>;
+};
+export declare function safeRun<T>(cb: () => Promise<T>): Promise<{
+    error: Error | undefined;
+    result: T | undefined;
+}>;