@datatruck/restic 0.0.1 → 0.0.3

package/config.schema.json

@@ -16,6 +16,33 @@
     "verbose": {
       "type": "boolean"
     },
+    "prunePolicy": {
+      "type": "object",
+      "properties": {
+        "keepMinutely": {
+          "type": "number"
+        },
+        "keepDaily": {
+          "type": "number"
+        },
+        "keepHourly": {
+          "type": "number"
+        },
+        "keepLast": {
+          "type": "number"
+        },
+        "keepMonthly": {
+          "type": "number"
+        },
+        "keepWeekly": {
+          "type": "number"
+        },
+        "keepYearly": {
+          "type": "number"
+        }
+      },
+      "additionalProperties": false
+    },
     "tasks": {
       "type": "array",
       "items": {
@@ -84,27 +111,7 @@
                 "type": "number"
               },
               "connection": {
-                "type": "object",
-                "properties": {
-                  "hostname": {
-                    "type": "string"
-                  },
-                  "username": {
-                    "type": "string"
-                  },
-                  "password": {
-                    "type": "string"
-                  },
-                  "database": {
-                    "type": "string"
-                  }
-                },
-                "additionalProperties": false,
-                "required": [
-                  "hostname",
-                  "password",
-                  "username"
-                ]
+                "$ref": "#/definitions/Omit<MysqlCliOptions,\"verbose\">"
               }
             },
             "additionalProperties": false,
@@ -140,6 +147,33 @@
             "items": {
               "type": "string"
             }
+          },
+          "prunePolicy": {
+            "type": "object",
+            "properties": {
+              "keepMinutely": {
+                "type": "number"
+              },
+              "keepDaily": {
+                "type": "number"
+              },
+              "keepHourly": {
+                "type": "number"
+              },
+              "keepLast": {
+                "type": "number"
+              },
+              "keepMonthly": {
+                "type": "number"
+              },
+              "keepWeekly": {
+                "type": "number"
+              },
+              "keepYearly": {
+                "type": "number"
+              }
+            },
+            "additionalProperties": false
           }
         },
         "additionalProperties": false,
@@ -162,6 +196,33 @@
           },
           "uri": {
             "type": "string"
+          },
+          "prunePolicy": {
+            "type": "object",
+            "properties": {
+              "keepMinutely": {
+                "type": "number"
+              },
+              "keepDaily": {
+                "type": "number"
+              },
+              "keepHourly": {
+                "type": "number"
+              },
+              "keepLast": {
+                "type": "number"
+              },
+              "keepMonthly": {
+                "type": "number"
+              },
+              "keepWeekly": {
+                "type": "number"
+              },
+              "keepYearly": {
+                "type": "number"
+              }
+            },
+            "additionalProperties": false
           }
         },
         "additionalProperties": false,
@@ -178,5 +239,59 @@
     "packages",
     "repositories"
   ],
+  "definitions": {
+    "Omit<MysqlCliOptions,\"verbose\">": {
+      "type": "object",
+      "properties": {
+        "password": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "path": {
+                  "type": "string"
+                }
+              },
+              "additionalProperties": false,
+              "required": [
+                "path"
+              ]
+            },
+            {
+              "type": "string"
+            }
+          ]
+        },
+        "hostname": {
+          "type": "string"
+        },
+        "port": {
+          "type": "number"
+        },
+        "username": {
+          "type": "string"
+        },
+        "database": {
+          "type": "string"
+        },
+        "ssl": {
+          "type": "boolean"
+        },
+        "vars": {
+          "$ref": "#/definitions/Record<string,string|number>"
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "hostname",
+        "password",
+        "username"
+      ]
+    },
+    "Record<string,string|number>": {
+      "type": "object",
+      "additionalProperties": false
+    }
+  },
   "$schema": "http://json-schema.org/draft-07/schema#"
 }

package/lib/actions/backup.d.ts

@@ -1,40 +1,42 @@
-import { GlobalConfig, type Config } from "../config.js";
-import { MySQLDump } from "../utils/mysql.js";
-import { Ntfy } from "../utils/ntfy.js";
-import { CommonResticBackupTags, ResticBackup } from "../utils/restic-backup.js";
-export type BackupRunOptions = {
+import { Action } from "./base.js";
+export type BackupOptions = {
     packages?: string[];
     repositories?: string[];
+    prune?: boolean;
 };
-export declare class Backup {
-    readonly config: Config;
-    readonly global?: GlobalConfig | undefined;
-    readonly ntfy: Ntfy;
-    protected verbose: boolean | undefined;
-    readonly tags: CommonResticBackupTags;
-    constructor(config: Config, global?: GlobalConfig | undefined);
-    protected createInstances(packageNames: string[], repositoryNames?: string[]): {
-        repositories: ResticBackup[];
-        sqlDumps: (readonly [MySQLDump, {
-            type: "mysql-dump";
-            packages: string[];
-            name: string;
-            config: {
-                database: string;
-                out: {
-                    package?: string;
-                    tables: string[];
-                    path: string | false;
-                }[] | string;
-                concurrency?: number;
-                connection: {
-                    hostname: string;
-                    username: string;
-                    password: string;
-                    database?: string;
-                };
-            };
-        }])[];
-    };
-    run(options?: BackupRunOptions): Promise<void>;
+export type CommonResticBackupTags = {
+    id: string;
+    shortId: string;
+    hostname: string;
+    date: string;
+    vendor: string;
+    version: string;
+};
+export type ResticBackupTags = CommonResticBackupTags & {
+    package: string;
+    tags?: string[];
+};
+export declare class Backup extends Action {
+    protected runSingle(repoName: string, pkgName: string, tags: CommonResticBackupTags): Promise<{
+        error: Error | undefined;
+        files: number;
+        bytes: number;
+        diffSize: number | undefined;
+    }>;
+    protected filterTasks(packageNames: string[]): {
+        type: "mysql-dump";
+        packages: string[];
+        name: string;
+        config: {
+            database: string;
+            out: {
+                package?: string;
+                tables: string[];
+                path: string | false;
+            }[] | string;
+            concurrency?: number;
+            connection: import("../utils/mysql.js").MySQLDumpOptions["connection"];
+        };
+    }[] | undefined;
+    run(options?: BackupOptions): Promise<void>;
 }

package/lib/actions/backup.js

@@ -1,78 +1,126 @@
+import { createRunner, safeRun } from "../utils/async.js";
+import { checkDiskSpace, fetchMultipleDiskStats } from "../utils/fs.js";
 import { MySQLDump } from "../utils/mysql.js";
-import { Ntfy } from "../utils/ntfy.js";
-import { ResticBackup, } from "../utils/restic-backup.js";
+import { Action } from "./base.js";
+import { Prune } from "./prune.js";
+import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
+import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
 import { formatBytes } from "@datatruck/cli/utils/bytes.js";
-import { duration } from "@datatruck/cli/utils/date.js";
+import { isLocalDir } from "@datatruck/cli/utils/fs.js";
+import { progressPercent } from "@datatruck/cli/utils/math.js";
+import { Restic } from "@datatruck/cli/utils/restic.js";
 import { match } from "@datatruck/cli/utils/string.js";
 import { randomUUID } from "crypto";
 import { hostname } from "os";
-export class Backup {
-    config;
-    global;
-    ntfy;
-    verbose;
-    tags;
-    constructor(config, global) {
-        this.config = config;
-        this.global = global;
-        this.tags = {
-            id: randomUUID().replaceAll("-", ""),
-            get shortId() {
-                return this.id.slice(0, 8);
+export class Backup extends Action {
+    async runSingle(repoName, pkgName, tags) {
+        const repo = this.cm.findRepository(repoName);
+        const pkg = this.cm.findPackage(pkgName);
+        const restic = new Restic({
+            log: this.verbose,
+            env: {
+                RESTIC_PASSWORD: repo.password,
+                RESTIC_REPOSITORY: repo.uri,
             },
-            hostname: this.config.hostname ?? hostname(),
-            date: new Date().toISOString(),
-            vendor: "dtt-restic",
-            version: "1",
-        };
-        this.verbose = this.global?.verbose ?? this.config.verbose;
-        this.ntfy = new Ntfy({
-            token: this.config.ntfyToken,
-            titlePrefix: `[${this.tags.hostname}] `,
         });
-    }
-    createInstances(packageNames, repositoryNames) {
-        const repositories = this.config.repositories
-            .filter((repo) => !repositoryNames || repositoryNames.includes(repo.name))
-            .map((repo) => new ResticBackup({
-            tags: this.tags,
-            minFreeSpace: this.config.minFreeSpace,
-            name: repo.name,
-            connection: {
-                password: repo.password,
-                uri: repo.uri,
-            },
-        }, this.ntfy, this.verbose));
-        const sqlDumps = this.config.tasks
-            ?.filter((task) => task.type === "mysql-dump" &&
-            task.packages.some((name) => match(name, packageNames)))
-            .map((task) => [
-            new MySQLDump({
+        let space;
+        let snapshotsAmount;
+        let bytes = 0;
+        let files = 0;
+        return await createRunner(async () => {
+            await restic.tryInit();
+            const targetPath = isLocalDir(repo.uri) ? repo.uri : undefined;
+            space = await checkDiskSpace({
                 minFreeSpace: this.config.minFreeSpace,
-                verbose: this.verbose,
-                name: task.name,
-                connection: task.config.connection,
-                concurrency: task.config.concurrency,
-            }, this.ntfy),
-            task,
-        ]) ?? [];
-        return { repositories, sqlDumps };
+                minFreeSpacePath: targetPath ?? process.cwd(),
+                targetPath,
+                rutine: () => {
+                    const pkgTags = {
+                        ...tags,
+                        package: pkg.name,
+                        tags: [],
+                    };
+                    return restic.backup({
+                        tags: ResticRepository.createSnapshotTags(pkgTags),
+                        paths: [pkg.path],
+                        exclude: pkg.exclude,
+                        onStream(data) {
+                            if (data.message_type === "summary") {
+                                files = data.total_files_processed;
+                                bytes = data.total_bytes_processed;
+                            }
+                        },
+                    });
+                },
+            });
+            const snapshots = await restic.snapshots({
+                json: true,
+                tags: [
+                    ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, pkg.name),
+                ],
+            });
+            snapshotsAmount = snapshots.length;
+        }).start(async (data) => {
+            await this.ntfy.send("Backup", {
+                Repository: repo.name,
+                Package: pkg.name,
+                Size: formatBytes(bytes) +
+                    (space !== undefined ? ` (${formatBytes(space.diff, true)})` : ""),
+                Files: files,
+                Snapshots: snapshotsAmount,
+                Duration: data.duration,
+                Error: data.error?.message,
+            }, data.error);
+            return {
+                error: data.error,
+                files,
+                bytes,
+                diffSize: space?.diff,
+            };
+        });
+    }
+    filterTasks(packageNames) {
+        return this.config.tasks?.filter((task) => task.type === "mysql-dump" &&
+            task.packages.some((pattern) => match(pattern, packageNames)));
     }
     async run(options = {}) {
-        const now = Date.now();
-        const packages = this.config.packages.filter((pkg) => options.packages ? match(pkg.name, options.packages) : true);
-        const packageNames = packages.map((p) => p.name);
-        const { sqlDumps, repositories } = this.createInstances(packageNames, options.repositories);
-        let fatalError;
-        try {
+        const sqlDumps = [];
+        const backups = [];
+        let localRepositoryPaths = [];
+        await createRunner(async () => {
+            const repositories = this.cm.filterRepositories(options.repositories);
+            const packages = this.cm.filterPackages(options.packages);
+            const packageNames = packages.map((p) => p.name);
+            const tasks = this.filterTasks(packageNames);
             await this.ntfy.send(`Backup start`, {
-                "- Packages": packageNames.length,
+                Repositories: repositories.length,
+                Packages: packageNames.length,
+                Tasks: tasks?.length,
             });
-            if (!packages.length)
-                throw new Error("None package found");
-            for (const [sqlDump, task] of sqlDumps) {
-                await sqlDump.run([
-                    {
+            localRepositoryPaths = repositories
+                .filter((repo) => isLocalDir(repo.uri))
+                .map((repo) => repo.uri);
+            const tags = {
+                id: randomUUID().replaceAll("-", ""),
+                get shortId() {
+                    return this.id.slice(0, 8);
+                },
+                hostname: this.config.hostname ?? hostname(),
+                date: new Date().toISOString(),
+                vendor: "dtt-restic",
+                version: "1",
+            };
+            for (const task of tasks ?? []) {
+                if (task.type === "mysql-dump") {
+                    const mysqlDump = new MySQLDump({
+                        minFreeSpace: this.config.minFreeSpace,
+                        verbose: this.verbose,
+                        name: task.name,
+                        connection: task.config.connection,
+                        concurrency: task.config.concurrency,
+                    }, this.ntfy);
+                    sqlDumps.push(mysqlDump);
+                    await mysqlDump.run({
                         database: task.config.database,
                         name: task.name,
                         out: typeof task.config.out === "string"
@@ -83,60 +131,79 @@ export class Backup {
                                     ? o.path
                                     : false,
                             })),
-                    },
-                ]);
+                    });
+                }
             }
-            for (const backup of repositories)
-                await backup.run(packages);
-        }
-        catch (inError) {
-            fatalError = inError;
-        }
-        finally {
-            for (const [sqlDump] of sqlDumps)
-                await sqlDump.cleanup();
-            const backupSummary = {};
             for (const repo of repositories) {
-                for (const process of repo.processes) {
-                    if (!backupSummary[process.name])
-                        backupSummary[process.name] = {
-                            name: process.name,
-                            total: 0,
-                            success: 0,
-                            errors: 0,
-                            bytes: 0,
-                        };
-                    backupSummary[process.name].total++;
-                    backupSummary[process.name].bytes += process.stats.bytes;
-                    if (process.error) {
-                        backupSummary[process.name].errors++;
-                    }
-                    else {
-                        backupSummary[process.name].success++;
-                    }
+                for (const pkg of packages) {
+                    const result = await this.runSingle(repo.name, pkg.name, tags);
+                    backups.push({
+                        pkgName: pkg.name,
+                        ...result,
+                    });
                 }
             }
-            const backups = Object.values(backupSummary);
-            const sqlDumpProccesses = sqlDumps.flatMap(([sql]) => sql.processes);
-            const error = !!fatalError ||
-                sqlDumpProccesses.some((p) => p.error) ||
-                backups.some((p) => p.errors);
+        }).start(async (data) => {
+            for (const sqlDump of sqlDumps)
+                await sqlDump.cleanup();
+            const summary = backups.reduce((acc, p) => {
+                if (!acc[p.pkgName])
+                    acc[p.pkgName] = {
+                        name: p.pkgName,
+                        total: 0,
+                        success: 0,
+                        errors: 0,
+                        bytes: 0,
+                    };
+                const group = acc[p.pkgName];
+                group.total++;
+                group.bytes += p.bytes;
+                group[p.error ? "errors" : "success"]++;
+                return acc;
+            }, {});
+            const backupsValues = Object.values(summary);
+            const sqlDumpProcesses = sqlDumps.flatMap((sql) => sql.processes);
+            const error = !!data.error ||
+                sqlDumpProcesses.some((p) => p.error) ||
+                backupsValues.some((p) => p.errors);
             const size = [
-                ...sqlDumpProccesses.map((p) => p.stats.bytes),
-                ...backups.map((b) => b.bytes),
+                ...sqlDumpProcesses.map((p) => p.stats.bytes),
+                ...backupsValues.map((b) => b.bytes),
             ].reduce((r, b) => r + b, 0);
-            await this.ntfy.send(`Backup end`, [
-                `- Duration: ${duration(Date.now() - now)}`,
-                `- Size: ${formatBytes(size)}`,
-                !!fatalError && `- Fatal error: ${fatalError.message}`,
-                !!sqlDumpProccesses.length && "## SQL Dumps",
-                ...sqlDumpProccesses.map((p) => `- ${p.error ? `❌ ` : ""}${p.name}: ${formatBytes(p.stats.bytes)}`),
-                !!backups.length && "## Backups",
-                ...backups.map((p) => `- ${p.errors ? `❌ ` : ""}${p.name}: ${p.success}/${p.total}`),
-            ], {
-                priority: error ? "high" : "default",
-                tags: [error ? "red_circle" : "green_circle"],
-            });
-        }
+            const diskStats = await safeRun(() => fetchMultipleDiskStats(localRepositoryPaths));
+            if (diskStats.error)
+                console.error(diskStats.error);
+            await this.ntfy.send("Backup end", {
+                Duration: data.duration,
+                Size: formatBytes(size),
+                Error: data.error?.message,
+                "": [
+                    !!diskStats.result?.length && { key: "Disk stats", value: "" },
+                    ...(diskStats.result?.map((p) => ({
+                        key: p.name,
+                        value: `${formatBytes(p.free)}/${formatBytes(p.total)} (${progressPercent(p.total, p.free)}%)`,
+                        level: 1,
+                    })) || []),
+                    !!sqlDumpProcesses.length && { key: "SQL Dumps", value: "" },
+                    ...sqlDumpProcesses.map((p) => ({
+                        key: p.name,
+                        value: formatBytes(p.stats.bytes),
+                        level: 1,
+                    })),
+                    !!backupsValues.length && { key: "Packages", value: "" },
+                    ...backupsValues.map((p) => ({
+                        key: p.name,
+                        value: `${p.success}/${p.total}`,
+                        level: 1,
+                    })),
+                ],
+            }, error);
+            if (options.prune) {
+                await new Prune(this.config, this.global).run({
+                    packages: options.packages,
+                    repositories: options.repositories,
+                });
+            }
+        });
     }
 }

package/lib/actions/base.d.ts

@@ -0,0 +1,10 @@
+import { Config, ConfigManager, GlobalConfig } from "../config.js";
+import { Ntfy } from "../utils/ntfy.js";
+export declare class Action {
+    readonly config: Config;
+    readonly global?: GlobalConfig | undefined;
+    readonly ntfy: Ntfy;
+    protected verbose: boolean | undefined;
+    protected cm: ConfigManager;
+    constructor(config: Config, global?: GlobalConfig | undefined);
+}

package/lib/actions/base.js

@@ -0,0 +1,19 @@
+import { ConfigManager } from "../config.js";
+import { Ntfy } from "../utils/ntfy.js";
+export class Action {
+    config;
+    global;
+    ntfy;
+    verbose;
+    cm;
+    constructor(config, global) {
+        this.config = config;
+        this.global = global;
+        this.cm = new ConfigManager(this.config);
+        this.verbose = this.global?.verbose ?? this.config.verbose;
+        this.ntfy = new Ntfy({
+            token: this.config.ntfyToken,
+            titlePrefix: this.config.hostname,
+        });
+    }
+}

package/lib/actions/copy.d.ts

@@ -1,16 +1,22 @@
-import { Config, GlobalConfig } from "../config.js";
-import { Ntfy } from "../utils/ntfy.js";
-export type CopyRunOptions = {
+import { Action } from "./base.js";
+export type CopyOptions = {
     packages?: string[];
     source: string;
     targets: string[];
+    prune?: boolean;
 };
-export declare class Copy {
-    readonly config: Config;
-    readonly global?: GlobalConfig | undefined;
-    readonly ntfy: Ntfy;
-    protected verbose: boolean | undefined;
-    constructor(config: Config, global?: GlobalConfig | undefined);
-    private findRepo;
-    run(options: CopyRunOptions): Promise<void>;
+export declare class Copy extends Action {
+    protected initializedRepos: Set<string>;
+    private findSnapshots;
+    private findPackageTag;
+    runSingle(options: {
+        source: string;
+        target: string;
+        snapshot: {
+            id: string;
+            short_id: string;
+            tags: string[];
+        };
+    }): Promise<number>;
+    run(options: CopyOptions): Promise<void>;
 }